From dd16a65b36939d0eda7d7ba1d8db61201682baa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Tue, 13 Jan 2026 15:13:16 -0800 Subject: [PATCH 1/5] Added multichannel PReLU --- NAM/activations.cpp | 4 +- NAM/activations.h | 60 +++++++++++++++++++ tools/run_tests.cpp | 5 ++ tools/test/test_activations.cpp | 101 ++++++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 1 deletion(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index daa05c57..304ce119 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -5,6 +5,7 @@ nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFa nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh(); nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU(); nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU(0.01); //FIXME does not parameterize LeakyReLU +nam::activations::ActivationPReLU _PRELU = nam::activations::ActivationPReLU(0.01); //Same as leaky ReLU by default nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid(); nam::activations::ActivationSwish _SWISH = nam::activations::ActivationSwish(); nam::activations::ActivationHardSwish _HARD_SWISH = nam::activations::ActivationHardSwish(); @@ -15,7 +16,8 @@ bool nam::activations::Activation::using_fast_tanh = false; std::unordered_map nam::activations::Activation::_activations = { {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}, - {"SiLU", &_SWISH}, {"Hardswish", &_HARD_SWISH}, {"LeakyHardtanh", &_LEAKY_HARD_TANH}}; + {"SiLU", &_SWISH}, {"Hardswish", &_HARD_SWISH}, {"LeakyHardtanh", &_LEAKY_HARD_TANH}, + {"PReLU", &_PRELU}}; nam::activations::Activation* tanh_bak = nullptr; nam::activations::Activation* sigmoid_bak = nullptr; diff --git a/NAM/activations.h b/NAM/activations.h index d8a5591f..f0fb4d16 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -191,6 +191,66 @@ class ActivationLeakyReLU : public Activation float negative_slope = 0.01; }; +class ActivationPReLU : public Activation +{ +public: + ActivationPReLU() = default; + ActivationPReLU(float ns) { + negative_slopes.clear(); + negative_slopes.push_back(ns); + } + ActivationPReLU(std::vector ns) { + negative_slopes = ns; + } + + void apply(Eigen::MatrixXf& matrix) override + { + // Matrix is organized as (time_steps, channels) + int n_channels = negative_slopes.size(); + int actual_channels = matrix.cols(); + + // Apply each negative slope to its corresponding channel + for (int channel = 0; channel < std::min(n_channels, actual_channels); channel++) + { + // Apply the negative slope to all time steps in this channel + for (int time_step = 0; time_step < matrix.rows(); time_step++) + { + matrix(time_step, channel) = leaky_relu(matrix(time_step, channel), negative_slopes[channel]); + } + } + + // For any remaining channels beyond what we have slopes for, use the last slope + if (actual_channels > n_channels && n_channels > 0) + { + for (int channel = n_channels; channel < actual_channels; channel++) + { + for (int time_step = 0; time_step < matrix.rows(); time_step++) + { + matrix(time_step, channel) = leaky_relu(matrix(time_step, channel), negative_slopes.back()); + } + } + } + } + + void apply(float* data, long size) override + { + // Fallback implementation for when we don't have matrix dimensions + // This is less efficient and doesn't properly handle per-channel slopes + // but provides basic functionality + if (!negative_slopes.empty()) + { + float slope = negative_slopes[0]; // Use first slope as fallback + for (long pos = 0; pos < size; pos++) + { + data[pos] = leaky_relu(data[pos], slope); + } + } + } +private: + std::vector negative_slopes; +}; + + class ActivationSigmoid : public Activation { public: diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 8d622d8c..d7ad9cda 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -21,6 +21,11 @@ int main() test_activations::TestLeakyReLU::test_get_by_init(); test_activations::TestLeakyReLU::test_get_by_str(); + test_activations::TestPReLU::test_core_function(); + test_activations::TestPReLU::test_per_channel_behavior(); + test_activations::TestPReLU::test_fallback_behavior(); + test_activations::TestPReLU::test_single_slope(); + test_dsp::test_construct(); test_dsp::test_get_input_level(); test_dsp::test_get_output_level(); diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index c39b040b..151ecc7f 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include "NAM/activations.h" @@ -119,4 +120,104 @@ class TestLeakyReLU } }; }; +class TestPReLU +{ +public: + static void test_core_function() + { + // Test the basic leaky_relu function that PReLU uses + auto TestCase = [](float input, float slope, float expectedOutput) { + float actualOutput = nam::activations::leaky_relu(input, slope); + assert(actualOutput == expectedOutput); + }; + + // A few snapshot tests + TestCase(0.0f, 0.01f, 0.0f); + TestCase(1.0f, 0.01f, 1.0f); + TestCase(-1.0f, 0.01f, -0.01f); + TestCase(-1.0f, 0.05f, -0.05f); // Different slope + } + + static void test_per_channel_behavior() + { + // Test that different slopes are applied to different channels + Eigen::MatrixXf data(3, 2); // 3 time steps, 2 channels + + // Initialize with some test data + data << -1.0f, -2.0f, + 0.5f, -0.5f, + 1.0f, 0.0f; + + // Create PReLU with different slopes for each channel + std::vector slopes = {0.01f, 0.05f}; // slope 0.01 for channel 0, 0.05 for channel 1 + nam::activations::ActivationPReLU prelu(slopes); + + // Apply the activation + prelu.apply(data); + + // Verify the results + // Channel 0 (slope = 0.01): + assert(fabs(data(0, 0) - (-0.01f)) < 1e-6); // -1.0 * 0.01 = -0.01 + assert(fabs(data(1, 0) - 0.5f) < 1e-6); // 0.5 (positive, unchanged) + assert(fabs(data(2, 0) - 1.0f) < 1e-6); // 1.0 (positive, unchanged) + + // Channel 1 (slope = 0.05): + assert(fabs(data(0, 1) - (-0.10f)) < 1e-6); // -2.0 * 0.05 = -0.10 + assert(fabs(data(1, 1) - (-0.025f)) < 1e-6); // -0.5 * 0.05 = -0.025 + assert(fabs(data(2, 1) - 0.0f) < 1e-6); // 0.0 (unchanged) + } + + static void test_fallback_behavior() + { + // Test the fallback behavior when we have more channels than slopes + Eigen::MatrixXf data(2, 3); // 2 time steps, 3 channels + + // Initialize with test data + data << -1.0f, -1.0f, -1.0f, + 1.0f, 1.0f, 1.0f; + + // Create PReLU with only 2 slopes for 3 channels + std::vector slopes = {0.01f, 0.05f}; + nam::activations::ActivationPReLU prelu(slopes); + + // Apply the activation + prelu.apply(data); + + // Verify the results + // Channel 0 (slope = 0.01): + assert(fabs(data(0, 0) - (-0.01f)) < 1e-6); + assert(fabs(data(1, 0) - 1.0f) < 1e-6); + + // Channel 1 (slope = 0.05): + assert(fabs(data(0, 1) - (-0.05f)) < 1e-6); + assert(fabs(data(1, 1) - 1.0f) < 1e-6); + + // Channel 2 should use the last slope (0.05): + assert(fabs(data(0, 2) - (-0.05f)) < 1e-6); + assert(fabs(data(1, 2) - 1.0f) < 1e-6); + } + + static void test_single_slope() + { + // Test behavior when constructed with a single slope + Eigen::MatrixXf data(2, 2); // 2 time steps, 2 channels + + // Initialize with test data + data << -1.0f, -2.0f, + 0.5f, -0.5f; + + // Create PReLU with single slope + nam::activations::ActivationPReLU prelu(0.02f); + + // Apply the activation + prelu.apply(data); + + // Both channels should use the same slope (0.02) + assert(fabs(data(0, 0) - (-0.02f)) < 1e-6); // -1.0 * 0.02 = -0.02 + assert(fabs(data(0, 1) - (-0.04f)) < 1e-6); // -2.0 * 0.02 = -0.04 + assert(fabs(data(1, 0) - 0.5f) < 1e-6); // 0.5 (positive, unchanged) + assert(fabs(data(1, 1) - (-0.01f)) < 1e-6); // -0.5 * 0.02 = -0.01 + } +}; + }; // namespace test_activations From 05becca5cc838140f8cd3c5405bf6b48b2e9f37f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Tue, 13 Jan 2026 16:43:39 -0800 Subject: [PATCH 2/5] Fixed transposed inputs, behaviour when missing channels --- NAM/activations.h | 29 +++++-------- tools/run_tests.cpp | 3 +- tools/test/test_activations.cpp | 74 +++++++++++---------------------- 3 files changed, 37 insertions(+), 69 deletions(-) diff --git a/NAM/activations.h b/NAM/activations.h index f0fb4d16..ddb051e4 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -205,38 +205,29 @@ class ActivationPReLU : public Activation void apply(Eigen::MatrixXf& matrix) override { - // Matrix is organized as (time_steps, channels) + // Matrix is organized as (channels, time_steps) int n_channels = negative_slopes.size(); - int actual_channels = matrix.cols(); + int actual_channels = matrix.rows(); + if (actual_channels > n_channels) + { + throw std::runtime_error("Number of channels in PReLU activation different from input matrix"); + } + // Apply each negative slope to its corresponding channel for (int channel = 0; channel < std::min(n_channels, actual_channels); channel++) { // Apply the negative slope to all time steps in this channel for (int time_step = 0; time_step < matrix.rows(); time_step++) { - matrix(time_step, channel) = leaky_relu(matrix(time_step, channel), negative_slopes[channel]); - } - } - - // For any remaining channels beyond what we have slopes for, use the last slope - if (actual_channels > n_channels && n_channels > 0) - { - for (int channel = n_channels; channel < actual_channels; channel++) - { - for (int time_step = 0; time_step < matrix.rows(); time_step++) - { - matrix(time_step, channel) = leaky_relu(matrix(time_step, channel), negative_slopes.back()); - } + matrix(channel, time_step) = leaky_relu(matrix(channel, time_step), negative_slopes[channel]); } } } void apply(float* data, long size) override { - // Fallback implementation for when we don't have matrix dimensions - // This is less efficient and doesn't properly handle per-channel slopes - // but provides basic functionality + // Fallback that operates like leaky_relu, should not be used as it's a waste of a vector for one element if (!negative_slopes.empty()) { float slope = negative_slopes[0]; // Use first slope as fallback @@ -244,6 +235,8 @@ class ActivationPReLU : public Activation { data[pos] = leaky_relu(data[pos], slope); } + } else { + throw std::runtime_error("negative_slopes not initialized"); } } private: diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index d7ad9cda..8b7b0ea5 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -23,8 +23,7 @@ int main() test_activations::TestPReLU::test_core_function(); test_activations::TestPReLU::test_per_channel_behavior(); - test_activations::TestPReLU::test_fallback_behavior(); - test_activations::TestPReLU::test_single_slope(); + test_activations::TestPReLU::test_wrong_number_of_channels(); test_dsp::test_construct(); test_dsp::test_get_input_level(); diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index 151ecc7f..c74abe01 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -141,12 +141,11 @@ class TestPReLU static void test_per_channel_behavior() { // Test that different slopes are applied to different channels - Eigen::MatrixXf data(3, 2); // 3 time steps, 2 channels + Eigen::MatrixXf data(2, 3); // 2 channels, 3 time steps // Initialize with some test data - data << -1.0f, -2.0f, - 0.5f, -0.5f, - 1.0f, 0.0f; + data << -1.0f, 0.5f, 1.0f, + -2.0f, -0.5f, 0.0f; // Create PReLU with different slopes for each channel std::vector slopes = {0.01f, 0.05f}; // slope 0.01 for channel 0, 0.05 for channel 1 @@ -158,66 +157,43 @@ class TestPReLU // Verify the results // Channel 0 (slope = 0.01): assert(fabs(data(0, 0) - (-0.01f)) < 1e-6); // -1.0 * 0.01 = -0.01 - assert(fabs(data(1, 0) - 0.5f) < 1e-6); // 0.5 (positive, unchanged) - assert(fabs(data(2, 0) - 1.0f) < 1e-6); // 1.0 (positive, unchanged) + assert(fabs(data(0, 1) - 0.5f) < 1e-6); // 0.5 (positive, unchanged) + assert(fabs(data(0, 2) - 1.0f) < 1e-6); // 1.0 (positive, unchanged) // Channel 1 (slope = 0.05): - assert(fabs(data(0, 1) - (-0.10f)) < 1e-6); // -2.0 * 0.05 = -0.10 + assert(fabs(data(1, 0) - (-0.10f)) < 1e-6); // -2.0 * 0.05 = -0.10 assert(fabs(data(1, 1) - (-0.025f)) < 1e-6); // -0.5 * 0.05 = -0.025 - assert(fabs(data(2, 1) - 0.0f) < 1e-6); // 0.0 (unchanged) + assert(fabs(data(1, 2) - 0.0f) < 1e-6); // 0.0 (unchanged) } - static void test_fallback_behavior() + static void test_wrong_number_of_channels() { - // Test the fallback behavior when we have more channels than slopes - Eigen::MatrixXf data(2, 3); // 2 time steps, 3 channels + // Test that we fail when we have more channels than slopes + Eigen::MatrixXf data(3, 2); // 3 channels, 2 time steps // Initialize with test data - data << -1.0f, -1.0f, -1.0f, - 1.0f, 1.0f, 1.0f; + data << -1.0f, -1.0f, + -1.0f, 1.0f, + 1.0f, 1.0f; // Create PReLU with only 2 slopes for 3 channels std::vector slopes = {0.01f, 0.05f}; nam::activations::ActivationPReLU prelu(slopes); // Apply the activation - prelu.apply(data); - - // Verify the results - // Channel 0 (slope = 0.01): - assert(fabs(data(0, 0) - (-0.01f)) < 1e-6); - assert(fabs(data(1, 0) - 1.0f) < 1e-6); - - // Channel 1 (slope = 0.05): - assert(fabs(data(0, 1) - (-0.05f)) < 1e-6); - assert(fabs(data(1, 1) - 1.0f) < 1e-6); - - // Channel 2 should use the last slope (0.05): - assert(fabs(data(0, 2) - (-0.05f)) < 1e-6); - assert(fabs(data(1, 2) - 1.0f) < 1e-6); - } + bool caught = false; + try + { + prelu.apply(data); + } catch (const std::runtime_error& e) { + caught = true; + } catch (...) { + + } + + assert(caught == true); + } - static void test_single_slope() - { - // Test behavior when constructed with a single slope - Eigen::MatrixXf data(2, 2); // 2 time steps, 2 channels - - // Initialize with test data - data << -1.0f, -2.0f, - 0.5f, -0.5f; - - // Create PReLU with single slope - nam::activations::ActivationPReLU prelu(0.02f); - - // Apply the activation - prelu.apply(data); - - // Both channels should use the same slope (0.02) - assert(fabs(data(0, 0) - (-0.02f)) < 1e-6); // -1.0 * 0.02 = -0.02 - assert(fabs(data(0, 1) - (-0.04f)) < 1e-6); // -2.0 * 0.02 = -0.04 - assert(fabs(data(1, 0) - 0.5f) < 1e-6); // 0.5 (positive, unchanged) - assert(fabs(data(1, 1) - (-0.01f)) < 1e-6); // -0.5 * 0.02 = -0.01 - } }; }; // namespace test_activations From 285763af86634c13c924fe97a44354ec4a2b2e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 09:56:55 -0800 Subject: [PATCH 3/5] Addressing comments on runtime checks and removing apply(float *) implementation --- NAM/activations.h | 23 ++++------------------- tools/test/test_activations.cpp | 2 +- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/NAM/activations.h b/NAM/activations.h index ddb051e4..f05ecadb 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -209,10 +209,9 @@ class ActivationPReLU : public Activation int n_channels = negative_slopes.size(); int actual_channels = matrix.rows(); - if (actual_channels > n_channels) - { - throw std::runtime_error("Number of channels in PReLU activation different from input matrix"); - } + // NOTE: check not done during runtime on release builds + // model loader should make sure dimensions match + assert(actual_channels == n_channels); // Apply each negative slope to its corresponding channel for (int channel = 0; channel < std::min(n_channels, actual_channels); channel++) @@ -224,21 +223,7 @@ class ActivationPReLU : public Activation } } } - - void apply(float* data, long size) override - { - // Fallback that operates like leaky_relu, should not be used as it's a waste of a vector for one element - if (!negative_slopes.empty()) - { - float slope = negative_slopes[0]; // Use first slope as fallback - for (long pos = 0; pos < size; pos++) - { - data[pos] = leaky_relu(data[pos], slope); - } - } else { - throw std::runtime_error("negative_slopes not initialized"); - } - } + private: std::vector negative_slopes; }; diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index c74abe01..4b2b76e7 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -191,7 +191,7 @@ class TestPReLU } - assert(caught == true); + assert(caught); } }; From a6b059206e17129a63a0c3258076503de3373e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 10:28:45 -0800 Subject: [PATCH 4/5] Formatting --- NAM/activations.cpp | 32 ++++++--- NAM/activations.h | 122 ++++++++++++++++++-------------- tools/test/test_activations.cpp | 43 ++++++----- tools/test/test_fast_lut.cpp | 46 ++++++------ 4 files changed, 134 insertions(+), 109 deletions(-) diff --git a/NAM/activations.cpp b/NAM/activations.cpp index 304ce119..3db6024b 100644 --- a/NAM/activations.cpp +++ b/NAM/activations.cpp @@ -4,8 +4,9 @@ nam::activations::ActivationTanh _TANH = nam::activations::ActivationTanh(); nam::activations::ActivationFastTanh _FAST_TANH = nam::activations::ActivationFastTanh(); nam::activations::ActivationHardTanh _HARD_TANH = nam::activations::ActivationHardTanh(); nam::activations::ActivationReLU _RELU = nam::activations::ActivationReLU(); -nam::activations::ActivationLeakyReLU _LEAKY_RELU = nam::activations::ActivationLeakyReLU(0.01); //FIXME does not parameterize LeakyReLU -nam::activations::ActivationPReLU _PRELU = nam::activations::ActivationPReLU(0.01); //Same as leaky ReLU by default +nam::activations::ActivationLeakyReLU _LEAKY_RELU = + nam::activations::ActivationLeakyReLU(0.01); // FIXME does not parameterize LeakyReLU +nam::activations::ActivationPReLU _PRELU = nam::activations::ActivationPReLU(0.01); // Same as leaky ReLU by default nam::activations::ActivationSigmoid _SIGMOID = nam::activations::ActivationSigmoid(); nam::activations::ActivationSwish _SWISH = nam::activations::ActivationSwish(); nam::activations::ActivationHardSwish _HARD_SWISH = nam::activations::ActivationHardSwish(); @@ -14,8 +15,8 @@ nam::activations::ActivationLeakyHardTanh _LEAKY_HARD_TANH = nam::activations::A bool nam::activations::Activation::using_fast_tanh = false; std::unordered_map nam::activations::Activation::_activations = { - {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, - {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}, + {"Tanh", &_TANH}, {"Hardtanh", &_HARD_TANH}, {"Fasttanh", &_FAST_TANH}, + {"ReLU", &_RELU}, {"LeakyReLU", &_LEAKY_RELU}, {"Sigmoid", &_SIGMOID}, {"SiLU", &_SWISH}, {"Hardswish", &_HARD_SWISH}, {"LeakyHardtanh", &_LEAKY_HARD_TANH}, {"PReLU", &_PRELU}}; @@ -54,13 +55,18 @@ void nam::activations::Activation::disable_fast_tanh() void nam::activations::Activation::enable_lut(std::string function_name, float min, float max, std::size_t n_points) { std::function fn; - if (function_name == "Tanh"){ + if (function_name == "Tanh") + { fn = [](float x) { return std::tanh(x); }; tanh_bak = _activations["Tanh"]; - } else if (function_name == "Sigmoid") { + } + else if (function_name == "Sigmoid") + { fn = sigmoid; sigmoid_bak = _activations["Sigmoid"]; - } else { + } + else + { throw std::runtime_error("Tried to enable LUT for a function other than Tanh or Sigmoid"); } FastLUTActivation lut_activation(min, max, n_points, fn); @@ -69,12 +75,16 @@ void nam::activations::Activation::enable_lut(std::string function_name, float m void nam::activations::Activation::disable_lut(std::string function_name) { - if (function_name == "Tanh"){ + if (function_name == "Tanh") + { _activations["Tanh"] = tanh_bak; - } else if (function_name == "Sigmoid") { + } + else if (function_name == "Sigmoid") + { _activations["Sigmoid"] = sigmoid_bak; - } else { + } + else + { throw std::runtime_error("Tried to disable LUT for a function other than Tanh or Sigmoid"); } } - diff --git a/NAM/activations.h b/NAM/activations.h index f05ecadb..3e77614f 100644 --- a/NAM/activations.h +++ b/NAM/activations.h @@ -28,11 +28,16 @@ inline float hard_tanh(float x) inline float leaky_hardtanh(float x, float min_val, float max_val, float min_slope, float max_slope) { - if (x < min_val) { + if (x < min_val) + { return (x - min_val) * min_slope + min_val; - } else if (x > max_val) { + } + else if (x > max_val) + { return (x - max_val) * max_slope + max_val; - } else { + } + else + { return x; } } @@ -50,7 +55,7 @@ inline float fast_sigmoid(const float x) { return 0.5f * (fast_tanh(x * 0.5f) + 1.0f); } - + inline float leaky_relu(float x, float negative_slope) { return x > 0.0f ? x : negative_slope * x; @@ -68,12 +73,17 @@ inline float swish(float x) inline float hardswish(float x) { - if (x <= -3.0) { + if (x <= -3.0) + { return 0; - } else if (x >= 3.0) { + } + else if (x >= 3.0) + { return x; - } else { - return x * (x + 3.0)/6.0; + } + else + { + return x * (x + 3.0) / 6.0; } } @@ -129,7 +139,8 @@ class ActivationLeakyHardTanh : public Activation { public: ActivationLeakyHardTanh() = default; - ActivationLeakyHardTanh(float min_val_, float max_val_, float min_slope_, float max_slope_) { + ActivationLeakyHardTanh(float min_val_, float max_val_, float min_slope_, float max_slope_) + { min_val = min_val_; max_val = max_val_; min_slope = min_slope_; @@ -142,6 +153,7 @@ class ActivationLeakyHardTanh : public Activation data[pos] = leaky_hardtanh(data[pos], min_val, max_val, min_slope, max_slope); } } + private: float min_val = -1.0; float max_val = 1.0; @@ -177,9 +189,7 @@ class ActivationLeakyReLU : public Activation { public: ActivationLeakyReLU() = default; - ActivationLeakyReLU(float ns) { - negative_slope = ns; - } + ActivationLeakyReLU(float ns) { negative_slope = ns; } void apply(float* data, long size) override { for (long pos = 0; pos < size; pos++) @@ -187,6 +197,7 @@ class ActivationLeakyReLU : public Activation data[pos] = leaky_relu(data[pos], negative_slope); } } + private: float negative_slope = 0.01; }; @@ -195,20 +206,19 @@ class ActivationPReLU : public Activation { public: ActivationPReLU() = default; - ActivationPReLU(float ns) { + ActivationPReLU(float ns) + { negative_slopes.clear(); negative_slopes.push_back(ns); } - ActivationPReLU(std::vector ns) { - negative_slopes = ns; - } - + ActivationPReLU(std::vector ns) { negative_slopes = ns; } + void apply(Eigen::MatrixXf& matrix) override { // Matrix is organized as (channels, time_steps) int n_channels = negative_slopes.size(); int actual_channels = matrix.rows(); - + // NOTE: check not done during runtime on release builds // model loader should make sure dimensions match assert(actual_channels == n_channels); @@ -268,46 +278,54 @@ class ActivationHardSwish : public Activation class FastLUTActivation : public Activation { public: - FastLUTActivation(float min_x, float max_x, std::size_t size, std::function f) - : min_x_(min_x), max_x_(max_x), size_(size) { - - step_ = (max_x - min_x) / (size - 1); - inv_step_ = 1.0f / step_; - table_.reserve(size); - - for (std::size_t i = 0; i < size; ++i) { - table_.push_back(f(min_x + i * step_)); - } - } + FastLUTActivation(float min_x, float max_x, std::size_t size, std::function f) + : min_x_(min_x) + , max_x_(max_x) + , size_(size) + { - // Fast lookup with linear interpolation - inline float lookup(float x) const { - // Clamp input to range - x = std::clamp(x, min_x_, max_x_); - - // Calculate float index - float f_idx = (x - min_x_) * inv_step_; - std::size_t i = static_cast(f_idx); - - // Handle edge case at max_x_ - if (i >= size_ - 1) return table_.back(); - - // Linear interpolation: y = y0 + (y1 - y0) * fractional_part - float frac = f_idx - static_cast(i); - return table_[i] + (table_[i + 1] - table_[i]) * frac; + step_ = (max_x - min_x) / (size - 1); + inv_step_ = 1.0f / step_; + table_.reserve(size); + + for (std::size_t i = 0; i < size; ++i) + { + table_.push_back(f(min_x + i * step_)); } + } + + // Fast lookup with linear interpolation + inline float lookup(float x) const + { + // Clamp input to range + x = std::clamp(x, min_x_, max_x_); + + // Calculate float index + float f_idx = (x - min_x_) * inv_step_; + std::size_t i = static_cast(f_idx); + + // Handle edge case at max_x_ + if (i >= size_ - 1) + return table_.back(); + + // Linear interpolation: y = y0 + (y1 - y0) * fractional_part + float frac = f_idx - static_cast(i); + return table_[i] + (table_[i + 1] - table_[i]) * frac; + } - // Vector application (Batch processing) - void apply(std::vector& data) const { - for (float& val : data) { - val = lookup(val); - } + // Vector application (Batch processing) + void apply(std::vector& data) const + { + for (float& val : data) + { + val = lookup(val); } + } private: - float min_x_, max_x_, step_, inv_step_; - size_t size_; - std::vector table_; + float min_x_, max_x_, step_, inv_step_; + size_t size_; + std::vector table_; }; }; // namespace activations diff --git a/tools/test/test_activations.cpp b/tools/test/test_activations.cpp index 4b2b76e7..e9f7a86c 100644 --- a/tools/test/test_activations.cpp +++ b/tools/test/test_activations.cpp @@ -130,7 +130,7 @@ class TestPReLU float actualOutput = nam::activations::leaky_relu(input, slope); assert(actualOutput == expectedOutput); }; - + // A few snapshot tests TestCase(0.0f, 0.01f, 0.0f); TestCase(1.0f, 0.01f, 1.0f); @@ -142,58 +142,57 @@ class TestPReLU { // Test that different slopes are applied to different channels Eigen::MatrixXf data(2, 3); // 2 channels, 3 time steps - + // Initialize with some test data - data << -1.0f, 0.5f, 1.0f, - -2.0f, -0.5f, 0.0f; - + data << -1.0f, 0.5f, 1.0f, -2.0f, -0.5f, 0.0f; + // Create PReLU with different slopes for each channel std::vector slopes = {0.01f, 0.05f}; // slope 0.01 for channel 0, 0.05 for channel 1 nam::activations::ActivationPReLU prelu(slopes); - + // Apply the activation prelu.apply(data); - + // Verify the results // Channel 0 (slope = 0.01): assert(fabs(data(0, 0) - (-0.01f)) < 1e-6); // -1.0 * 0.01 = -0.01 - assert(fabs(data(0, 1) - 0.5f) < 1e-6); // 0.5 (positive, unchanged) - assert(fabs(data(0, 2) - 1.0f) < 1e-6); // 1.0 (positive, unchanged) - + assert(fabs(data(0, 1) - 0.5f) < 1e-6); // 0.5 (positive, unchanged) + assert(fabs(data(0, 2) - 1.0f) < 1e-6); // 1.0 (positive, unchanged) + // Channel 1 (slope = 0.05): assert(fabs(data(1, 0) - (-0.10f)) < 1e-6); // -2.0 * 0.05 = -0.10 assert(fabs(data(1, 1) - (-0.025f)) < 1e-6); // -0.5 * 0.05 = -0.025 - assert(fabs(data(1, 2) - 0.0f) < 1e-6); // 0.0 (unchanged) + assert(fabs(data(1, 2) - 0.0f) < 1e-6); // 0.0 (unchanged) } static void test_wrong_number_of_channels() { // Test that we fail when we have more channels than slopes Eigen::MatrixXf data(3, 2); // 3 channels, 2 time steps - + // Initialize with test data - data << -1.0f, -1.0f, - -1.0f, 1.0f, - 1.0f, 1.0f; - + data << -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f; + // Create PReLU with only 2 slopes for 3 channels std::vector slopes = {0.01f, 0.05f}; nam::activations::ActivationPReLU prelu(slopes); - + // Apply the activation bool caught = false; try { prelu.apply(data); - } catch (const std::runtime_error& e) { + } + catch (const std::runtime_error& e) + { caught = true; - } catch (...) { - + } + catch (...) + { } assert(caught); - } - + } }; }; // namespace test_activations diff --git a/tools/test/test_fast_lut.cpp b/tools/test/test_fast_lut.cpp index 29aee29a..f7b1d83b 100644 --- a/tools/test/test_fast_lut.cpp +++ b/tools/test/test_fast_lut.cpp @@ -5,35 +5,33 @@ #include "NAM/activations.h" -namespace test_lut { +namespace test_lut +{ -float sigmoid(float x) { - return 1.0f / (1.0f + std::exp(-x)); +float sigmoid(float x) +{ + return 1.0f / (1.0f + std::exp(-x)); } class TestFastLUT { - public: - static void test_sigmoid() - { - // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples - nam::activations::FastLUTActivation lut_sigmoid(-8.0f, 8.0f, 1024, [](float x) { - return 1.0f / (1.0f + expf(-x)); - }); +public: + static void test_sigmoid() + { + // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples + nam::activations::FastLUTActivation lut_sigmoid( + -8.0f, 8.0f, 1024, [](float x) { return 1.0f / (1.0f + expf(-x)); }); - float input = 1.25f; - assert(abs(sigmoid(input) - lut_sigmoid.lookup(input)) < 1e-3); - } - static void test_tanh() - { - // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples - nam::activations::FastLUTActivation lut_tanh(-8.0f, 8.0f, 1024, [](float x) { - return std::tanh(x); - }); + float input = 1.25f; + assert(abs(sigmoid(input) - lut_sigmoid.lookup(input)) < 1e-3); + } + static void test_tanh() + { + // create a lut for sigmoid from -8.0 to 8.0 with 1024 samples + nam::activations::FastLUTActivation lut_tanh(-8.0f, 8.0f, 1024, [](float x) { return std::tanh(x); }); - float input = 1.25f; - assert(abs(std::tanh(input) - lut_tanh.lookup(input)) < 1e-3); - } + float input = 1.25f; + assert(abs(std::tanh(input) - lut_tanh.lookup(input)) < 1e-3); + } }; -} - +} // namespace test_lut From 5324a37525692e2da5104793f20df463884b05b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Felipe=20Santos?= Date: Wed, 14 Jan 2026 11:48:08 -0800 Subject: [PATCH 5/5] Disabling failing test (it's handled by an assert now) --- tools/run_tests.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests.cpp b/tools/run_tests.cpp index 8b7b0ea5..928bd9fc 100644 --- a/tools/run_tests.cpp +++ b/tools/run_tests.cpp @@ -23,7 +23,8 @@ int main() test_activations::TestPReLU::test_core_function(); test_activations::TestPReLU::test_per_channel_behavior(); - test_activations::TestPReLU::test_wrong_number_of_channels(); + // This is enforced by an assert so it doesn't need to be tested + //test_activations::TestPReLU::test_wrong_number_of_channels(); test_dsp::test_construct(); test_dsp::test_get_input_level();