[T1-1-1] S-hhhhh

include/infiniop.h

@@ -17,5 +17,14 @@
 #include "infiniop/ops/sub.h"
 #include "infiniop/ops/swiglu.h"
 #include "infiniop/tensor_descriptor.h"
+#include "infiniop/ops/exp.h"
+#include "infiniop/ops/sin.h"
+#include "infiniop/ops/cos.h"
+#include "infiniop/ops/tanh.h"
+#include "infiniop/ops/sigmoid_backward.h"
+#include "infiniop/ops/hardswish.h"
+#include "infiniop/ops/where.h"
+#include "infiniop/ops/leaky_relu.h"
+#include "infiniop/ops/cast.h"
 
 #endif // __INFINIOP_API_H__

include/infiniop/ops/cast.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_CAST_API_H__
+#define __INFINIOP_CAST_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopCastDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateCastDescriptor(infiniopHandle_t handle,
+                                                         infiniopCastDescriptor_t *desc_ptr,
+                                                         infiniopTensorDescriptor_t output,
+                                                         infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetCastWorkspaceSize(infiniopCastDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopCast(infiniopCastDescriptor_t desc,
+                                         void *workspace,
+                                         size_t workspace_size,
+                                         void *output,
+                                         const void *input,
+                                         void *stream);
+
+__C __export infiniStatus_t infiniopDestroyCastDescriptor(infiniopCastDescriptor_t desc);
+
+#endif

include/infiniop/ops/cos.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_COS_API_H__
+#define __INFINIOP_COS_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopCosDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateCosDescriptor(infiniopHandle_t handle,
+                                                        infiniopCosDescriptor_t *desc_ptr,
+                                                        infiniopTensorDescriptor_t output,
+                                                        infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetCosWorkspaceSize(infiniopCosDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopCos(infiniopCosDescriptor_t desc,
+                                        void *workspace,
+                                        size_t workspace_size,
+                                        void *output,
+                                        const void *input,
+                                        void *stream);
+
+__C __export infiniStatus_t infiniopDestroyCosDescriptor(infiniopCosDescriptor_t desc);
+
+#endif

include/infiniop/ops/exp.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_EXP_API_H__
+#define __INFINIOP_EXP_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopExpDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateExpDescriptor(infiniopHandle_t handle,
+                                                        infiniopExpDescriptor_t *desc_ptr,
+                                                        infiniopTensorDescriptor_t output,
+                                                        infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetExpWorkspaceSize(infiniopExpDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopExp(infiniopExpDescriptor_t desc,
+                                        void *workspace,
+                                        size_t workspace_size,
+                                        void *output,
+                                        const void *input,
+                                        void *stream);
+
+__C __export infiniStatus_t infiniopDestroyExpDescriptor(infiniopExpDescriptor_t desc);
+
+#endif

include/infiniop/ops/hardswish.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_HARDSWISH_API_H__
+#define __INFINIOP_HARDSWISH_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopHardSwishDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateHardSwishDescriptor(infiniopHandle_t handle,
+                                                              infiniopHardSwishDescriptor_t *desc_ptr,
+                                                              infiniopTensorDescriptor_t output,
+                                                              infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetHardSwishWorkspaceSize(infiniopHardSwishDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopHardSwish(infiniopHardSwishDescriptor_t desc,
+                                              void *workspace,
+                                              size_t workspace_size,
+                                              void *output,
+                                              const void *input,
+                                              void *stream);
+
+__C __export infiniStatus_t infiniopDestroyHardSwishDescriptor(infiniopHardSwishDescriptor_t desc);
+
+#endif

include/infiniop/ops/leaky_relu.h

@@ -0,0 +1,25 @@
+#ifndef __INFINIOP_LEAKY_RELU_API_H__
+#define __INFINIOP_LEAKY_RELU_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopLeakyReLUDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateLeakyReLUDescriptor(infiniopHandle_t handle,
+                                                              infiniopLeakyReLUDescriptor_t *desc_ptr,
+                                                              infiniopTensorDescriptor_t output,
+                                                              infiniopTensorDescriptor_t input,
+                                                              float negative_slope);
+
+__C __export infiniStatus_t infiniopGetLeakyReLUWorkspaceSize(infiniopLeakyReLUDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopLeakyReLU(infiniopLeakyReLUDescriptor_t desc,
+                                              void *workspace,
+                                              size_t workspace_size,
+                                              void *output,
+                                              const void *input,
+                                              void *stream);
+
+__C __export infiniStatus_t infiniopDestroyLeakyReLUDescriptor(infiniopLeakyReLUDescriptor_t desc);
+
+#endif

include/infiniop/ops/sigmoid_backward.h

@@ -0,0 +1,26 @@
+#ifndef __INFINIOP_SIGMOID_BACKWARD_API_H__
+#define __INFINIOP_SIGMOID_BACKWARD_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopSigmoidBackwardDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateSigmoidBackwardDescriptor(infiniopHandle_t handle,
+                                                                    infiniopSigmoidBackwardDescriptor_t *desc_ptr,
+                                                                    infiniopTensorDescriptor_t grad_input,
+                                                                    infiniopTensorDescriptor_t input,
+                                                                    infiniopTensorDescriptor_t grad_output);
+
+__C __export infiniStatus_t infiniopGetSigmoidBackwardWorkspaceSize(infiniopSigmoidBackwardDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopSigmoidBackward(infiniopSigmoidBackwardDescriptor_t desc,
+                                                   void *workspace,
+                                                   size_t workspace_size,
+                                                   void *grad_input,
+                                                   const void *input,
+                                                   const void *grad_output,
+                                                   void *stream);
+
+__C __export infiniStatus_t infiniopDestroySigmoidBackwardDescriptor(infiniopSigmoidBackwardDescriptor_t desc);
+
+#endif

include/infiniop/ops/sin.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_SIN_API_H__
+#define __INFINIOP_SIN_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopSinDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateSinDescriptor(infiniopHandle_t handle,
+                                                        infiniopSinDescriptor_t *desc_ptr,
+                                                        infiniopTensorDescriptor_t output,
+                                                        infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetSinWorkspaceSize(infiniopSinDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopSin(infiniopSinDescriptor_t desc,
+                                        void *workspace,
+                                        size_t workspace_size,
+                                        void *output,
+                                        const void *input,
+                                        void *stream);
+
+__C __export infiniStatus_t infiniopDestroySinDescriptor(infiniopSinDescriptor_t desc);
+
+#endif

include/infiniop/ops/tanh.h

@@ -0,0 +1,24 @@
+#ifndef __INFINIOP_TANH_API_H__
+#define __INFINIOP_TANH_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopTanhDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateTanhDescriptor(infiniopHandle_t handle,
+                                                         infiniopTanhDescriptor_t *desc_ptr,
+                                                         infiniopTensorDescriptor_t output,
+                                                         infiniopTensorDescriptor_t input);
+
+__C __export infiniStatus_t infiniopGetTanhWorkspaceSize(infiniopTanhDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopTanh(infiniopTanhDescriptor_t desc,
+                                         void *workspace,
+                                         size_t workspace_size,
+                                         void *output,
+                                         const void *input,
+                                         void *stream);
+
+__C __export infiniStatus_t infiniopDestroyTanhDescriptor(infiniopTanhDescriptor_t desc);
+
+#endif

include/infiniop/ops/where.h

@@ -0,0 +1,28 @@
+#ifndef __INFINIOP_WHERE_API_H__
+#define __INFINIOP_WHERE_API_H__
+
+#include "../operator_descriptor.h"
+
+typedef struct InfiniopDescriptor *infiniopWhereDescriptor_t;
+
+__C __export infiniStatus_t infiniopCreateWhereDescriptor(infiniopHandle_t handle,
+                                                          infiniopWhereDescriptor_t *desc_ptr,
+                                                          infiniopTensorDescriptor_t c,
+                                                          infiniopTensorDescriptor_t condition,
+                                                          infiniopTensorDescriptor_t a,
+                                                          infiniopTensorDescriptor_t b);
+
+__C __export infiniStatus_t infiniopGetWhereWorkspaceSize(infiniopWhereDescriptor_t desc, size_t *size);
+
+__C __export infiniStatus_t infiniopWhere(infiniopWhereDescriptor_t desc,
+                                          void *workspace,
+                                          size_t workspace_size,
+                                          void *c,
+                                          const void *condition,
+                                          const void *a,
+                                          const void *b,
+                                          void *stream);
+
+__C __export infiniStatus_t infiniopDestroyWhereDescriptor(infiniopWhereDescriptor_t desc);
+
+#endif

scripts/python_test.py

@@ -24,6 +24,15 @@ def run_tests(args):
         "rope.py",
         "sub.py",
         "swiglu.py",
+        "exp.py",
+        "sin.py",
+        "cos.py",
+        "tanh.py",
+        "sigmoid_backward.py",
+        "hardswish.py",
+        "where.py",
+        "leaky_relu.py",
+        "cast.py",
     ]:
         result = subprocess.run(
             f"python {test} {args} --debug", text=True, encoding="utf-8", shell=True

src/infiniop-test/include/ops.hpp

@@ -16,6 +16,15 @@ DECLARE_INFINIOP_TEST(add)
 DECLARE_INFINIOP_TEST(causal_softmax)
 DECLARE_INFINIOP_TEST(rearrange)
 DECLARE_INFINIOP_TEST(sub)
+DECLARE_INFINIOP_TEST(exp)
+DECLARE_INFINIOP_TEST(sin)
+DECLARE_INFINIOP_TEST(cos)
+DECLARE_INFINIOP_TEST(tanh)
+DECLARE_INFINIOP_TEST(sigmoid_backward)
+DECLARE_INFINIOP_TEST(hardswish)
+DECLARE_INFINIOP_TEST(where)
+DECLARE_INFINIOP_TEST(leakyrelu)
+DECLARE_INFINIOP_TEST(cast)
 
 #define REGISTER_INFINIOP_TEST(name)                      \
     {                                                     \
@@ -30,19 +39,28 @@ DECLARE_INFINIOP_TEST(sub)
 /*
  * Register all the tests here
  */
-#define TEST_BUILDER_MAPPINGS                  \
-    {                                          \
-        REGISTER_INFINIOP_TEST(gemm)           \
-        REGISTER_INFINIOP_TEST(random_sample)  \
-        REGISTER_INFINIOP_TEST(add)            \
-        REGISTER_INFINIOP_TEST(mul)            \
-        REGISTER_INFINIOP_TEST(clip)           \
-        REGISTER_INFINIOP_TEST(swiglu)         \
-        REGISTER_INFINIOP_TEST(rope)           \
-        REGISTER_INFINIOP_TEST(rms_norm)       \
-        REGISTER_INFINIOP_TEST(causal_softmax) \
-        REGISTER_INFINIOP_TEST(rearrange)      \
-        REGISTER_INFINIOP_TEST(sub)            \
+#define TEST_BUILDER_MAPPINGS                    \
+    {                                            \
+        REGISTER_INFINIOP_TEST(gemm)             \
+        REGISTER_INFINIOP_TEST(random_sample)    \
+        REGISTER_INFINIOP_TEST(add)              \
+        REGISTER_INFINIOP_TEST(mul)              \
+        REGISTER_INFINIOP_TEST(clip)             \
+        REGISTER_INFINIOP_TEST(swiglu)           \
+        REGISTER_INFINIOP_TEST(rope)             \
+        REGISTER_INFINIOP_TEST(rms_norm)         \
+        REGISTER_INFINIOP_TEST(causal_softmax)   \
+        REGISTER_INFINIOP_TEST(rearrange)        \
+        REGISTER_INFINIOP_TEST(sub)              \
+        REGISTER_INFINIOP_TEST(exp)              \
+        REGISTER_INFINIOP_TEST(sin)              \
+        REGISTER_INFINIOP_TEST(cos)              \
+        REGISTER_INFINIOP_TEST(tanh)             \
+        REGISTER_INFINIOP_TEST(sigmoid_backward) \
+        REGISTER_INFINIOP_TEST(hardswish)        \
+        REGISTER_INFINIOP_TEST(where)            \
+        REGISTER_INFINIOP_TEST(leakyrelu)        \
+        REGISTER_INFINIOP_TEST(cast)             \
     }
 
 namespace infiniop_test {
@@ -64,4 +82,4 @@ bool check_names(
 
 } // namespace infiniop_test
 
-#endif
+#endif

src/infiniop-test/include/test.hpp

@@ -47,18 +47,19 @@ std::vector<std::shared_ptr<Result>> runAllTests(
     const GGUFFileReader &,
     infiniDevice_t device, int device_id,
     size_t warm_ups, size_t iterations,
-    double rtol, double atol);
+    double rtol, double atol, bool equal_nan = false);
 
 // Run a single test read from a GGUF file
 std::shared_ptr<Result> runTest(
     const GGUFFileReader &,
     infiniDevice_t device, int device_id,
     size_t warm_ups, size_t iterations,
     double rtol, double atol,
-    size_t test_id);
+    size_t test_id,
+    bool equal_nan = false);
 
 // Check if two tensors are close within given tolerance
-void allClose(std::shared_ptr<Tensor> actual, std::shared_ptr<Tensor> expected, double rtol = 1e-3, double atol = 1e-3);
+void allClose(std::shared_ptr<Tensor> actual, std::shared_ptr<Tensor> expected, double rtol = 1e-3, double atol = 1e-3, bool equal_nan = false);
 
 // Check if two tensors are equal
 void allEqual(std::shared_ptr<Tensor> actual, std::shared_ptr<Tensor> expected);
@@ -85,13 +86,14 @@ class Test {
     namespace infiniop_test::name {                                           \
     class Test : public infiniop_test::base::Test {                           \
         double _rtol, _atol;                                                  \
+        bool _equal_nan;                                                      \
                                                                               \
     public:                                                                   \
         static std::string op_name() { return #name; }                        \
         static std::shared_ptr<Test> build(                                   \
             std::unordered_map<std::string, std::vector<uint8_t>> attributes, \
             std::unordered_map<std::string, std::shared_ptr<Tensor>> tensors, \
-            double, double);                                                  \
+            double, double, bool);                                            \
                                                                               \
         static std::vector<std::string> attribute_names();                    \
         static std::vector<std::string> tensor_names();                       \
@@ -109,15 +111,16 @@ class Test {
         struct Attributes;                                                    \
         Attributes *_attributes;                                              \
         Test() = delete;                                                      \
-        Test(double rtol, double atol) : _rtol(rtol), _atol(atol) {}          \
+        Test(double rtol, double atol, bool equal_nan = false)                \
+            : _rtol(rtol), _atol(atol), _equal_nan(equal_nan) {}              \
     };                                                                        \
     }
 
 namespace infiniop_test {
 using BuilderFunc = std::function<std::shared_ptr<infiniop_test::base::Test>(
     std::unordered_map<std::string, std::vector<uint8_t>>,
     std::unordered_map<std::string, std::shared_ptr<Tensor>>,
-    double, double)>;
+    double, double, bool)>;
 
 // Testcase Registry
 // Each testcase should provid a formatted builder, attribute names, and tensor names