Reuse math_brute_force ulp threshold in spir test

Spir test compares floating-point kernel results bit-by-bit with
correct results. However, for math_brute_force kernels, specification
does not ask for SPIR and OpenCL C path to match bit-to-bit. This patch
reuses ulps threshold from math_brute_force folder for math_brute_force
kernels in spir test.

Signed-off-by: Wenju He <wenju.he@intel.com>

test_conformance/math_brute_force/FunctionList.c

@@ -22,6 +22,34 @@
 
 #define STRINGIFY( _s)                  #_s
 
+// Only use ulps information in spir test
+#ifdef FUNCTION_LIST_ULPS_ONLY
+
+#define ENTRY(      _name, _ulp, _embedded_ulp, _rmode, _type )                 { STRINGIFY(_name), STRINGIFY(_name),                 {NULL}, {NULL}, _ulp, _ulp, _embedded_ulp, _rmode, _type }
+#define HALF_ENTRY( _name, _ulp, _embedded_ulp, _rmode, _type )                 { "half_" STRINGIFY(_name), "half_" STRINGIFY(_name), {NULL}, {NULL}, _ulp, _ulp, _embedded_ulp, _rmode, _type }
+#define OPERATOR_ENTRY(_name, _operator, _ulp, _embedded_ulp, _rmode, _type)    { STRINGIFY(_name), _operator,                        {NULL}, {NULL}, _ulp, _ulp, _embedded_ulp, _rmode, _type }
+#define unaryF                NULL
+#define i_unaryF              NULL
+#define unaryF_u              NULL
+#define macro_unaryF          NULL
+#define binaryF               NULL
+#define binaryF_nextafter     NULL
+#define binaryOperatorF       NULL
+#define binaryF_i             NULL
+#define macro_binaryF         NULL
+#define ternaryF              NULL
+#define unaryF_two_results    NULL
+#define unaryF_two_results_i  NULL
+#define binaryF_two_results_i NULL
+#define mad_function          NULL
+
+#define reference_sqrt        NULL
+#define reference_sqrtl       NULL
+#define reference_divide      NULL
+#define reference_dividel     NULL
+
+#else // FUNCTION_LIST_ULPS_ONLY
+
 #define ENTRY( _name, _ulp, _embedded_ulp, _rmode, _type )                           { STRINGIFY(_name), STRINGIFY(_name),                 {reference_##_name}, {reference_##_name##l}, _ulp, _ulp, _embedded_ulp, _rmode, _type }
 #define HALF_ENTRY( _name, _ulp, _embedded_ulp, _rmode, _type )                        { "half_" STRINGIFY(_name), "half_" STRINGIFY(_name), {reference_##_name}, {NULL},                    _ulp, _ulp, _embedded_ulp, _rmode, _type }
 #define OPERATOR_ENTRY(_name, _operator, _ulp, _embedded_ulp, _rmode, _type)        { STRINGIFY(_name), _operator,                        {reference_##_name}, {reference_##_name##l}, _ulp, _ulp, _embedded_ulp, _rmode, _type }
@@ -62,6 +90,7 @@ extern const vtbl _mad_tbl;             // float mad( float, float, float )
 #define binaryF_two_results_i  &_binary_two_results_i
 #define mad_function        &_mad_tbl
 
+#endif // FUNCTION_LIST_ULPS_ONLY
 
 const Func  functionList[] = {
                                     ENTRY( acos,                  4.0f,         4.0f,         FTZ_OFF,     unaryF),

test_conformance/math_brute_force/FunctionList.h

@@ -30,6 +30,10 @@
 #include "../../test_common/harness/compat.h"
 #include "../../test_common/harness/mt19937.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef union fptr
 {
     void    *p;
@@ -82,7 +86,7 @@ typedef struct Func
     float           double_ulps;
     float           float_embedded_ulps;
     int             ftz;
-    const vtbl      *vtbl;
+    const vtbl      *vtbl_ptr;
 }Func;
 
 
@@ -90,6 +94,9 @@ extern const Func  functionList[];
 
 extern const size_t functionListCount;
 
+#ifdef __cplusplus
+}
+#endif
 
 #endif
 

test_conformance/math_brute_force/main.c

@@ -232,7 +232,7 @@ int main (int argc, const char * argv[])
             if( gTestFloat )
             {
                 gTestCount++;
-                if( f->vtbl->TestFunc( f, d )  )
+                if( f->vtbl_ptr->TestFunc( f, d )  )
                 {
                     gFailCount++;
                     error++;
@@ -243,12 +243,12 @@ int main (int argc, const char * argv[])
 
 //            while(1)
             {
-                if( gHasDouble && NULL != f->vtbl->DoubleTestFunc && NULL != f->dfunc.p )
+                if( gHasDouble && NULL != f->vtbl_ptr->DoubleTestFunc && NULL != f->dfunc.p )
                 {
                     gTestCount++;
                     if( gTestFloat )
                         vlog( "    " );
-                    if( f->vtbl->DoubleTestFunc( f, d )  )
+                    if( f->vtbl_ptr->DoubleTestFunc( f, d )  )
                     {
                         gFailCount++;
                         error++;
@@ -259,7 +259,7 @@ int main (int argc, const char * argv[])
             }
 #if defined( __APPLE__ )
             {
-                if( gHasBasicDouble && NULL != f->vtbl->DoubleTestFunc && NULL != f->dfunc.p)
+                if( gHasBasicDouble && NULL != f->vtbl_ptr->DoubleTestFunc && NULL != f->dfunc.p)
                 {
                     int isBasicTest = 0;
                     for( j = 0; j < gNumBasicDoubleFuncs; j++ ) {
@@ -272,7 +272,7 @@ int main (int argc, const char * argv[])
                         gTestCount++;
                         if( gTestFloat )
                             vlog( "    " );
-                        if( f->vtbl->DoubleTestFunc( f, d )  )
+                        if( f->vtbl_ptr->DoubleTestFunc( f, d )  )
                         {
                             gFailCount++;
                             error++;

test_conformance/spir/CMakeLists.txt

@@ -3,6 +3,9 @@ function (install_spir_artifacts suite_name)
           COMPONENT OpenCLCTS)
 endfunction()
 
+# Import function list from math_brute_force
+add_definitions(-DFUNCTION_LIST_ULPS_ONLY)
+
 add_executable(
     conformance_test_spir
     main.cpp
@@ -10,6 +13,7 @@ add_executable(
     run_build_test.cpp
     run_services.cpp
     kernelargs.cpp
+    ../math_brute_force/FunctionList.c
     ../../test_common/harness/errorHelpers.c
     ../../test_common/harness/kernelHelpers.c
     ../../test_common/harness/mt19937.c

test_conformance/spir/kernelargs.h

@@ -110,7 +110,7 @@ public:
         return m_buffer;
     }
 
-    virtual bool compare( const KernelArg& rhs ) const
+    virtual bool compare( const KernelArg& rhs, float ulps ) const
     {
         if( m_argInfo != rhs.m_argInfo )
         {
@@ -133,21 +133,58 @@ public:
             return true;
         }
 
+        bool match = true;
         if( memcmp( m_buffer, rhs.m_buffer, m_size) )
         {
+            std::string typeName = m_argInfo.getTypeName();
             size_t compared = 0;
-            while (compared < m_size)
+            if (typeName.compare("float*") == 0)
             {
-                if ( *(((char*)m_buffer)+compared) != *(((char*)rhs.m_buffer)+compared) )
+                while (compared < m_size)
                 {
-                    std::cerr << std::endl << " difference is at offset " << compared << std::endl;
+                    float l = *(float*)(((char*)m_buffer)+compared);
-                    return false;
+                    float r = *(float*)(((char*)rhs.m_buffer)+compared);
+                    if (fabsf(Ulp_Error(l, r)) > ulps)
+                    {
+                        match = false;
+                        break;
+                    }
+                    compared += sizeof(float);
                 }
-                compared++;
+            }
+            else if (typeName.compare("double*") == 0)
+            {
+                while (compared < m_size)
+                {
+                    double l = *(double*)(((char*)m_buffer)+compared);
+                    double r = *(double*)(((char*)rhs.m_buffer)+compared);
+                    if (fabsf(Ulp_Error_Double(l, r)) > ulps)
+                    {
+                        match = false;
+                        break;
+                    }
+                    compared += sizeof(double);
+                }
+            }
+            else
+            {
+                while (compared < m_size)
+                {
+                    if ( *(((char*)m_buffer)+compared) != *(((char*)rhs.m_buffer)+compared) )
+                    {
+                        match = false;
+                        break;
+                    }
+                    compared++;
+                }
+            }
+            if (!match)
+            {
+                std::cerr << std::endl << " difference is at offset " << compared << std::endl;
             }
         }
 
-        return true;
+        return match;
     }
 
     virtual void readToHost(cl_command_queue queue)

test_conformance/spir/run_build_test.cpp

@@ -38,6 +38,7 @@
 #include "datagen.h"
 #include "run_services.h"
 #include "run_build_test.h"
+#include "../math_brute_force/FunctionList.h"
 #include <CL/cl.h>
 //
 // Task
@@ -262,7 +263,8 @@ size_t KernelEnumerator::size() const {
  Run the single test - run the test for both CL and SPIR versions of the kernel
  */
 static bool run_test(cl_context context, cl_command_queue queue, cl_program clprog,
-    cl_program bcprog, const std::string& kernel_name, std::string& err, const cl_device_id device)
+    cl_program bcprog, const std::string& kernel_name, std::string& err, const cl_device_id device,
+    float ulps)
 {
     WorkSizeInfo ws;
     TestResult cl_result;
@@ -274,7 +276,7 @@ static bool run_test(cl_context context, cl_command_queue queue, cl_program clpr
         // based on the kernel characteristics, we are generating and initializing the arguments for both phases (cl and bc executions)
         generate_kernel_data(context, kernel, ws, cl_result);
         bc_result.reset(cl_result.clone(context, ws, kernel, device));
-        assert (compare_results(cl_result, *bc_result) && "not equal?");
+        assert (compare_results(cl_result, *bc_result, ulps) && "not equal?");
         run_kernel( kernel, queue, ws, cl_result );
     }
     // now, run the single BC test
@@ -292,7 +294,7 @@ static bool run_test(cl_context context, cl_command_queue queue, cl_program clpr
     }
 
     // compare the results
-    if( !compare_results(cl_result, *bc_result) )
+    if( !compare_results(cl_result, *bc_result, ulps) )
     {
         err = " (result diff in kernel '" + kernel_name + "').";
         return false;
@@ -300,6 +302,37 @@ static bool run_test(cl_context context, cl_command_queue queue, cl_program clpr
     return true;
 }
 
+/**
+ Get the maximum relative error defined as ULP of floating-point math functions
+ */
+static float get_max_ulps(const char *test_name)
+{
+    float ulps = 0.f;
+    // Get ULP values from math_brute_force functionList
+    if (strstr(test_name, "math_kernel"))
+    {
+        for( size_t i = 0; i < functionListCount; i++ )
+        {
+            char name[64];
+            const Func *func = &functionList[ i ];
+            sprintf(name, ".%s_float", func->name);
+            if (strstr(test_name, name))
+            {
+                ulps = func->float_ulps;
+            }
+            else
+            {
+                sprintf(name, ".%s_double", func->name);
+                if (strstr(test_name, name))
+                {
+                    ulps = func->double_ulps;
+                }
+            }
+        }
+    }
+    return ulps;
+}
+
 TestRunner::TestRunner(EventHandler *success, EventHandler *failure,
                        const OclExtensions& devExt):
     m_successHandler(success), m_failureHandler(failure), m_devExt(&devExt) {}
@@ -321,6 +354,8 @@ bool TestRunner::runBuildTest(cl_device_id device, const char *folder,
 
     log_info("%s...\n", test_name);
 
+    float ulps = get_max_ulps(test_name);
+
     // Figure out whether the test can run on the device. If not, we skip it.
     const KhrSupport& khrDb = *KhrSupport::get(csvName);
     cl_bool images = khrDb.isImagesRequired(folder, test_name);
@@ -415,7 +450,7 @@ bool TestRunner::runBuildTest(cl_device_id device, const char *folder,
         std::string err;
         try
         {
-            bool success = run_test(context, queue, clprog, bcprog, kernel_name, err, device);
+            bool success = run_test(context, queue, clprog, bcprog, kernel_name, err, device, ulps);
             if (success)
             {
                 log_info("kernel '%s' passed.\n", kernel_name.c_str());

test_conformance/spir/run_services.cpp

@@ -746,7 +746,7 @@ void run_kernel( cl_kernel kernel, cl_command_queue queue, WorkSizeInfo &ws, Tes
 /**
  Compare two test results
  */
-bool compare_results( const TestResult& lhs, const TestResult& rhs )
+bool compare_results( const TestResult& lhs, const TestResult& rhs, float ulps )
 {
     if( lhs.kernelArgs().getArgCount() != rhs.kernelArgs().getArgCount() )
     {
@@ -756,7 +756,7 @@ bool compare_results( const TestResult& lhs, const TestResult& rhs )
 
     for( size_t i = 0 ; i < lhs.kernelArgs().getArgCount(); ++i )
     {
-        if( ! lhs.kernelArgs().getArg(i)->compare( *rhs.kernelArgs().getArg(i)) )
+        if( ! lhs.kernelArgs().getArg(i)->compare( *rhs.kernelArgs().getArg(i), ulps ) )
         {
             log_error("the kernel parameter (%d) is different between SPIR and CL version of the kernel\n", i);
             return false;

test_conformance/spir/run_services.h

@@ -217,6 +217,6 @@ void generate_kernel_data(cl_context context, cl_kernel kernel,
                           WorkSizeInfo &ws, TestResult& res);
 
 void run_kernel(cl_kernel kernel, cl_command_queue queue, WorkSizeInfo &ws, TestResult& result);
-bool compare_results(const TestResult& lhs, const TestResult& rhs);
+bool compare_results(const TestResult& lhs, const TestResult& rhs, float ulps);
 
 #endif