Save Kernel Sources to Disk - Infrastructure Change (#412)

* Introduces the ability to capture the kernel sources and the build
options used by the conformance tests to a .cl and .options files.
This is achieved via a new compilation cache mode called dump-cl-files.
Files will be generated to the folder pointed by the compilation cache
path.

When a conformance test is executed with dump-cl-files cache mode and online
compilation mode, the files will be saved to the disk and the regular execution
flow of the conformance continues uninterrupted.

When a conformance test is executed with dump-cl-files cache mode and binary or spir-v
compilation mode, the files will be saved to the disk as well, however the compilation
will fail - leading to a failure in all test cases.

* Introduces the ability to capture the kernel sources and the build
options used by the conformance tests to a .cl and .options files.
This is achieved via a new compilation cache mode called dump-cl-files.
Files will be generated to the folder pointed by the compilation cache
path.

When a conformance test is executed with dump-cl-files cache mode and online
compilation mode, the files will be saved to the disk and the regular execution
flow of the conformance continues uninterrupted.

When a conformance test is executed with dump-cl-files cache mode and binary or spir-v
compilation mode, the files will be saved to the disk as well, however the compilation
will fail - leading to a failure in all test cases.

* merged with latest CTS code

test_common/harness/kernelHelpers.c

@@ -41,13 +41,6 @@ std::string slash = "/";
 
 static cl_int get_first_device_id(const cl_context context, cl_device_id &device);
 
-std::string get_file_name(const std::string &baseName, int index, const std::string &extension)
-{
-    std::ostringstream fileName;
-    fileName << baseName << "." << index << extension;
-    return fileName.str();
-}
-
 long get_file_size(const std::string &fileName)
 {
     std::ifstream ifs(fileName.c_str(), std::ios::binary);
@@ -59,24 +52,6 @@ long get_file_size(const std::string &fileName)
     return static_cast<long>(length);
 }
 
-std::vector<char> get_file_content(const std::string &fileName)
-{
-    std::ifstream ifs(fileName.c_str(), std::ios::binary);
-    if (!ifs.good())
-        return std::vector<char>(0);
-    // get length of file:
-    ifs.seekg(0, std::ios::end);
-    std::ios::pos_type length = ifs.tellg();
-    ifs.seekg(0, std::ios::beg);
-
-    // allocate memory:
-    std::vector<char> content(static_cast<size_t>(length));
-
-    // read data as a block:
-    ifs.read(&content[0], length);
-    return content;
-}
-
 static std::string get_kernel_content(unsigned int numKernelLines, const char *const *kernelProgram)
 {
     std::string kernel;
@@ -91,9 +66,6 @@ static std::string get_kernel_content(unsigned int numKernelLines, const char *c
 
 std::string get_kernel_name(const std::string &source)
 {
-    // Count CRC
-    cl_uint crc = crc32(source.data(), source.size());
-
     // Create list of kernel names
     std::string kernelsList;
     size_t kPos = source.find("kernel");
@@ -174,48 +146,9 @@ std::string get_kernel_name(const std::string &source)
         }
         oss << kernelsList;
     }
-    oss << std::hex << std::setfill('0') << std::setw(8) << crc;
     return oss.str();
 }
 
-std::string add_build_options(const std::string &baseName, const char *options)
-{
-    if (options == 0 || options[0] == 0)
-        return get_file_name(baseName, 0, "");
-
-    bool equal = false;
-    int i = 0;
-
-    do
-    {
-        i++;
-        std::string fileName = gCompilationCachePath + slash + get_file_name(baseName, i, ".options");
-        long fileSize = get_file_size(fileName);
-        if (fileSize == 0)
-            break;
-        //if(fileSize == strlen(options))
-        {
-            std::vector<char> options2 = get_file_content(fileName);
-            options2.push_back(0); //terminate string
-            equal = strcmp(options, &options2[0]) == 0;
-        }
-    } while (!equal);
-    if (equal)
-        return get_file_name(baseName, i, "");
-
-    std::string fileName = gCompilationCachePath + slash + get_file_name(baseName, i, ".options");
-    std::ofstream ofs(fileName.c_str(), std::ios::binary);
-    if (!ofs.good())
-    {
-        log_info("OfflineCompiler: can't create options: %s\n", fileName.c_str());
-        return "";
-    }
-    // write data as a block:
-    ofs.write(options, strlen(options));
-    log_info("OfflineCompiler: options added: %s\n", fileName.c_str());
-    return get_file_name(baseName, i, "");
-}
-
 static std::string get_offline_compilation_file_type_str(const CompilationMode compilationMode)
 {
     switch (compilationMode)
@@ -233,6 +166,131 @@ static std::string get_offline_compilation_file_type_str(const CompilationMode c
     }
 }
 
+static std::string get_unique_filename_prefix(unsigned int numKernelLines,
+                                              const char *const *kernelProgram,
+                                              const char *buildOptions)
+{
+    std::string kernel = get_kernel_content(numKernelLines, kernelProgram);
+    std::string kernelName = get_kernel_name(kernel);
+    cl_uint kernelCrc = crc32(kernel.data(), kernel.size());
+    std::ostringstream oss;
+    oss << kernelName <<  std::hex << std::setfill('0') << std::setw(8) << kernelCrc;
+    if(buildOptions) {
+        cl_uint bOptionsCrc = crc32(buildOptions, strlen(buildOptions));
+        oss << '.' << std::hex << std::setfill('0') << std::setw(8) << bOptionsCrc;
+    }
+    return oss.str();
+}
+
+
+static std::string 
+get_cl_build_options_filename_with_path(const std::string& filePath,
+                                        const std::string& fileNamePrefix) {
+    return filePath + slash + fileNamePrefix + ".options";
+}
+
+static std::string 
+get_cl_source_filename_with_path(const std::string& filePath,
+                                 const std::string& fileNamePrefix) {
+    return filePath + slash + fileNamePrefix + ".cl";
+}
+
+static std::string 
+get_binary_filename_with_path(CompilationMode mode,
+                              cl_uint deviceAddrSpaceSize,
+                              const std::string& filePath,
+                              const std::string& fileNamePrefix) {
+    std::string binaryFilename = filePath + slash + fileNamePrefix;
+    if(kSpir_v == mode) {
+        std::ostringstream extension;
+        extension << ".spv" << deviceAddrSpaceSize;
+        binaryFilename += extension.str();
+    }
+    return binaryFilename;
+}
+
+static bool file_exist_on_disk(const std::string& filePath,
+                               const std::string& fileName) {
+    std::string fileNameWithPath = filePath + slash + fileName;
+    bool exist = false;
+    std::ifstream ifs;
+
+    ifs.open(fileNameWithPath.c_str(), std::ios::binary);
+    if(ifs.good())
+        exist = true;
+    ifs.close();
+    return exist;
+}
+
+static bool should_save_kernel_source_to_disk(CompilationMode mode,
+                                              CompilationCacheMode cacheMode,
+                                              const std::string& binaryPath, 
+                                              const std::string& binaryName)
+{
+    bool saveToDisk = false;
+    if(cacheMode == kCacheModeDumpCl ||
+       (cacheMode == kCacheModeOverwrite && mode != kOnline)) {
+        saveToDisk = true;
+    }
+    if(cacheMode == kCacheModeCompileIfAbsent && mode != kOnline) {
+        saveToDisk = !file_exist_on_disk(binaryPath, binaryName);
+    }
+    return saveToDisk;
+}
+
+static int save_kernel_build_options_to_disk(const std::string& path,
+                                             const std::string& prefix,
+                                             const char *buildOptions) {
+    std::string filename = get_cl_build_options_filename_with_path(path, prefix);
+    std::ofstream ofs(filename.c_str(), std::ios::binary);
+    if (!ofs.good())
+    {
+        log_info("Can't save kernel build options: %s\n", filename.c_str());
+        return -1;
+    }
+    ofs.write(buildOptions, strlen(buildOptions));
+    ofs.close();
+    log_info("Saved kernel build options to file: %s\n", filename.c_str());
+    return CL_SUCCESS;
+}
+
+static int save_kernel_source_to_disk(const std::string& path,
+                                      const std::string& prefix,
+                                      const std::string& source) {
+    std::string filename = get_cl_source_filename_with_path(path, prefix);
+    std::ofstream ofs(filename.c_str(), std::ios::binary);
+    if (!ofs.good())
+    {
+        log_info("Can't save kernel source: %s\n", filename.c_str());
+        return -1;
+    }
+    ofs.write(source.c_str(), source.size());
+    ofs.close();
+    log_info("Saved kernel source to file: %s\n", filename.c_str());
+    return CL_SUCCESS;
+}
+
+static int save_kernel_source_and_options_to_disk(unsigned int numKernelLines,
+                                                  const char *const *kernelProgram,
+                                                  const char *buildOptions)
+{
+    int error;
+
+    std::string kernel = get_kernel_content(numKernelLines, kernelProgram);
+    std::string kernelNamePrefix = get_unique_filename_prefix(numKernelLines,
+                                                             kernelProgram,
+                                                             buildOptions);
+
+    // save kernel source to disk
+    error = save_kernel_source_to_disk(gCompilationCachePath, kernelNamePrefix, kernel);
+    
+    // save kernel build options to disk if exists
+    if (buildOptions != NULL)
+        error |= save_kernel_build_options_to_disk(gCompilationCachePath, kernelNamePrefix, buildOptions);
+
+    return error;
+}
+
 static std::string get_compilation_mode_str(const CompilationMode compilationMode)
 {
     switch (compilationMode)
@@ -506,79 +564,44 @@ static cl_int get_device_address_bits(const cl_device_id device, cl_uint &device
 }
 
 static int get_offline_compiler_output(std::ifstream &ifs,
-                                       cl_context context,
+                                       const cl_device_id device,
-                                       cl_device_id device,
+                                       cl_uint deviceAddrSpaceSize,
-                                       const std::string &kernel,
                                        const bool openclCXX,
                                        const CompilationMode compilationMode,
                                        const std::string &bOptions,
-                                       const std::string &kernelName)
+                                       const std::string &kernelPath,
+                                       const std::string &kernelNamePrefix)
 {
-    std::string baseFilename = gCompilationCachePath + slash + kernelName;
+    std::string sourceFilename = get_cl_source_filename_with_path(kernelPath, kernelNamePrefix);
+    std::string outputFilename = get_binary_filename_with_path(compilationMode,
+                                                               deviceAddrSpaceSize,
+                                                               kernelPath,
+                                                               kernelNamePrefix);
 
-    // Get device CL_DEVICE_ADDRESS_BITS
+    ifs.open(outputFilename.c_str(), std::ios::binary);
-    cl_uint device_address_space_size = 0;
+    if(!ifs.good()) {
-    int error = get_device_address_bits(device, device_address_space_size);
+       std::string file_type = get_offline_compilation_file_type_str(compilationMode);
-    if (error != CL_SUCCESS)
+        if (gCompilationCacheMode == kCacheModeForceRead) {
-        return error;
-
-    std::string outputFilename = baseFilename;
-    if (compilationMode == kSpir_v)
-    {
-        std::ostringstream extension;
-        extension << ".spv" << device_address_space_size;
-        outputFilename += extension.str();
-    }
-
-    // try to read cached output file when test is run with gCompilationCacheMode != kCacheModeOverwrite
-    if (gCompilationCacheMode != kCacheModeOverwrite)
-        ifs.open(outputFilename.c_str(), std::ios::binary);
-
-    if (gCompilationCacheMode == kCacheModeOverwrite || !ifs.good())
-    {
-        std::string file_type = get_offline_compilation_file_type_str(compilationMode);
-
-        if (gCompilationCacheMode == kCacheModeForceRead)
-        {
             log_info("OfflineCompiler: can't open cached %s file: %s\n",
                      file_type.c_str(), outputFilename.c_str());
             return -1;
         }
+        else {
+            int error = invoke_offline_compiler(device, deviceAddrSpaceSize, compilationMode,
+                                                bOptions, sourceFilename, outputFilename, openclCXX);
+            if (error != CL_SUCCESS)
+                return error;
 
-        ifs.close();
+            // read output file
-
+            ifs.open(outputFilename.c_str(), std::ios::binary);
-        if (gCompilationCacheMode != kCacheModeOverwrite)
+            if (!ifs.good())
-            log_info("OfflineCompiler: can't find cached %s file: %s\n",
+            {
-                     file_type.c_str(), outputFilename.c_str());
+                log_info("OfflineCompiler: can't read generated %s file: %s\n",
-
+                         file_type.c_str(), outputFilename.c_str());
-        std::string sourceFilename = baseFilename + ".cl";
+                return -1;
-
+            }
-        std::ofstream ofs(sourceFilename.c_str(), std::ios::binary);
+       }
-        if (!ofs.good())
-        {
-            log_info("OfflineCompiler: can't create source file: %s\n", sourceFilename.c_str());
-            return -1;
-        }
-
-        // write source to input file
-        ofs.write(kernel.c_str(), kernel.size());
-        ofs.close();
-
-        error = invoke_offline_compiler(device, device_address_space_size, compilationMode,
-                                        bOptions, sourceFilename, outputFilename, openclCXX);
-        if (error != CL_SUCCESS)
-            return error;
-
-        // read output file
-        ifs.open(outputFilename.c_str(), std::ios::binary);
-        if (!ifs.good())
-        {
-            log_info("OfflineCompiler: can't read generated %s file: %s\n",
-                     file_type.c_str(), outputFilename.c_str());
-            return -1;
-        }
     }
-
     return CL_SUCCESS;
 }
 
@@ -591,15 +614,25 @@ static int create_single_kernel_helper_create_program_offline(cl_context context
                                                               const bool openclCXX,
                                                               CompilationMode compilationMode)
 {
+    if(kCacheModeDumpCl == gCompilationCacheMode) {
+        return -1;
+    }
+
+    // Get device CL_DEVICE_ADDRESS_BITS
     int error;
-    std::string kernel = get_kernel_content(numKernelLines, kernelProgram);
+    cl_uint device_address_space_size = 0;
-    std::string kernelName = get_kernel_name(kernel);
+    error = get_device_address_bits(device, device_address_space_size);
+    if (error != CL_SUCCESS)
+        return error;
 
     // set build options
     std::string bOptions;
     bOptions += buildOptions ? std::string(buildOptions) : "";
 
-    kernelName = add_build_options(kernelName, buildOptions);
+    std::string kernelName = get_unique_filename_prefix(numKernelLines,
+                                                        kernelProgram,
+                                                        buildOptions);
+
 
     if (device == NULL)
     {
@@ -608,9 +641,9 @@ static int create_single_kernel_helper_create_program_offline(cl_context context
     }
 
     std::ifstream ifs;
-    error = get_offline_compiler_output(ifs, context, device, kernel, openclCXX, compilationMode, bOptions, kernelName);
+    error = get_offline_compiler_output(ifs, device, device_address_space_size, openclCXX, compilationMode, bOptions, gCompilationCachePath, kernelName);
     if (error != CL_SUCCESS)
-        return error;
+      return error;
 
     // -----------------------------------------------------------------------------------
     // ------------- ONLY FOR OPENCL 22 CONFORMANCE TEST 22 DEVELOPMENT ------------------
@@ -678,6 +711,22 @@ static int create_single_kernel_helper_create_program(cl_context context,
                                                       const bool openclCXX,
                                                       CompilationMode compilationMode)
 {
+    std::string filePrefix = get_unique_filename_prefix(numKernelLines,
+                                                        kernelProgram,
+                                                        buildOptions);
+    bool shouldSaveToDisk = should_save_kernel_source_to_disk(compilationMode, 
+                                                              gCompilationCacheMode,
+                                                              gCompilationCachePath,
+                                                              filePrefix);
+
+    if(shouldSaveToDisk)
+    {
+        if(CL_SUCCESS != save_kernel_source_and_options_to_disk(numKernelLines, kernelProgram, buildOptions))
+        {
+            log_error("Unable to dump kernel source to disk");
+            return -1;
+        }
+    }
     if (compilationMode == kOnline)
     {
         int error = CL_SUCCESS;

test_common/harness/parseParameters.cpp

@@ -47,8 +47,9 @@ void helpInfo ()
              "        --compilation-cache-mode <cache-mode>  Specify a compilation caching mode:\n"
              "                                 compile-if-absent  Read from cache if already populated, or\n"
              "                                                    else perform offline compilation (default)\n"
-             "                                 force-read      Force reading from the cache\n"
+             "                                 force-read        Force reading from the cache\n"
-             "                                 overwrite       Disable reading from the cache\n"
+             "                                 overwrite         Disable reading from the cache\n"
+             "                                 dump-cl-files     Dumps the .cl and build .options files used by the test suite\n"
              "        --compilation-cache-path <path>   Path for offline compiler output and CL source\n"
              "        --compilation-program <prog>      Program to use for offline compilation,\n"
              "                                          defaults to " DEFAULT_COMPILATION_PROGRAM "\n"
@@ -135,6 +136,10 @@ int parseCustomParam (int argc, const char *argv[], const char *ignore)
                 {
                     gCompilationCacheMode = kCacheModeOverwrite;
                 }
+                else if (!strcmp(mode, "dump-cl-files"))
+                {
+                    gCompilationCacheMode = kCacheModeDumpCl;
+                }
                 else
                 {
                     log_error("Compilation cache mode not recognized: %s\n", mode);
@@ -185,7 +190,8 @@ int parseCustomParam (int argc, const char *argv[], const char *ignore)
         i -= delArg;
     }
 
-    if (gCompilationCacheMode != kCacheModeCompileIfAbsent && gCompilationMode == kOnline)
+    if ((gCompilationCacheMode == kCacheModeForceRead || gCompilationCacheMode == kCacheModeOverwrite)
+         && gCompilationMode == kOnline)
     {
         log_error("Compilation cache mode can only be specified when using an offline compilation mode.\n");
         return -1;

test_common/harness/parseParameters.h

@@ -30,7 +30,8 @@ enum CompilationCacheMode
 {
     kCacheModeCompileIfAbsent = 0,
     kCacheModeForceRead,
-    kCacheModeOverwrite
+    kCacheModeOverwrite,
+    kCacheModeDumpCl
 };
 
 extern CompilationMode gCompilationMode;