From 8d44302935fa59593bbb1d8606cb83286d5e50a8 Mon Sep 17 00:00:00 2001
From: Kenneth Benzie <k.benzie@codeplay.com>
Date: Tue, 20 Oct 2020 07:08:40 +0100
Subject: [PATCH] Return a non-zero exit-code when a sub-test fails (#951)

When running sub-tests explicitly on the command line and that sub-test
fails, the test harness returns `0` from `main()`. This is problematic
in automated testing environments as failed tests can be hidden.

This patch iterates over all the test results in the `resultTestList`
after all sub-tests have completed, if any of the tests failed a
non-zero value is returned from `parseAndCallCommandLineTests()` which
is then propagated up the call stack to `main()`.
---
 test_common/harness/testHarness.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/test_common/harness/testHarness.cpp b/test_common/harness/testHarness.cpp
index 83a575b4..51bbba05 100644
--- a/test_common/harness/testHarness.cpp
+++ b/test_common/harness/testHarness.cpp
@@ -15,6 +15,7 @@
 //
 #include "testHarness.h"
 #include "compat.h"
+#include <algorithm>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -639,6 +640,19 @@ int parseAndCallCommandLineTests( int argc, const char *argv[], cl_device_id dev
         }
     }
 
+    if (std::any_of(resultTestList, resultTestList + testNum,
+                    [](test_status result) {
+                        switch (result)
+                        {
+                            case TEST_PASS:
+                            case TEST_SKIP: return false;
+                            case TEST_FAIL: return true;
+                        };
+                    }))
+    {
+        ret = EXIT_FAILURE;
+    }
+
     free( selectedTestList );
     free( resultTestList );