208 lines · plain
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file contains the definition of the runTests function, which executes a11/// a suite of tests and print a formatted report for each.12///13//===----------------------------------------------------------------------===//14 15#ifndef MATHTEST_TESTRUNNER_HPP16#define MATHTEST_TESTRUNNER_HPP17 18#include "mathtest/DeviceContext.hpp"19#include "mathtest/GpuMathTest.hpp"20#include "mathtest/Numerics.hpp"21#include "mathtest/TestConfig.hpp"22 23#include "llvm/ADT/STLExtras.h"24#include "llvm/ADT/SmallVector.h"25#include "llvm/ADT/StringRef.h"26#include "llvm/ADT/Twine.h"27#include "llvm/Support/Error.h"28#include "llvm/Support/FormatVariadic.h"29#include "llvm/Support/raw_ostream.h"30 31#include <chrono>32#include <cstddef>33#include <memory>34#include <tuple>35 36namespace mathtest {37namespace detail {38 39template <auto Func>40void printPreamble(const TestConfig &Config, size_t Index,41 size_t Total) noexcept {42 using FunctionConfig = FunctionConfig<Func>;43 44 llvm::errs() << "[" << (Index + 1) << "/" << Total << "] "45 << "Running conformance test '" << FunctionConfig::Name46 << "' with '" << Config.Provider << "' on '" << Config.Platform47 << "'\n";48 llvm::errs().flush();49}50 51template <typename T>52void printValue(llvm::raw_ostream &OS, const T &Value) noexcept {53 if constexpr (IsFloatingPoint_v<T>) {54 if constexpr (sizeof(T) < sizeof(float))55 OS << float(Value);56 else57 OS << Value;58 59 const FPBits<T> Bits(Value);60 OS << llvm::formatv(" (0x{0})", llvm::Twine::utohexstr(Bits.uintval()));61 } else {62 OS << Value;63 }64}65 66template <typename... Ts>67void printValues(llvm::raw_ostream &OS,68 const std::tuple<Ts...> &ValuesTuple) noexcept {69 std::apply(70 [&OS](const auto &...Values) {71 bool IsFirst = true;72 auto PrintWithComma = [&](const auto &Value) {73 if (!IsFirst)74 OS << ", ";75 printValue(OS, Value);76 IsFirst = false;77 };78 (PrintWithComma(Values), ...);79 },80 ValuesTuple);81}82 83template <typename TestCaseType>84void printWorstFailingCase(llvm::raw_ostream &OS,85 const TestCaseType &TestCase) noexcept {86 OS << "--- Worst Failing Case ---\n";87 OS << llvm::formatv(" {0,-14} : ", "Input(s)");88 printValues(OS, TestCase.Inputs);89 OS << "\n";90 91 OS << llvm::formatv(" {0,-14} : ", "Actual");92 printValue(OS, TestCase.Actual);93 OS << "\n";94 95 OS << llvm::formatv(" {0,-14} : ", "Expected");96 printValue(OS, TestCase.Expected);97 OS << "\n";98}99 100template <typename TestType, typename ResultType>101void printReport(const TestType &Test, const ResultType &Result,102 const std::chrono::steady_clock::duration &Duration) noexcept {103 using FunctionConfig = typename TestType::FunctionConfig;104 105 const auto Context = Test.getContext();106 const auto ElapsedMilliseconds =107 std::chrono::duration_cast<std::chrono::milliseconds>(Duration).count();108 const bool Passed = Result.hasPassed();109 110 llvm::errs() << llvm::formatv("=== Test Report for '{0}' === \n",111 FunctionConfig::Name);112 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Provider",113 Test.getProvider());114 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Platform",115 Context->getPlatform());116 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Device", Context->getName());117 llvm::errs() << llvm::formatv("{0,-17}: {1} ms\n", "Elapsed time",118 ElapsedMilliseconds);119 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "ULP tolerance",120 FunctionConfig::UlpTolerance);121 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Max ULP distance",122 Result.getMaxUlpDistance());123 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Test cases",124 Result.getTestCaseCount());125 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Failures",126 Result.getFailureCount());127 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Status",128 Passed ? "PASSED" : "FAILED");129 130 if (const auto &Worst = Result.getWorstFailingCase())131 printWorstFailingCase(llvm::errs(), Worst.value());132 133 llvm::errs().flush();134}135 136template <auto Func, typename TestType = GpuMathTest<Func>>137[[nodiscard]] llvm::Expected<bool>138runTest(typename TestType::GeneratorType &Generator, const TestConfig &Config,139 llvm::StringRef DeviceBinaryDir) {140 const auto &Platforms = getPlatforms();141 142 if (!llvm::any_of(Platforms, [&](llvm::StringRef Platform) {143 return Platform.equals_insensitive(Config.Platform);144 }))145 return llvm::createStringError("Platform '" + Config.Platform +146 "' is not available on this system");147 148 auto Context =149 std::make_shared<DeviceContext>(Config.Platform, /*DeviceId=*/0);150 auto ExpectedTest =151 TestType::create(Context, Config.Provider, DeviceBinaryDir);152 153 if (!ExpectedTest)154 return ExpectedTest.takeError();155 156 const auto StartTime = std::chrono::steady_clock::now();157 158 auto Result = ExpectedTest->run(Generator);159 160 const auto EndTime = std::chrono::steady_clock::now();161 const auto Duration = EndTime - StartTime;162 163 printReport(*ExpectedTest, Result, Duration);164 165 return Result.hasPassed();166}167} // namespace detail168 169template <auto Func, typename TestType = GpuMathTest<Func>>170[[nodiscard]] bool runTests(typename TestType::GeneratorType &Generator,171 const llvm::SmallVector<TestConfig, 4> &Configs,172 llvm::StringRef DeviceBinaryDir,173 bool IsVerbose = false) {174 const size_t NumConfigs = Configs.size();175 176 if (NumConfigs == 0)177 llvm::errs() << "There is no test configuration to run a test\n";178 179 bool Passed = true;180 181 for (const auto &[Index, Config] : llvm::enumerate(Configs)) {182 detail::printPreamble<Func>(Config, Index, NumConfigs);183 184 Generator.reset();185 186 auto ExpectedPassed =187 detail::runTest<Func, TestType>(Generator, Config, DeviceBinaryDir);188 189 if (!ExpectedPassed) {190 const auto Details = llvm::toString(ExpectedPassed.takeError());191 llvm::errs()192 << "WARNING: Conformance test not supported on this system\n";193 194 if (IsVerbose)195 llvm::errs() << "Details: " << Details << "\n";196 } else {197 Passed &= *ExpectedPassed;198 }199 200 llvm::errs() << "\n";201 }202 203 return Passed;204}205} // namespace mathtest206 207#endif // MATHTEST_TESTRUNNER_HPP208