243 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Simple drivers to test the mustache spec found at:10// https://github.com/mustache/spec11//12// It is used to verify that the current implementation conforms to the spec.13// Simply download the spec and pass the test JSON files to the driver. Each14// spec file should have a list of tests for compliance with the spec. These15// are loaded as test cases, and rendered with our Mustache implementation,16// which is then compared against the expected output from the spec.17//18// The current implementation only supports non-optional parts of the spec, so19// we do not expect any of the dynamic-names, inheritance, or lambda tests to20// pass. Additionally, Triple Mustache is not supported. Unsupported tests are21// marked as XFail and are removed from the XFail list as they are fixed.22//23// Usage:24// llvm-test-mustache-spec path/to/test/file.json path/to/test/file2.json ...25//===----------------------------------------------------------------------===//26 27#include "llvm/ADT/StringSet.h"28#include "llvm/Support/CommandLine.h"29#include "llvm/Support/Debug.h"30#include "llvm/Support/Error.h"31#include "llvm/Support/MemoryBuffer.h"32#include "llvm/Support/Mustache.h"33#include "llvm/Support/Path.h"34#include <string>35 36using namespace llvm;37using namespace llvm::json;38using namespace llvm::mustache;39 40#define DEBUG_TYPE "llvm-test-mustache-spec"41 42static cl::OptionCategory Cat("llvm-test-mustache-spec Options");43 44static cl::list<std::string>45 InputFiles(cl::Positional, cl::desc("<input files>"), cl::OneOrMore);46 47static cl::opt<bool> ReportErrors("report-errors",48 cl::desc("Report errors in spec tests"),49 cl::cat(Cat));50 51static ExitOnError ExitOnErr;52 53static int NumXFail = 0;54static int NumSuccess = 0;55 56static const StringMap<StringSet<>> XFailTestNames = {{57 {"~dynamic-names.json",58 {59 "Basic Behavior - Partial",60 "Basic Behavior - Name Resolution",61 "Context",62 "Dotted Names",63 "Dotted Names - Failed Lookup",64 "Dotted names - Context Stacking",65 "Dotted names - Context Stacking Under Repetition",66 "Dotted names - Context Stacking Failed Lookup",67 "Recursion",68 "Surrounding Whitespace",69 "Inline Indentation",70 "Standalone Line Endings",71 "Standalone Without Previous Line",72 "Standalone Without Newline",73 "Standalone Indentation",74 "Padding Whitespace",75 }},76 {"~inheritance.json",77 {78 "Default",79 "Variable",80 "Triple Mustache",81 "Sections",82 "Negative Sections",83 "Mustache Injection",84 "Inherit",85 "Overridden content",86 "Data does not override block default",87 "Two overridden parents",88 "Override parent with newlines",89 "Inherit indentation",90 "Only one override",91 "Parent template",92 "Recursion",93 "Multi-level inheritance, no sub child",94 "Text inside parent",95 "Text inside parent",96 "Block scope",97 "Standalone parent",98 "Standalone block",99 "Block reindentation",100 "Intrinsic indentation",101 "Nested block reindentation",102 }},103 {"~lambdas.json",104 {105 "Interpolation",106 "Interpolation - Expansion",107 "Interpolation - Alternate Delimiters",108 "Interpolation - Multiple Calls",109 "Escaping",110 "Section",111 "Section - Expansion",112 "Section - Alternate Delimiters",113 "Section - Multiple Calls",114 }},115}};116 117struct TestData {118 TestData() = default;119 explicit TestData(const json::Object &TestCase)120 : TemplateStr(*TestCase.getString("template")),121 ExpectedStr(*TestCase.getString("expected")),122 Name(*TestCase.getString("name")), Data(TestCase.get("data")),123 Partials(TestCase.get("partials")) {}124 125 static Expected<TestData> createTestData(json::Object *TestCase,126 StringRef InputFile) {127 // If any of the needed elements are missing, we cannot continue.128 // NOTE: partials are optional in the test schema.129 if (!TestCase || !TestCase->getString("template") ||130 !TestCase->getString("expected") || !TestCase->getString("name") ||131 !TestCase->get("data"))132 return createStringError(133 llvm::inconvertibleErrorCode(),134 "invalid JSON schema in test file: " + InputFile + "\n");135 136 return TestData(*TestCase);137 }138 139 StringRef TemplateStr;140 StringRef ExpectedStr;141 StringRef Name;142 const Value *Data;143 const Value *Partials;144};145 146static void reportTestFailure(const TestData &TD, StringRef ActualStr,147 bool IsXFail) {148 LLVM_DEBUG(dbgs() << "Template: " << TD.TemplateStr << "\n");149 if (TD.Partials) {150 LLVM_DEBUG(dbgs() << "Partial: ");151 LLVM_DEBUG(TD.Partials->print(dbgs()));152 LLVM_DEBUG(dbgs() << "\n");153 }154 LLVM_DEBUG(dbgs() << "JSON Data: ");155 LLVM_DEBUG(TD.Data->print(dbgs()));156 LLVM_DEBUG(dbgs() << "\n");157 outs() << formatv("Test {}: {}\n", (IsXFail ? "XFailed" : "Failed"), TD.Name);158 if (ReportErrors) {159 outs() << " Expected: \'" << TD.ExpectedStr << "\'\n"160 << " Actual: \'" << ActualStr << "\'\n"161 << " ====================\n";162 }163}164 165static void registerPartials(const Value *Partials, Template &T) {166 if (!Partials)167 return;168 for (const auto &[Partial, Str] : *Partials->getAsObject())169 T.registerPartial(Partial.str(), Str.getAsString()->str());170}171 172static json::Value readJsonFromFile(StringRef &InputFile) {173 std::unique_ptr<MemoryBuffer> Buffer =174 ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile)));175 return ExitOnErr(parse(Buffer->getBuffer()));176}177 178static bool isTestXFail(StringRef FileName, StringRef TestName) {179 auto P = llvm::sys::path::filename(FileName);180 auto It = XFailTestNames.find(P);181 return It != XFailTestNames.end() && It->second.contains(TestName);182}183 184static bool evaluateTest(StringRef &InputFile, TestData &TestData,185 std::string &ActualStr) {186 bool IsXFail = isTestXFail(InputFile, TestData.Name);187 bool Matches = TestData.ExpectedStr == ActualStr;188 if ((Matches && IsXFail) || (!Matches && !IsXFail)) {189 reportTestFailure(TestData, ActualStr, IsXFail);190 return false;191 }192 IsXFail ? NumXFail++ : NumSuccess++;193 return true;194}195 196static void runTest(StringRef InputFile) {197 NumXFail = 0;198 NumSuccess = 0;199 outs() << "Running Tests: " << InputFile << "\n";200 json::Value Json = readJsonFromFile(InputFile);201 202 json::Object *Obj = Json.getAsObject();203 Array *TestArray = Obj->getArray("tests");204 // Even though we parsed the JSON, it can have a bad format, so check it.205 if (!TestArray)206 ExitOnErr(createStringError(207 llvm::inconvertibleErrorCode(),208 "invalid JSON schema in test file: " + InputFile + "\n"));209 210 const size_t Total = TestArray->size();211 212 for (Value V : *TestArray) {213 auto TestData =214 ExitOnErr(TestData::createTestData(V.getAsObject(), InputFile));215 BumpPtrAllocator Allocator;216 StringSaver Saver(Allocator);217 MustacheContext Ctx(Allocator, Saver);218 Template T(TestData.TemplateStr, Ctx);219 registerPartials(TestData.Partials, T);220 221 std::string ActualStr;222 raw_string_ostream OS(ActualStr);223 T.render(*TestData.Data, OS);224 evaluateTest(InputFile, TestData, ActualStr);225 }226 227 const int NumFailed = Total - NumSuccess - NumXFail;228 outs() << formatv("===Results===\n"229 " Suceeded: {}\n"230 " Expectedly Failed: {}\n"231 " Failed: {}\n"232 " Total: {}\n",233 NumSuccess, NumXFail, NumFailed, Total);234}235 236int main(int argc, char **argv) {237 ExitOnErr.setBanner(std::string(argv[0]) + " error: ");238 cl::ParseCommandLineOptions(argc, argv);239 for (const auto &FileName : InputFiles)240 runTest(FileName);241 return 0;242}243