542 lines · python
1#!/usr/bin/env python32"""3Generate C test files that call ACLE builtins found in a JSON manifest.4 5Expected JSON input format (array of objects):6[7 {8 "guard": "sve,(sve2p1|sme)",9 "streaming_guard": "sme",10 "flags": "feature-dependent",11 "builtin": "svint16_t svrevd_s16_z(svbool_t, svint16_t);"12 },13 ...14]15"""16 17from __future__ import annotations18 19import argparse20import json21import re22import sys23from collections import defaultdict24from dataclasses import dataclass25from enum import Enum26from itertools import product27from pathlib import Path28from typing import Any, Dict, Iterable, List, Sequence, Tuple29 30assert sys.version_info >= (3, 7), "Only Python 3.7+ is supported."31 32 33# Are we testing arm_sve.h or arm_sme.h based builtins.34class Mode(Enum):35 SVE = "sve"36 SME = "sme"37 38 39class FunctionType(Enum):40 NORMAL = "normal"41 STREAMING = "streaming"42 STREAMING_COMPATIBLE = "streaming-compatible"43 44 45# Builtins are grouped by their required features.46@dataclass(frozen=True, order=True)47class BuiltinContext:48 guard: str49 streaming_guard: str50 flags: Tuple[str, ...]51 52 def __str__(self) -> str:53 return (54 f"// Properties: "55 f'guard="{self.guard}" '56 f'streaming_guard="{self.streaming_guard}" '57 f'flags="{",".join(self.flags)}"'58 )59 60 @classmethod61 def from_json(cls, obj: dict[str, Any]) -> "BuiltinContext":62 flags = tuple(p.strip() for p in obj["flags"].split(",") if p.strip())63 return cls(obj["guard"], obj["streaming_guard"], flags)64 65 66# --- Parsing builtins -------------------------------------------------------67 68# Captures the full function *declaration* inside the builtin string, e.g.:69# "svint16_t svrevd_s16_z(svbool_t, svint16_t);"70# group(1) => "svint16_t svrevd_s16_z"71# group(2) => "svbool_t, svint16_t"72FUNC_RE = re.compile(r"^\s*([a-zA-Z_][\w\s\*]*[\w\*])\s*\(\s*([^)]*)\s*\)\s*;\s*$")73 74# Pulls the final word out of the left side (the function name).75NAME_RE = re.compile(r"([a-zA-Z_][\w]*)\s*$")76 77 78def parse_builtin_declaration(decl: str) -> Tuple[str, List[str]]:79 """Return (func_name, param_types) from a builtin declaration string.80 81 Example:82 decl = "svint16_t svrevd_s16_z(svbool_t, svint16_t);"83 => ("svrevd_s16_z", ["svbool_t", "svint16_t"])84 """85 m = FUNC_RE.match(decl)86 if not m:87 raise ValueError(f"Unrecognized builtin declaration syntax: {decl!r}")88 89 left = m.group(1) # return type + name90 params = m.group(2).strip()91 92 name_m = NAME_RE.search(left)93 if not name_m:94 raise ValueError(f"Could not find function name in: {decl!r}")95 func_name = name_m.group(1)96 97 param_types: List[str] = []98 if params:99 # Split by commas respecting no pointers/arrays with commas (not expected here)100 param_types = [p.strip() for p in params.split(",") if p.strip()]101 102 return func_name, param_types103 104 105# --- Variable synthesis -----------------------------------------------------106 107# Pick a safe (ideally non-zero) value for literal types108LITERAL_TYPES_MAP: dict[str, str] = {109 "ImmCheck0_0": "0",110 "ImmCheck0_1": "1",111 "ImmCheck0_2": "2",112 "ImmCheck0_3": "2",113 "ImmCheck0_7": "2",114 "ImmCheck0_13": "2",115 "ImmCheck0_15": "2",116 "ImmCheck0_31": "2",117 "ImmCheck0_63": "2",118 "ImmCheck0_255": "2",119 "ImmCheck1_1": "1",120 "ImmCheck1_3": "2",121 "ImmCheck1_7": "2",122 "ImmCheck1_16": "2",123 "ImmCheck1_32": "2",124 "ImmCheck1_64": "2",125 "ImmCheck2_4_Mul2": "2",126 "ImmCheckComplexRot90_270": "90",127 "ImmCheckComplexRotAll90": "90",128 "ImmCheckCvt": "2",129 "ImmCheckExtract": "2",130 "ImmCheckLaneIndexCompRotate": "1",131 "ImmCheckLaneIndexDot": "1",132 "ImmCheckLaneIndex": "1",133 "ImmCheckShiftLeft": "2",134 "ImmCheckShiftRightNarrow": "2",135 "ImmCheckShiftRight": "2",136 "enum svpattern": "SV_MUL3",137 "enum svprfop": "SV_PSTL1KEEP",138 "void": "",139}140 141 142def make_arg_for_type(ty: str) -> Tuple[str, str]:143 """Return (var_decl, var_use) for a parameter type.144 145 Literal types return an empty declaration and a value that will be accepted146 by clang's semantic literal validation.147 """148 # Compress whitespace and remove non-relevant qualifiers.149 ty = re.sub(r"\s+", " ", ty.strip()).replace(" const", "")150 if ty in LITERAL_TYPES_MAP:151 return "", LITERAL_TYPES_MAP[ty]152 153 if ty.startswith("ImmCheck") or ty.startswith("enum "):154 print(f"Failed to parse potential literal type: {ty}", file=sys.stderr)155 156 name = ty.replace(" ", "_").replace("*", "ptr") + "_val"157 return f"{ty} {name};", name158 159 160# NOTE: Parsing is limited to the minimum required for guard strings.161# Specifically the expected input is of the form:162# feat1,feat2,...(feat3 | feat4 | ...),...163def expand_feature_guard(164 guard: str, flags: Sequence[str], base_feature: str = ""165) -> list[set[str]]:166 """167 Expand a guard expression where ',' = AND and '|' = OR, with parentheses168 grouping OR-expressions. Returns a list of feature sets.169 """170 if not guard:171 return []172 173 parts = re.split(r",(?![^(]*\))", guard)174 175 choices_per_part = []176 for part in parts:177 if part.startswith("(") and part.endswith(")"):178 choices_per_part.append(part[1:-1].split("|"))179 else:180 choices_per_part.append([part])181 182 # Add feature that is common to all183 if base_feature:184 choices_per_part.append([base_feature])185 186 if "requires-zt" in flags:187 choices_per_part.append(["sme2"])188 189 # construct list of feature sets190 results = []191 for choice in product(*choices_per_part):192 choice_set = set(choice)193 results.append(choice_set)194 195 # remove superset and duplicates196 unique = []197 for s in results:198 if any(s > other for other in results):199 continue200 if s not in unique:201 unique.append(s)202 203 return unique204 205 206def cc1_args_for_features(features: set[str]) -> str:207 return " ".join("-target-feature +" + s for s in sorted(features))208 209 210def sanitise_guard(s: str) -> str:211 """Rewrite guard strings in a form more suitable for file naming."""212 replacements = {213 ",": "_AND_",214 "|": "_OR_",215 "(": "_LP_",216 ")": "_RP_",217 }218 for k, v in replacements.items():219 s = s.replace(k, v)220 221 # Collapse multiple underscores222 s = re.sub(r"_+", "_", s)223 return s.strip("_")224 225 226def make_filename(prefix: str, ctx: BuiltinContext, ext: str) -> str:227 parts = [sanitise_guard(ctx.guard), sanitise_guard(ctx.streaming_guard)]228 sanitised_guard = "___".join(p for p in parts if p)229 230 if "streaming-only" in ctx.flags:231 prefix += "_streaming_only"232 elif "streaming-compatible" in ctx.flags:233 prefix += "_streaming_compatible"234 elif "feature-dependent" in ctx.flags:235 prefix += "_feature_dependent"236 else:237 prefix += "_non_streaming_only"238 239 return f"{prefix}_{sanitised_guard}{ext}"240 241 242# --- Code Generation --------------------------------------------------------243 244 245def emit_streaming_guard_run_lines(ctx: BuiltinContext) -> str:246 """Emit lit RUN lines that will exercise the relevant Sema diagnistics."""247 run_prefix = "// RUN: %clang_cc1 %s -fsyntax-only -triple aarch64-none-linux-gnu"248 out: List[str] = []249 250 # All RUN lines have SVE and SME enabled251 guard_features = expand_feature_guard(ctx.guard, ctx.flags, "sme")252 streaming_guard_features = expand_feature_guard(253 ctx.streaming_guard, ctx.flags, "sve"254 )255 256 if "streaming-only" in ctx.flags:257 assert not guard_features258 # Generate RUN lines for features only available to streaming functions259 for feats in streaming_guard_features:260 out.append(261 f"{run_prefix} {cc1_args_for_features(feats)} -verify=streaming-guard"262 )263 elif "streaming-compatible" in ctx.flags:264 assert not guard_features265 # NOTE: Streaming compatible builtins don't require SVE.266 # Generate RUN lines for features available to all functions.267 for feats in expand_feature_guard(ctx.streaming_guard, ctx.flags):268 out.append(f"{run_prefix} {cc1_args_for_features(feats)} -verify")269 out.append("// expected-no-diagnostics")270 elif "feature-dependent" in ctx.flags:271 assert guard_features and streaming_guard_features272 combined_features = expand_feature_guard(273 ctx.guard + "," + ctx.streaming_guard, ctx.flags274 )275 276 # Generate RUN lines for features only available to normal functions277 for feats in guard_features:278 if feats not in combined_features:279 out.append(f"{run_prefix} {cc1_args_for_features(feats)} -verify=guard")280 281 # Geneate RUN lines for features only available to streaming functions282 for feats in streaming_guard_features:283 if feats not in combined_features:284 out.append(285 f"{run_prefix} {cc1_args_for_features(feats)} -verify=streaming-guard"286 )287 288 # Generate RUN lines for features available to all functions289 for feats in combined_features:290 out.append(f"{run_prefix} {cc1_args_for_features(feats)} -verify")291 292 out.append("// expected-no-diagnostics")293 else:294 assert not streaming_guard_features295 # Geneate RUN lines for features only available to normal functions296 for feats in guard_features:297 out.append(f"{run_prefix} {cc1_args_for_features(feats)} -verify=guard")298 299 return "\n".join(out)300 301 302def emit_streaming_guard_function(303 ctx: BuiltinContext,304 var_decls: Sequence[str],305 calls: Sequence[str],306 func_name: str,307 func_type: FunctionType = FunctionType.NORMAL,308) -> str:309 """Emit a C function calling all builtins.310 311 `calls` is a sequence of tuples: (name, call_line)312 """313 # Expected Sema diagnostics for invalid usage314 require_diagnostic = require_streaming_diagnostic = False315 if "streaming-only" in ctx.flags:316 if func_type != FunctionType.STREAMING:317 require_streaming_diagnostic = True318 elif "streaming-compatible" in ctx.flags:319 pass # streaming compatible builtins are always available320 elif "feature-dependent" in ctx.flags:321 guard_features = expand_feature_guard(ctx.guard, ctx.flags, "sme")322 streaming_guard_features = expand_feature_guard(323 ctx.streaming_guard, ctx.flags, "sve"324 )325 combined_features = expand_feature_guard(326 ctx.guard + "," + ctx.streaming_guard, ctx.flags327 )328 329 if func_type != FunctionType.NORMAL:330 if any(feats not in combined_features for feats in guard_features):331 require_diagnostic = True332 if func_type != FunctionType.STREAMING:333 if any(334 feats not in combined_features for feats in streaming_guard_features335 ):336 require_streaming_diagnostic = True337 else:338 if func_type != FunctionType.NORMAL:339 require_diagnostic = True340 341 out: List[str] = []342 343 # Emit test function declaration344 attr: list[str] = []345 if func_type == FunctionType.STREAMING:346 attr.append("__arm_streaming")347 elif func_type == FunctionType.STREAMING_COMPATIBLE:348 attr.append("__arm_streaming_compatible")349 350 if "requires-za" in ctx.flags:351 attr.append('__arm_inout("za")')352 if "requires-zt" in ctx.flags:353 attr.append('__arm_inout("zt0")')354 out.append(f"void {func_name}(void) " + " ".join(attr) + "{")355 356 # Emit variable declarations357 for v in var_decls:358 out.append(f" {v}")359 if var_decls:360 out.append("")361 362 # Emit calls363 for call in calls:364 if require_diagnostic and require_streaming_diagnostic:365 out.append(366 " // guard-error@+2 {{builtin can only be called from a non-streaming function}}"367 )368 out.append(369 " // streaming-guard-error@+1 {{builtin can only be called from a streaming function}}"370 )371 elif require_diagnostic:372 out.append(373 " // guard-error@+1 {{builtin can only be called from a non-streaming function}}"374 )375 elif require_streaming_diagnostic:376 out.append(377 " // streaming-guard-error@+1 {{builtin can only be called from a streaming function}}"378 )379 out.append(f" {call}")380 381 out.append("}")382 return "\n".join(out) + "\n"383 384 385def natural_key(s: str):386 """Allow sorting akin to "sort -V"""387 return [int(text) if text.isdigit() else text for text in re.split(r"(\d+)", s)]388 389 390def build_calls_for_group(builtins: Iterable[str]) -> Tuple[List[str], List[str]]:391 """From a list of builtin declaration strings, produce:392 - a sorted list of unique variable declarations393 - a sorted list of builtin calls394 """395 var_decls: List[str] = []396 seen_types: set[str] = set()397 calls: List[str] = []398 399 for decl in builtins:400 fn, param_types = parse_builtin_declaration(decl)401 402 arg_names: List[str] = []403 for i, ptype in enumerate(param_types):404 vdecl, vname = make_arg_for_type(ptype)405 if vdecl and vdecl not in seen_types:406 seen_types.add(vdecl)407 var_decls.append(vdecl)408 arg_names.append(vname)409 410 calls.append(f"{fn}(" + ", ".join(arg_names) + ");")411 412 # Natural sort (e.g. int8_t before int16_t)413 calls.sort(key=natural_key)414 var_decls.sort(key=natural_key)415 416 return var_decls, calls417 418 419def gen_streaming_guard_tests(mode: Mode, json_path: Path, out_dir: Path) -> None:420 """Generate a set of Clang Sema test files to ensure SVE/SME builtins are421 callable based on the function type, or the required diagnostic is emitted.422 """423 try:424 data = json.loads(json_path.read_text())425 except json.JSONDecodeError as e:426 print(f"Failed to parse JSON {json_path}: {e}", file=sys.stderr)427 return428 429 # Group by (guard, streaming_guard)430 by_guard: Dict[BuiltinContext, List[str]] = defaultdict(list)431 for obj in data:432 by_guard[BuiltinContext.from_json(obj)].append(obj["builtin"])433 434 # For each guard pair, emit 3 functions435 for builtin_ctx, builtin_decls in by_guard.items():436 var_decls, calls = build_calls_for_group(builtin_decls)437 438 out_parts: List[str] = []439 out_parts.append(440 "// NOTE: File has been autogenerated by utils/aarch64_builtins_test_generator.py"441 )442 out_parts.append(emit_streaming_guard_run_lines(builtin_ctx))443 out_parts.append("")444 out_parts.append("// REQUIRES: aarch64-registered-target")445 out_parts.append("")446 out_parts.append(f"#include <arm_{mode.value}.h>")447 out_parts.append("")448 out_parts.append(str(builtin_ctx))449 out_parts.append("")450 out_parts.append(451 emit_streaming_guard_function(builtin_ctx, var_decls, calls, "test")452 )453 out_parts.append(454 emit_streaming_guard_function(455 builtin_ctx, var_decls, calls, "test_streaming", FunctionType.STREAMING456 )457 )458 out_parts.append(459 emit_streaming_guard_function(460 builtin_ctx,461 var_decls,462 calls,463 "test_streaming_compatible",464 FunctionType.STREAMING_COMPATIBLE,465 )466 )467 468 output = "\n".join(out_parts).rstrip() + "\n"469 470 if out_dir:471 out_dir.mkdir(parents=True, exist_ok=True)472 filename = make_filename(f"arm_{mode.value}", builtin_ctx, ".c")473 (out_dir / filename).write_text(output)474 else:475 print(output)476 477 return478 479 480# --- Main -------------------------------------------------------------------481 482 483def existing_file(path: str) -> Path:484 p = Path(path)485 if not p.is_file():486 raise argparse.ArgumentTypeError(f"{p} is not a valid file")487 return p488 489 490def main(argv: Sequence[str] | None = None) -> int:491 ap = argparse.ArgumentParser(description="Emit C tests for SVE/SME builtins")492 ap.add_argument(493 "json", type=existing_file, help="Path to json formatted builtin descriptions"494 )495 ap.add_argument(496 "--out-dir", type=Path, default=None, help="Output directory (default: stdout)"497 )498 ap.add_argument(499 "--gen-streaming-guard-tests",500 action="store_true",501 help="Generate C tests to validate SVE/SME builtin usage base on streaming attribute",502 )503 ap.add_argument(504 "--gen-target-guard-tests",505 action="store_true",506 help="Generate C tests to validate SVE/SME builtin usage based on target features",507 )508 ap.add_argument(509 "--gen-builtin-tests",510 action="store_true",511 help="Generate C tests to exercise SVE/SME builtins",512 )513 ap.add_argument(514 "--base-target-feature",515 choices=["sve", "sme"],516 help="Force builtin source (sve: arm_sve.h, sme: arm_sme.h)",517 )518 519 args = ap.parse_args(argv)520 521 # When not forced, try to infer the mode from the input, defaulting to SVE.522 if args.base_target_feature:523 mode = Mode(args.base_target_feature)524 elif args.json and args.json.name == "arm_sme_builtins.json":525 mode = Mode.SME526 else:527 mode = Mode.SVE528 529 # Generate test file530 if args.gen_streaming_guard_tests:531 gen_streaming_guard_tests(mode, args.json, args.out_dir)532 if args.gen_target_guard_tests:533 ap.error("--gen-target-guard-tests not implemented yet!")534 if args.gen_builtin_tests:535 ap.error("--gen-builtin-tests not implemented yet!")536 537 return 0538 539 540if __name__ == "__main__":541 raise SystemExit(main())542