320 lines · python
1"""YAML Summariser2 3The flang plugin ``flang-omp-report`` takes one Fortran4file in and returns a YAML report file of the input file.5This becomes an issue when you want to analyse an entire project6into one final report.7The purpose of this Python script is to generate a final YAML8summary from all of the files generated by ``flang-omp-report``.9 10Currently, it requires ``ruamel.yaml``,11which can be installed with:12 13 ``pip3 install ruamel.yaml``14 15By default it scans the directory it is ran in16for any YAML files and outputs a summary to17stdout. It can be ran as:18 19 ``python3 yaml_summarizer.py``20 21Parameters:22 23 -d --directory Specify which directory to scan. Multiple directories can be searched by24 providing a semicolon separated list of directories.25 26 -l --log Combine all yaml files into one log (instead of generating a summary)27 28 -o --output Specify a directory in which to save the summary file29 30 -r --recursive Recursively search directory for all yaml files31 32Examples:33 34 ``python3 yaml_summarizer.py -d ~/llvm-project/build/ -r``35 36 ``python3 yaml_summarizer.py -d "~/llvm-project/build/;~/llvm-project/flang/test/Examples"``37 38 ``python3 yaml_summarizer.py -l -o ~/examples/report.yaml``39 40Pseudo-examples:41 42 Summary:43 44 $ python3 yaml_summarizer.py file_1.yaml file_2.yaml45 <Unique OMP constructs with there grouped clauses from file_1.yaml and file_2.yaml>46 47 Constructs are in the form:48 - construct: someOMPconstruct49 count: 850 clauses:51 - clause: clauseOne52 count: 453 - clause: ClauseTwo54 count: 255 56 Log:57 58 $ python3 yaml_summarizer.py -l file_1.yaml file_2.yaml59 file_1.yaml60 <OMP clauses and constructs from file_1.yaml>61 file_2.yaml62 <OMP clauses and constructs from file_2.yaml>63 64 Constructs are in the form:65 - construct: someOMPConstruct66 line: 1267 clauses:68 - clause: clauseOne69 details: 'someDetailForClause'70"""71 72import sys73import glob74import argparse75from pathlib import Path76from os.path import isdir77 78from ruamel.yaml import YAML79 80 81def find_yaml_files(search_directory: Path, search_pattern: str):82 """83 Find all '.yaml' files and returns an iglob iterator to them.84 85 Keyword arguments:86 search_pattern -- Search pattern for 'iglob' to use for finding '.yaml' files.87 If this is set to 'None', then it will default to just searching88 for all '.yaml' files in the current directory.89 """90 # @TODO: Currently *all* yaml files are read - regardless of whether they have91 # been generated with 'flang-omp-report' or not. This might result in the script92 # reading files that it should ignore.93 if search_directory:94 return glob.iglob(95 str(search_directory.joinpath(search_pattern)), recursive=True96 )97 98 return glob.iglob(str("/" + search_pattern), recursive=True)99 100 101def process_log(data, result: list):102 """103 Process the data input as a 'log' to the result array. This esssentially just104 stitches together all of the input '.yaml' files into one result.105 106 Keyword arguments:107 data -- Data from yaml.load() for a yaml file. So the type can be 'Any'.108 result -- Array to add the processed data to.109 """110 for datum in data:111 items = result.get(datum["file"], [])112 items.append(113 {114 "construct": datum["construct"],115 "line": datum["line"],116 "clauses": datum["clauses"],117 }118 )119 result[datum["file"]] = items120 121 122def add_clause(datum, construct):123 """124 Add clauses to the construct if they're missing125 Otherwise increment their count by one.126 127 Keyword arguments:128 datum -- Data construct containing clauses to check.129 construct -- Construct to add or increment clause count.130 """131 to_check = [i["clause"] for i in construct["clauses"]]132 to_add = [i["clause"] for i in datum["clauses"]]133 clauses = construct["clauses"]134 for item in to_add:135 if item in to_check:136 for clause in clauses:137 if clause["clause"] == item:138 clause["count"] += 1139 else:140 clauses.append({"clause": item, "count": 1})141 142 143def process_summary(data, result: dict):144 """145 Process the data input as a 'summary' to the 'result' dictionary.146 147 Keyword arguments:148 data -- Data from yaml.load() for a yaml file. So the type can be 'Any'.149 result -- Dictionary to add the processed data to.150 """151 for datum in data:152 construct = next(153 (item for item in result if item["construct"] == datum["construct"]), None154 )155 clauses = []156 # Add the construct and clauses to the summary if157 # they haven't been seen before158 if not construct:159 for i in datum["clauses"]:160 clauses.append({"clause": i["clause"], "count": 1})161 result.append(162 {"construct": datum["construct"], "count": 1, "clauses": clauses}163 )164 else:165 construct["count"] += 1166 167 add_clause(datum, construct)168 169 170def clean_summary(result):171 """Cleans the result after processing the yaml files with summary format."""172 # Remove all "clauses" that are empty to keep things compact173 for construct in result:174 if construct["clauses"] == []:175 construct.pop("clauses")176 177 178def clean_log(result):179 """Cleans the result after processing the yaml files with log format."""180 for constructs in result.values():181 for construct in constructs:182 if construct["clauses"] == []:183 construct.pop("clauses")184 185 186def output_result(yaml: YAML, result, output_file: Path):187 """188 Outputs result to either 'stdout' or to a output file.189 190 Keyword arguments:191 result -- Format result to output.192 output_file -- File to output result to. If this is 'None' then result will be193 outputted to 'stdout'.194 """195 if output_file:196 with open(output_file, "w+", encoding="utf-8") as file:197 if output_file.suffix == ".yaml":198 yaml.dump(result, file)199 else:200 file.write(result)201 else:202 yaml.dump(result, sys.stdout)203 204 205def process_yaml(206 search_directories: list, search_pattern: str, result_format: str, output_file: Path207):208 """209 Reads each yaml file, calls the appropiate format function for210 the file and then ouputs the result to either 'stdout' or to an output file.211 212 Keyword arguments:213 search_directories -- List of directory paths to search for '.yaml' files in.214 search_pattern -- String pattern formatted for use with glob.iglob to find all215 '.yaml' files.216 result_format -- String representing output format. Current supported strings are: 'log'.217 output_file -- Path to output file (If value is None, then default to outputting to 'stdout').218 """219 if result_format == "log":220 result = {}221 action = process_log222 clean_report = clean_log223 else:224 result = []225 action = process_summary226 clean_report = clean_summary227 228 yaml = YAML()229 230 for search_directory in search_directories:231 for file in find_yaml_files(search_directory, search_pattern):232 with open(file, "r", encoding="utf-8") as yaml_file:233 data = yaml.load(yaml_file)234 action(data, result)235 236 if clean_report is not None:237 clean_report(result)238 239 output_result(yaml, result, output_file)240 241 242def create_arg_parser():243 """Create and return a argparse.ArgumentParser modified for script."""244 parser = argparse.ArgumentParser()245 parser.add_argument(246 "-d", "--directory", help="Specify a directory to scan", dest="dir", type=str247 )248 parser.add_argument(249 "-o",250 "--output",251 help="Writes to a file instead of\252 stdout",253 dest="output",254 type=str,255 )256 parser.add_argument(257 "-r",258 "--recursive",259 help="Recursive search for .yaml files",260 dest="recursive",261 type=bool,262 nargs="?",263 const=True,264 default=False,265 )266 267 exclusive_parser = parser.add_mutually_exclusive_group()268 exclusive_parser.add_argument(269 "-l",270 "--log",271 help="Modifies report format: " "Combines the log '.yaml' files into one file.",272 action="store_true",273 dest="log",274 )275 return parser276 277 278def parse_arguments():279 """Parses arguments given to script and returns a tuple of processed arguments."""280 parser = create_arg_parser()281 args = parser.parse_args()282 283 if args.dir:284 search_directory = [Path(path) for path in args.dir.split(";")]285 else:286 search_directory = [Path.cwd()]287 288 if args.recursive:289 search_pattern = "**/*.yaml"290 else:291 search_pattern = "*.yaml"292 293 if args.log:294 result_format = "log"295 else:296 result_format = "summary"297 298 if args.output:299 if isdir(args.output):300 output_file = Path(args.output).joinpath("summary.yaml")301 elif isdir(Path(args.output).resolve().parent):302 output_file = Path(args.output)303 else:304 output_file = None305 306 return (search_directory, search_pattern, result_format, output_file)307 308 309def main():310 """Main function of script."""311 (search_directory, search_pattern, result_format, output_file) = parse_arguments()312 313 process_yaml(search_directory, search_pattern, result_format, output_file)314 315 return 0316 317 318if __name__ == "__main__":319 sys.exit(main())320