95 lines · python
1# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.2# See https://llvm.org/LICENSE.txt for license information.3# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception4"""Caches .lit_test_times.txt files between premerge invocations.5 6.lit_test_times.txt files are used by lit to order tests to best take advantage7of parallelism. Having them around and up to date can result in a ~15%8improvement in test times. This script downloading cached test time files and9uploading new versions to the GCS buckets used for caching.10"""11 12import sys13import os14import logging15import multiprocessing.pool16import pathlib17import glob18 19from google.cloud import storage20from google.api_core import exceptions21 22GCS_PARALLELISM = 10023 24 25def _maybe_upload_timing_file(bucket, timing_file_path):26 if os.path.exists(timing_file_path):27 timing_file_blob = bucket.blob("lit_timing/" + timing_file_path)28 timing_file_blob.upload_from_filename(timing_file_path)29 30 31def upload_timing_files(storage_client, bucket_name: str):32 bucket = storage_client.bucket(bucket_name)33 with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:34 futures = []35 for timing_file_path in glob.glob("**/.lit_test_times.txt", recursive=True):36 futures.append(37 thread_pool.apply_async(38 _maybe_upload_timing_file, (bucket, timing_file_path)39 )40 )41 for future in futures:42 future.get()43 print("Done uploading")44 45 46def _maybe_download_timing_file(blob):47 file_name = blob.name.removeprefix("lit_timing/")48 pathlib.Path(os.path.dirname(file_name)).mkdir(parents=True, exist_ok=True)49 blob.download_to_filename(file_name)50 51 52def download_timing_files(storage_client, bucket_name: str):53 bucket = storage_client.bucket(bucket_name)54 try:55 blobs = bucket.list_blobs(prefix="lit_timing")56 except exceptions.ClientError as client_error:57 print(58 "::warning file=cache_lit_timing_files.py::Failed to list blobs "59 "in bucket."60 )61 sys.exit(0)62 with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:63 futures = []64 for timing_file_blob in blobs:65 futures.append(66 thread_pool.apply_async(67 _maybe_download_timing_file, (timing_file_blob,)68 )69 )70 for future in futures:71 future.wait()72 if not future.successful():73 print(74 "::warning file=cache_lit_timing_files.py::Failed to "75 "download lit timing file."76 )77 continue78 print("Done downloading")79 80 81if __name__ == "__main__":82 if len(sys.argv) != 2:83 logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")84 sys.exit(1)85 action = sys.argv[1]86 storage_client = storage.Client()87 bucket_name = os.environ["CACHE_GCS_BUCKET"]88 if action == "download":89 download_timing_files(storage_client, bucket_name)90 elif action == "upload":91 upload_timing_files(storage_client, bucket_name)92 else:93 logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")94 sys.exit(1)95