257 lines · python
1#!/usr/bin/env python32# ===----------------------------------------------------------------------===##3#4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5# See https://llvm.org/LICENSE.txt for license information.6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7#8# ===----------------------------------------------------------------------===##9 10"""adb_run.py is a utility for running a libc++ test program via adb.11"""12 13import argparse14import hashlib15import os16import re17import shlex18import socket19import subprocess20import sys21from typing import List, Tuple22 23 24# Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file.25REMOTE_BASE_DIR = "/data/local/tmp/adb_run"26 27g_job_limit_socket = None28g_verbose = False29 30 31def run_adb_sync_command(command: List[str]) -> None:32 """Run an adb command and discard the output, unless the command fails. If33 the command fails, dump the output instead, and exit the script with34 failure.35 """36 if g_verbose:37 sys.stderr.write(f"running: {shlex.join(command)}\n")38 proc = subprocess.run(command, universal_newlines=True,39 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,40 stderr=subprocess.STDOUT, encoding="utf-8")41 if proc.returncode != 0:42 # adb's stdout (e.g. for adb push) should normally be discarded, but43 # on failure, it should be shown. Print it to stderr because it's44 # unrelated to the test program's stdout output. A common error caught45 # here is "No space left on device".46 sys.stderr.write(f"{proc.stdout}\n"47 f"error: adb command exited with {proc.returncode}: "48 f"{shlex.join(command)}\n")49 sys.exit(proc.returncode)50 51 52def sync_test_dir(local_dir: str, remote_dir: str) -> None:53 """Sync the libc++ test directory on the host to the remote device."""54 55 # Optimization: The typical libc++ test directory has only a single56 # *.tmp.exe file in it. In that case, skip the `mkdir` command, which is57 # normally necessary because we don't know if the target directory already58 # exists on the device.59 local_files = os.listdir(local_dir)60 if len(local_files) == 1:61 local_file = os.path.join(local_dir, local_files[0])62 remote_file = os.path.join(remote_dir, local_files[0])63 if not os.path.islink(local_file) and os.path.isfile(local_file):64 run_adb_sync_command(["adb", "push", "--sync", local_file,65 remote_file])66 return67 68 assert os.path.basename(local_dir) == os.path.basename(remote_dir)69 run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir])70 run_adb_sync_command(["adb", "push", "--sync", local_dir,71 os.path.dirname(remote_dir)])72 73 74def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str:75 components = []76 for arg in env_args:77 k, v = arg.split("=", 1)78 components.append(f"export {k}={shlex.quote(v)}; ")79 for k, v in prepend_path_args:80 components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ")81 return "".join(components)82 83 84def run_command(args: argparse.Namespace) -> int:85 local_dir = args.execdir86 assert local_dir.startswith("/")87 assert not local_dir.endswith("/")88 89 # Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using90 # a hash of local_dir so that concurrent adb_run invocations don't create91 # the same intermediate parent directory. At least `adb push` has trouble92 # with concurrent mkdir syscalls on common parent directories. (Somehow93 # mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug,94 # b/289311228.)95 local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest()96 remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}"97 sync_test_dir(local_dir, remote_dir)98 99 adb_shell_command = (100 # Set the environment early so that PATH can be overridden. Overriding101 # PATH is useful for:102 # - Replacing older shell utilities with toybox (e.g. on old devices).103 # - Adding a `bash` command that delegates to `sh` (mksh).104 f"{build_env_arg(args.env, args.prepend_path_env)}"105 106 # Set a high oom_score_adj so that, if the test program uses too much107 # memory, it is killed before anything else on the device. The default108 # oom_score_adj is -1000, so a test using too much memory typically109 # crashes the device.110 "echo 1000 >/proc/self/oom_score_adj; "111 112 # If we're running as root, switch to the shell user. The libc++113 # filesystem tests require running without root permissions. Some x86114 # emulator devices (before Android N) do not have a working `adb unroot`115 # and always run as root. Non-debug builds typically lack `su` and only116 # run as the shell user.117 #118 # Some libc++ tests create temporary files in the working directory,119 # which might be owned by root. Before switching to shell, make the120 # cwd writable (and readable+executable) to every user.121 #122 # N.B.:123 # - Avoid "id -u" because it wasn't supported until Android M.124 # - The `env` and `which` commands were also added in Android M.125 # - Starting in Android M, su from root->shell resets PATH, so we need126 # to modify it again in the new environment.127 # - Avoid chmod's "a+rwx" syntax because it's not supported until128 # Android N.129 # - Defining this function allows specifying the arguments to the test130 # program (i.e. "$@") only once.131 "run_without_root() {"132 " chmod 777 .;"133 " case \"$(id)\" in"134 " *\"uid=0(root)\"*)"135 " if command -v env >/dev/null; then"136 " su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";"137 " else"138 " su shell \"$@\";"139 " fi;;"140 " *) \"$@\";;"141 " esac;"142 "}; "143 )144 145 # Older versions of Bionic limit the length of argv[0] to 127 bytes146 # (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this147 # limit. Changing the working directory works around this limit. The limit148 # is increased to 4095 (PATH_MAX-1) in Android M (API 23).149 command_line = [arg.replace(local_dir + "/", "./") for arg in args.command]150 151 # Prior to the adb feature "shell_v2" (added in Android N), `adb shell`152 # always created a pty:153 # - This merged stdout and stderr together.154 # - The pty converts LF to CRLF.155 # - The exit code of the shell command wasn't propagated.156 # Work around all three limitations, unless "shell_v2" is present.157 proc = subprocess.run(["adb", "features"], check=True,158 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,159 encoding="utf-8")160 adb_features = set(proc.stdout.strip().split())161 has_shell_v2 = "shell_v2" in adb_features162 if has_shell_v2:163 adb_shell_command += (164 f"cd {remote_dir} && run_without_root {shlex.join(command_line)}"165 )166 else:167 adb_shell_command += (168 f"{{"169 f" stdout=$("170 f" cd {remote_dir} && run_without_root {shlex.join(command_line)};"171 f" echo -n __libcxx_adb_exit__=$?"172 f" ); "173 f"}} 2>&1; "174 f"echo -n __libcxx_adb_stdout__\"$stdout\""175 )176 177 adb_command_line = ["adb", "shell", adb_shell_command]178 if g_verbose:179 sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n")180 181 if has_shell_v2:182 proc = subprocess.run(adb_command_line, shell=False, check=False,183 encoding="utf-8")184 return proc.returncode185 else:186 proc = subprocess.run(adb_command_line, shell=False, check=False,187 stdout=subprocess.PIPE, stderr=subprocess.STDOUT,188 encoding="utf-8")189 # The old `adb shell` mode used a pty, which converted LF to CRLF.190 # Convert it back.191 output = proc.stdout.replace("\r\n", "\n")192 193 if proc.returncode:194 sys.stderr.write(f"error: adb failed:\n"195 f" command: {shlex.join(adb_command_line)}\n"196 f" output: {output}\n")197 return proc.returncode198 199 match = re.match(r"(.*)__libcxx_adb_stdout__(.*)__libcxx_adb_exit__=(\d+)$",200 output, re.DOTALL)201 if not match:202 sys.stderr.write(f"error: could not parse adb output:\n"203 f" command: {shlex.join(adb_command_line)}\n"204 f" output: {output}\n")205 return 1206 207 sys.stderr.write(match.group(1))208 sys.stdout.write(match.group(2))209 return int(match.group(3))210 211 212def connect_to_job_limiter_server(sock_addr: str) -> None:213 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)214 215 try:216 sock.connect(sock_addr)217 except (FileNotFoundError, ConnectionRefusedError) as e:218 # Copying-and-pasting an adb_run.py command-line from a lit test failure219 # is likely to fail because the socket no longer exists (or is220 # inactive), so just give a warning.221 sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n")222 return223 224 # The connect call can succeed before the server has called accept, because225 # of the listen backlog, so wait for the server to send a byte.226 sock.recv(1)227 228 # Keep the socket open until this process ends, then let the OS close the229 # connection automatically.230 global g_job_limit_socket231 g_job_limit_socket = sock232 233 234def main() -> int:235 """Main function (pylint wants this docstring)."""236 parser = argparse.ArgumentParser()237 parser.add_argument("--execdir", type=str, required=True)238 parser.add_argument("--env", type=str, required=False, action="append",239 default=[], metavar="NAME=VALUE")240 parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False,241 action="append", default=[],242 metavar=("NAME", "PATH"))243 parser.add_argument("--job-limit-socket")244 parser.add_argument("--verbose", "-v", default=False, action="store_true")245 parser.add_argument("command", nargs=argparse.ONE_OR_MORE)246 args = parser.parse_args()247 248 global g_verbose249 g_verbose = args.verbose250 if args.job_limit_socket is not None:251 connect_to_job_limiter_server(args.job_limit_socket)252 return run_command(args)253 254 255if __name__ == '__main__':256 sys.exit(main())257