607 lines · c
1//===-- Shared memory RPC server instantiation ------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is intended to be used externally as part of the `shared/`10// interface. For that purpose, we manually define a few options normally11// handled by the libc build system.12//13//===----------------------------------------------------------------------===//14 15#ifndef LLVM_LIBC_SRC___SUPPORT_RPC_RPC_SERVER_H16#define LLVM_LIBC_SRC___SUPPORT_RPC_RPC_SERVER_H17 18#include "src/__support/macros/properties/compiler.h"19 20// Workaround for missing __has_builtin in < GCC 10.21#ifndef __has_builtin22#define __has_builtin(x) 023#endif24 25// Workaround for missing __builtin_is_constant_evaluated in < GCC 10.26// Also this builtin is defined for GCC 9.27#if !(__has_builtin(__builtin_is_constant_evaluated) || \28 (defined(LIBC_COMPILER_IS_GCC) && (LIBC_COMPILER_GCC_VER >= 900)))29#define __builtin_is_constant_evaluated(x) 030#endif31 32// Configs for using the LLVM libc writer interface.33#define LIBC_COPT_USE_C_ASSERT34#define LIBC_COPT_MEMCPY_USE_EMBEDDED_TINY35#define LIBC_COPT_ARRAY_ARG_LIST36#define LIBC_COPT_PRINTF_DISABLE_WRITE_INT37#define LIBC_COPT_PRINTF_DISABLE_INDEX_MODE38#define LIBC_COPT_PRINTF_DISABLE_STRERROR39 40// The 'long double' type is 8 bytes.41#define LIBC_TYPES_LONG_DOUBLE_IS_FLOAT6442 43#include "shared/rpc.h"44#include "shared/rpc_opcodes.h"45 46#include "src/__support/arg_list.h"47#include "src/stdio/printf_core/converter.h"48#include "src/stdio/printf_core/parser.h"49#include "src/stdio/printf_core/writer.h"50 51#include "hdr/stdio_overlay.h"52#include "hdr/stdlib_overlay.h"53 54namespace LIBC_NAMESPACE_DECL {55namespace internal {56 57// Minimal replacement for 'std::vector' that works for trivial types.58template <typename T> class TempVector {59 static_assert(cpp::is_trivially_constructible<T>::value &&60 cpp::is_trivially_destructible<T>::value,61 "Not a trivial type.");62 T *data;63 size_t current;64 size_t capacity;65 66public:67 LIBC_INLINE TempVector() : data(nullptr), current(0), capacity(0) {}68 69 LIBC_INLINE ~TempVector() { free(data); }70 71 LIBC_INLINE void push_back(const T &value) {72 if (current == capacity)73 grow();74 data[current] = T(value);75 ++current;76 }77 78 LIBC_INLINE void push_back(T &&value) {79 if (current == capacity)80 grow();81 data[current] = T(static_cast<T &&>(value));82 ++current;83 }84 85 LIBC_INLINE void pop_back() { --current; }86 87 LIBC_INLINE bool empty() { return current == 0; }88 89 LIBC_INLINE size_t size() { return current; }90 91 LIBC_INLINE T &operator[](size_t index) { return data[index]; }92 93 LIBC_INLINE T &back() { return data[current - 1]; }94 95private:96 LIBC_INLINE void grow() {97 size_t new_capacity = capacity ? capacity * 2 : 1;98 void *new_data = realloc(data, new_capacity * sizeof(T));99 data = static_cast<T *>(new_data);100 capacity = new_capacity;101 }102};103 104struct TempStorage {105 LIBC_INLINE char *alloc(size_t size) {106 storage.push_back(reinterpret_cast<char *>(malloc(size)));107 return storage.back();108 }109 110 LIBC_INLINE ~TempStorage() {111 for (size_t i = 0; i < storage.size(); ++i)112 free(storage[i]);113 }114 115 TempVector<char *> storage;116};117 118// Get the associated stream out of an encoded number.119LIBC_INLINE static ::FILE *to_stream(uintptr_t f) {120 enum Stream {121 File = 0,122 Stdin = 1,123 Stdout = 2,124 Stderr = 3,125 };126 127 ::FILE *stream = reinterpret_cast<FILE *>(f & ~0x3ull);128 Stream type = static_cast<Stream>(f & 0x3ull);129 if (type == Stdin)130 return stdin;131 if (type == Stdout)132 return stdout;133 if (type == Stderr)134 return stderr;135 return stream;136}137 138template <bool packed, uint32_t num_lanes>139LIBC_INLINE static void handle_printf(rpc::Server::Port &port,140 TempStorage &temp_storage) {141 FILE *files[num_lanes] = {nullptr};142 // Get the appropriate output stream to use.143 if (port.get_opcode() == LIBC_PRINTF_TO_STREAM ||144 port.get_opcode() == LIBC_PRINTF_TO_STREAM_PACKED) {145 port.recv([&](rpc::Buffer *buffer, uint32_t id) {146 files[id] = reinterpret_cast<FILE *>(buffer->data[0]);147 });148 } else if (port.get_opcode() == LIBC_PRINTF_TO_STDOUT ||149 port.get_opcode() == LIBC_PRINTF_TO_STDOUT_PACKED) {150 for (uint32_t i = 0; i < num_lanes; ++i)151 files[i] = stdout;152 } else {153 for (uint32_t i = 0; i < num_lanes; ++i)154 files[i] = stderr;155 }156 157 uint64_t format_sizes[num_lanes] = {0};158 void *format[num_lanes] = {nullptr};159 160 uint64_t args_sizes[num_lanes] = {0};161 void *args[num_lanes] = {nullptr};162 163 // Recieve the format string and arguments from the client.164 port.recv_n(format, format_sizes,165 [&](uint64_t size) { return temp_storage.alloc(size); });166 167 // Parse the format string to get the expected size of the buffer.168 for (uint32_t lane = 0; lane < num_lanes; ++lane) {169 if (!format[lane])170 continue;171 172 printf_core::WriteBuffer<173 printf_core::WriteMode::FILL_BUFF_AND_DROP_OVERFLOW>174 wb(nullptr, 0);175 printf_core::Writer writer(wb);176 177 internal::DummyArgList<packed> printf_args;178 printf_core::Parser<internal::DummyArgList<packed> &> parser(179 reinterpret_cast<const char *>(format[lane]), printf_args);180 181 for (printf_core::FormatSection cur_section = parser.get_next_section();182 !cur_section.raw_string.empty();183 cur_section = parser.get_next_section())184 ;185 args_sizes[lane] = printf_args.read_count();186 }187 port.send([&](rpc::Buffer *buffer, uint32_t id) {188 buffer->data[0] = args_sizes[id];189 });190 port.recv_n(args, args_sizes,191 [&](uint64_t size) { return temp_storage.alloc(size); });192 193 // Identify any arguments that are actually pointers to strings on the client.194 // Additionally we want to determine how much buffer space we need to print.195 TempVector<void *> strs_to_copy[num_lanes];196 int buffer_size[num_lanes] = {0};197 for (uint32_t lane = 0; lane < num_lanes; ++lane) {198 if (!format[lane])199 continue;200 201 printf_core::WriteBuffer<202 printf_core::WriteMode::FILL_BUFF_AND_DROP_OVERFLOW>203 wb(nullptr, 0);204 printf_core::Writer writer(wb);205 206 internal::StructArgList<packed> printf_args(args[lane], args_sizes[lane]);207 printf_core::Parser<internal::StructArgList<packed>> parser(208 reinterpret_cast<const char *>(format[lane]), printf_args);209 210 for (printf_core::FormatSection cur_section = parser.get_next_section();211 !cur_section.raw_string.empty();212 cur_section = parser.get_next_section()) {213 if (cur_section.has_conv && cur_section.conv_name == 's' &&214 cur_section.conv_val_ptr) {215 strs_to_copy[lane].push_back(cur_section.conv_val_ptr);216 // Get the minimum size of the string in the case of padding.217 char c = '\0';218 cur_section.conv_val_ptr = &c;219 convert(&writer, cur_section);220 } else if (cur_section.has_conv) {221 // Ignore conversion errors for the first pass.222 convert(&writer, cur_section);223 } else {224 writer.write(cur_section.raw_string);225 }226 }227 buffer_size[lane] = writer.get_chars_written();228 }229 230 // Recieve any strings from the client and push them into a buffer.231 TempVector<void *> copied_strs[num_lanes];232 auto HasPendingCopies = [](TempVector<void *> v[num_lanes]) {233 for (uint32_t i = 0; i < num_lanes; ++i)234 if (!v[i].empty() && v[i].back())235 return true;236 return false;237 };238 while (HasPendingCopies(strs_to_copy)) {239 port.send([&](rpc::Buffer *buffer, uint32_t id) {240 void *ptr = !strs_to_copy[id].empty() ? strs_to_copy[id].back() : nullptr;241 buffer->data[1] = reinterpret_cast<uintptr_t>(ptr);242 if (!strs_to_copy[id].empty())243 strs_to_copy[id].pop_back();244 });245 uint64_t str_sizes[num_lanes] = {0};246 void *strs[num_lanes] = {nullptr};247 port.recv_n(strs, str_sizes,248 [&](uint64_t size) { return temp_storage.alloc(size); });249 for (uint32_t lane = 0; lane < num_lanes; ++lane) {250 if (!strs[lane])251 continue;252 253 copied_strs[lane].push_back(strs[lane]);254 buffer_size[lane] += str_sizes[lane];255 }256 }257 258 // Perform the final formatting and printing using the LLVM C library printf.259 int results[num_lanes] = {0};260 for (uint32_t lane = 0; lane < num_lanes; ++lane) {261 if (!format[lane])262 continue;263 264 char *buffer = temp_storage.alloc(buffer_size[lane]);265 printf_core::WriteBuffer<266 printf_core::WriteMode::FILL_BUFF_AND_DROP_OVERFLOW>267 wb(buffer, buffer_size[lane]);268 printf_core::Writer writer(wb);269 270 internal::StructArgList<packed> printf_args(args[lane], args_sizes[lane]);271 printf_core::Parser<internal::StructArgList<packed>> parser(272 reinterpret_cast<const char *>(format[lane]), printf_args);273 274 // Parse and print the format string using the arguments we copied from275 // the client.276 int ret = 0;277 for (printf_core::FormatSection cur_section = parser.get_next_section();278 !cur_section.raw_string.empty();279 cur_section = parser.get_next_section()) {280 // If this argument was a string we use the memory buffer we copied from281 // the client by replacing the raw pointer with the copied one.282 if (cur_section.has_conv && cur_section.conv_name == 's') {283 if (!copied_strs[lane].empty()) {284 cur_section.conv_val_ptr = copied_strs[lane].back();285 copied_strs[lane].pop_back();286 } else {287 cur_section.conv_val_ptr = nullptr;288 }289 }290 if (cur_section.has_conv) {291 ret = convert(&writer, cur_section);292 if (ret == -1)293 break;294 } else {295 writer.write(cur_section.raw_string);296 }297 }298 299 results[lane] = static_cast<int>(300 fwrite(buffer, 1, writer.get_chars_written(), files[lane]));301 if (size_t(results[lane]) != writer.get_chars_written() || ret == -1)302 results[lane] = -1;303 }304 305 // Send the final return value and signal completion by setting the string306 // argument to null.307 port.send([&](rpc::Buffer *buffer, uint32_t id) {308 buffer->data[0] = static_cast<uint64_t>(results[id]);309 buffer->data[1] = reinterpret_cast<uintptr_t>(nullptr);310 });311}312 313template <uint32_t num_lanes>314LIBC_INLINE static rpc::Status handle_port_impl(rpc::Server::Port &port) {315 TempStorage temp_storage;316 317 switch (port.get_opcode()) {318 case LIBC_WRITE_TO_STREAM:319 case LIBC_WRITE_TO_STDERR:320 case LIBC_WRITE_TO_STDOUT:321 case LIBC_WRITE_TO_STDOUT_NEWLINE: {322 uint64_t sizes[num_lanes] = {0};323 void *strs[num_lanes] = {nullptr};324 FILE *files[num_lanes] = {nullptr};325 if (port.get_opcode() == LIBC_WRITE_TO_STREAM) {326 port.recv([&](rpc::Buffer *buffer, uint32_t id) {327 files[id] = reinterpret_cast<FILE *>(buffer->data[0]);328 });329 } else {330 for (uint32_t i = 0; i < num_lanes; ++i)331 files[i] = port.get_opcode() == LIBC_WRITE_TO_STDERR ? stderr : stdout;332 }333 334 port.recv_n(strs, sizes,335 [&](uint64_t size) { return temp_storage.alloc(size); });336 port.send([&](rpc::Buffer *buffer, uint32_t id) {337 flockfile(files[id]);338 buffer->data[0] = fwrite_unlocked(strs[id], 1, sizes[id], files[id]);339 if (port.get_opcode() == LIBC_WRITE_TO_STDOUT_NEWLINE &&340 buffer->data[0] == sizes[id])341 buffer->data[0] += fwrite_unlocked("\n", 1, 1, files[id]);342 funlockfile(files[id]);343 });344 break;345 }346 case LIBC_READ_FROM_STREAM: {347 uint64_t sizes[num_lanes] = {0};348 void *data[num_lanes] = {nullptr};349 port.recv([&](rpc::Buffer *buffer, uint32_t id) {350 data[id] = temp_storage.alloc(buffer->data[0]);351 sizes[id] =352 fread(data[id], 1, buffer->data[0], to_stream(buffer->data[1]));353 });354 port.send_n(data, sizes);355 port.send([&](rpc::Buffer *buffer, uint32_t id) {356 __builtin_memcpy(buffer->data, &sizes[id], sizeof(uint64_t));357 });358 break;359 }360 case LIBC_READ_FGETS: {361 uint64_t sizes[num_lanes] = {0};362 void *data[num_lanes] = {nullptr};363 port.recv([&](rpc::Buffer *buffer, uint32_t id) {364 data[id] = temp_storage.alloc(buffer->data[0]);365 const char *str = ::fgets(reinterpret_cast<char *>(data[id]),366 static_cast<int>(buffer->data[0]),367 to_stream(buffer->data[1]));368 sizes[id] = !str ? 0 : __builtin_strlen(str) + 1;369 });370 port.send_n(data, sizes);371 break;372 }373 case LIBC_OPEN_FILE: {374 uint64_t sizes[num_lanes] = {0};375 void *paths[num_lanes] = {nullptr};376 port.recv_n(paths, sizes,377 [&](uint64_t size) { return temp_storage.alloc(size); });378 port.recv_and_send([&](rpc::Buffer *buffer, uint32_t id) {379 FILE *file = fopen(reinterpret_cast<char *>(paths[id]),380 reinterpret_cast<char *>(buffer->data));381 buffer->data[0] = reinterpret_cast<uintptr_t>(file);382 });383 break;384 }385 case LIBC_CLOSE_FILE: {386 port.recv_and_send([&](rpc::Buffer *buffer, uint32_t) {387 FILE *file = reinterpret_cast<FILE *>(buffer->data[0]);388 buffer->data[0] = ::fclose(file);389 });390 break;391 }392 case LIBC_EXIT: {393 // Send a response to the client to signal that we are ready to exit.394 port.recv_and_send([](rpc::Buffer *, uint32_t) {});395 port.recv([](rpc::Buffer *buffer, uint32_t) {396 int status = 0;397 __builtin_memcpy(&status, buffer->data, sizeof(int));398 // We want a quick exit to avoid conflicts with offloading library399 // teardowns when called from the GPU.400 quick_exit(status);401 });402 break;403 }404 case LIBC_ABORT: {405 // Send a response to the client to signal that we are ready to abort.406 port.recv_and_send([](rpc::Buffer *, uint32_t) {});407 port.recv([](rpc::Buffer *, uint32_t) {});408 abort();409 break;410 }411 case LIBC_HOST_CALL: {412 uint64_t sizes[num_lanes] = {0};413 unsigned long long results[num_lanes] = {0};414 void *args[num_lanes] = {nullptr};415 port.recv_n(args, sizes,416 [&](uint64_t size) { return temp_storage.alloc(size); });417 port.recv([&](rpc::Buffer *buffer, uint32_t id) {418 using func_ptr_t = unsigned long long (*)(void *);419 auto func = reinterpret_cast<func_ptr_t>(buffer->data[0]);420 results[id] = func(args[id]);421 });422 port.send([&](rpc::Buffer *buffer, uint32_t id) {423 buffer->data[0] = static_cast<uint64_t>(results[id]);424 });425 break;426 }427 case LIBC_FEOF: {428 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {429 buffer->data[0] = feof(to_stream(buffer->data[0]));430 });431 break;432 }433 case LIBC_FERROR: {434 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {435 buffer->data[0] = ferror(to_stream(buffer->data[0]));436 });437 break;438 }439 case LIBC_CLEARERR: {440 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {441 clearerr(to_stream(buffer->data[0]));442 });443 break;444 }445 case LIBC_FSEEK: {446 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {447 buffer->data[0] =448 fseek(to_stream(buffer->data[0]), static_cast<long>(buffer->data[1]),449 static_cast<int>(buffer->data[2]));450 });451 break;452 }453 case LIBC_FTELL: {454 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {455 buffer->data[0] = ftell(to_stream(buffer->data[0]));456 });457 break;458 }459 case LIBC_FFLUSH: {460 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {461 buffer->data[0] = fflush(to_stream(buffer->data[0]));462 });463 break;464 }465 case LIBC_UNGETC: {466 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {467 buffer->data[0] =468 ungetc(static_cast<int>(buffer->data[0]), to_stream(buffer->data[1]));469 });470 break;471 }472 case LIBC_PRINTF_TO_STREAM_PACKED:473 case LIBC_PRINTF_TO_STDOUT_PACKED:474 case LIBC_PRINTF_TO_STDERR_PACKED: {475 handle_printf<true, num_lanes>(port, temp_storage);476 break;477 }478 case LIBC_PRINTF_TO_STREAM:479 case LIBC_PRINTF_TO_STDOUT:480 case LIBC_PRINTF_TO_STDERR: {481 handle_printf<false, num_lanes>(port, temp_storage);482 break;483 }484 case LIBC_REMOVE: {485 uint64_t sizes[num_lanes] = {0};486 void *args[num_lanes] = {nullptr};487 port.recv_n(args, sizes,488 [&](uint64_t size) { return temp_storage.alloc(size); });489 port.send([&](rpc::Buffer *buffer, uint32_t id) {490 buffer->data[0] = static_cast<uint64_t>(491 remove(reinterpret_cast<const char *>(args[id])));492 });493 break;494 }495 case LIBC_RENAME: {496 uint64_t oldsizes[num_lanes] = {0};497 uint64_t newsizes[num_lanes] = {0};498 void *oldpath[num_lanes] = {nullptr};499 void *newpath[num_lanes] = {nullptr};500 port.recv_n(oldpath, oldsizes,501 [&](uint64_t size) { return temp_storage.alloc(size); });502 port.recv_n(newpath, newsizes,503 [&](uint64_t size) { return temp_storage.alloc(size); });504 port.send([&](rpc::Buffer *buffer, uint32_t id) {505 buffer->data[0] = static_cast<uint64_t>(506 rename(reinterpret_cast<const char *>(oldpath[id]),507 reinterpret_cast<const char *>(newpath[id])));508 });509 break;510 }511 case LIBC_SYSTEM: {512 uint64_t sizes[num_lanes] = {0};513 void *args[num_lanes] = {nullptr};514 port.recv_n(args, sizes,515 [&](uint64_t size) { return temp_storage.alloc(size); });516 port.send([&](rpc::Buffer *buffer, uint32_t id) {517 buffer->data[0] = static_cast<uint64_t>(518 system(reinterpret_cast<const char *>(args[id])));519 });520 break;521 }522 case LIBC_TEST_INCREMENT: {523 port.recv_and_send([](rpc::Buffer *buffer, uint32_t) {524 reinterpret_cast<uint64_t *>(buffer->data)[0] += 1;525 });526 break;527 }528 case LIBC_TEST_INTERFACE: {529 bool end_with_recv;530 uint64_t cnt;531 port.recv([&](rpc::Buffer *buffer, uint32_t) {532 end_with_recv = buffer->data[0];533 });534 port.recv([&](rpc::Buffer *buffer, uint32_t) { cnt = buffer->data[0]; });535 port.send([&](rpc::Buffer *buffer, uint32_t) {536 buffer->data[0] = cnt = cnt + 1;537 });538 port.recv([&](rpc::Buffer *buffer, uint32_t) { cnt = buffer->data[0]; });539 port.send([&](rpc::Buffer *buffer, uint32_t) {540 buffer->data[0] = cnt = cnt + 1;541 });542 port.recv([&](rpc::Buffer *buffer, uint32_t) { cnt = buffer->data[0]; });543 port.recv([&](rpc::Buffer *buffer, uint32_t) { cnt = buffer->data[0]; });544 port.send([&](rpc::Buffer *buffer, uint32_t) {545 buffer->data[0] = cnt = cnt + 1;546 });547 port.send([&](rpc::Buffer *buffer, uint32_t) {548 buffer->data[0] = cnt = cnt + 1;549 });550 if (end_with_recv)551 port.recv([&](rpc::Buffer *buffer, uint32_t) { cnt = buffer->data[0]; });552 else553 port.send([&](rpc::Buffer *buffer, uint32_t) {554 buffer->data[0] = cnt = cnt + 1;555 });556 557 break;558 }559 case LIBC_TEST_STREAM: {560 uint64_t sizes[num_lanes] = {0};561 void *dst[num_lanes] = {nullptr};562 port.recv_n(dst, sizes,563 [](uint64_t size) -> void * { return new char[size]; });564 port.send_n(dst, sizes);565 for (uint64_t i = 0; i < num_lanes; ++i) {566 if (dst[i])567 delete[] reinterpret_cast<uint8_t *>(dst[i]);568 }569 break;570 }571 case LIBC_NOOP: {572 port.recv([](rpc::Buffer *, uint32_t) {});573 break;574 }575 default:576 return rpc::RPC_UNHANDLED_OPCODE;577 }578 579 return rpc::RPC_SUCCESS;580}581 582} // namespace internal583} // namespace LIBC_NAMESPACE_DECL584 585namespace LIBC_NAMESPACE_DECL {586namespace rpc {587 588// Handles any opcode generated from the 'libc' client code.589LIBC_INLINE ::rpc::Status handle_libc_opcodes(::rpc::Server::Port &port,590 uint32_t num_lanes) {591 switch (num_lanes) {592 case 1:593 return internal::handle_port_impl<1>(port);594 case 32:595 return internal::handle_port_impl<32>(port);596 case 64:597 return internal::handle_port_impl<64>(port);598 default:599 return ::rpc::RPC_ERROR;600 }601}602 603} // namespace rpc604} // namespace LIBC_NAMESPACE_DECL605 606#endif // LLVM_LIBC_SRC___SUPPORT_RPC_RPC_SERVER_H607