613 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2018 Google Inc.4 * Author: Eric Dumazet (edumazet@google.com)5 *6 * Reference program demonstrating tcp mmap() usage,7 * and SO_RCVLOWAT hints for receiver.8 *9 * Note : NIC with header split is needed to use mmap() on TCP :10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.11 *12 * How to use on loopback interface :13 *14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)15 * tcp_mmap -s -z &16 * tcp_mmap -H ::1 -z17 *18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)19 * (4096 : page size on x86, 12: TCP TS option length)20 * tcp_mmap -s -z -M $((4096+12)) &21 * tcp_mmap -H ::1 -z -M $((4096+12))22 *23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)25 *26 * $ ./tcp_mmap -s & # Without mmap()27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches36 * $ kill %1 # kill tcp_mmap server37 *38 * $ ./tcp_mmap -s -z & # With mmap()39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches48 */49#define _GNU_SOURCE50#include <pthread.h>51#include <sys/types.h>52#include <fcntl.h>53#include <error.h>54#include <sys/socket.h>55#include <sys/mman.h>56#include <sys/resource.h>57#include <unistd.h>58#include <string.h>59#include <stdlib.h>60#include <stdio.h>61#include <errno.h>62#include <time.h>63#include <sys/time.h>64#include <netinet/in.h>65#include <arpa/inet.h>66#include <poll.h>67#include <linux/tcp.h>68#include <assert.h>69#include <openssl/pem.h>70 71#ifndef MSG_ZEROCOPY72#define MSG_ZEROCOPY 0x400000073#endif74 75#ifndef min76#define min(a, b) ((a) < (b) ? (a) : (b))77#endif78 79#define FILE_SZ (1ULL << 35)80static int cfg_family = AF_INET6;81static socklen_t cfg_alen = sizeof(struct sockaddr_in6);82static int cfg_port = 8787;83 84static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */85static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */86static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */87static int xflg; /* hash received data (simple xor) (-h option) */88static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */89static int integrity; /* -i option: sender and receiver compute sha256 over the data.*/90 91static size_t chunk_size = 512*1024;92 93static size_t map_align;94 95unsigned long htotal;96unsigned int digest_len;97 98static inline void prefetch(const void *x)99{100#if defined(__x86_64__)101 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));102#endif103}104 105void hash_zone(void *zone, unsigned int length)106{107 unsigned long temp = htotal;108 109 while (length >= 8*sizeof(long)) {110 prefetch(zone + 384);111 temp ^= *(unsigned long *)zone;112 temp ^= *(unsigned long *)(zone + sizeof(long));113 temp ^= *(unsigned long *)(zone + 2*sizeof(long));114 temp ^= *(unsigned long *)(zone + 3*sizeof(long));115 temp ^= *(unsigned long *)(zone + 4*sizeof(long));116 temp ^= *(unsigned long *)(zone + 5*sizeof(long));117 temp ^= *(unsigned long *)(zone + 6*sizeof(long));118 temp ^= *(unsigned long *)(zone + 7*sizeof(long));119 zone += 8*sizeof(long);120 length -= 8*sizeof(long);121 }122 while (length >= 1) {123 temp ^= *(unsigned char *)zone;124 zone += 1;125 length--;126 }127 htotal = temp;128}129 130#define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))131#define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))132 133 134static void *mmap_large_buffer(size_t need, size_t *allocated)135{136 void *buffer;137 size_t sz;138 139 /* Attempt to use huge pages if possible. */140 sz = ALIGN_UP(need, map_align);141 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,142 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);143 144 if (buffer == (void *)-1) {145 sz = need;146 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,147 MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE,148 -1, 0);149 if (buffer != (void *)-1)150 fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n");151 }152 *allocated = sz;153 return buffer;154}155 156static uint32_t tcp_info_get_rcv_mss(int fd)157{158 socklen_t sz = sizeof(struct tcp_info);159 struct tcp_info info;160 161 if (getsockopt(fd, IPPROTO_TCP, TCP_INFO, &info, &sz)) {162 fprintf(stderr, "Error fetching TCP_INFO\n");163 return 0;164 }165 166 return info.tcpi_rcv_mss;167}168 169void *child_thread(void *arg)170{171 unsigned char digest[SHA256_DIGEST_LENGTH];172 unsigned long total_mmap = 0, total = 0;173 struct tcp_zerocopy_receive zc;174 unsigned char *buffer = NULL;175 unsigned long delta_usec;176 EVP_MD_CTX *ctx = NULL;177 int flags = MAP_SHARED;178 struct timeval t0, t1;179 void *raddr = NULL;180 void *addr = NULL;181 double throughput;182 struct rusage ru;183 size_t buffer_sz;184 int lu, fd;185 186 fd = (int)(unsigned long)arg;187 188 gettimeofday(&t0, NULL);189 190 fcntl(fd, F_SETFL, O_NDELAY);191 buffer = mmap_large_buffer(chunk_size, &buffer_sz);192 if (buffer == (void *)-1) {193 perror("mmap");194 goto error;195 }196 if (zflg) {197 raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0);198 if (raddr == (void *)-1) {199 perror("mmap");200 zflg = 0;201 } else {202 addr = ALIGN_PTR_UP(raddr, map_align);203 }204 }205 if (integrity) {206 ctx = EVP_MD_CTX_new();207 if (!ctx) {208 perror("cannot enable SHA computing");209 goto error;210 }211 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);212 }213 while (1) {214 struct pollfd pfd = { .fd = fd, .events = POLLIN, };215 int sub;216 217 poll(&pfd, 1, 10000);218 if (zflg) {219 socklen_t zc_len = sizeof(zc);220 int res;221 222 memset(&zc, 0, sizeof(zc));223 zc.address = (__u64)((unsigned long)addr);224 zc.length = min(chunk_size, FILE_SZ - total);225 226 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,227 &zc, &zc_len);228 if (res == -1)229 break;230 231 if (zc.length) {232 assert(zc.length <= chunk_size);233 if (integrity)234 EVP_DigestUpdate(ctx, addr, zc.length);235 total_mmap += zc.length;236 if (xflg)237 hash_zone(addr, zc.length);238 /* It is more efficient to unmap the pages right now,239 * instead of doing this in next TCP_ZEROCOPY_RECEIVE.240 */241 madvise(addr, zc.length, MADV_DONTNEED);242 total += zc.length;243 }244 if (zc.recv_skip_hint) {245 assert(zc.recv_skip_hint <= chunk_size);246 lu = read(fd, buffer, min(zc.recv_skip_hint,247 FILE_SZ - total));248 if (lu > 0) {249 if (integrity)250 EVP_DigestUpdate(ctx, buffer, lu);251 if (xflg)252 hash_zone(buffer, lu);253 total += lu;254 }255 if (lu == 0)256 goto end;257 }258 continue;259 }260 sub = 0;261 while (sub < chunk_size) {262 lu = read(fd, buffer + sub, min(chunk_size - sub,263 FILE_SZ - total));264 if (lu == 0)265 goto end;266 if (lu < 0)267 break;268 if (integrity)269 EVP_DigestUpdate(ctx, buffer + sub, lu);270 if (xflg)271 hash_zone(buffer + sub, lu);272 total += lu;273 sub += lu;274 }275 }276end:277 gettimeofday(&t1, NULL);278 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;279 280 if (integrity) {281 fcntl(fd, F_SETFL, 0);282 EVP_DigestFinal_ex(ctx, digest, &digest_len);283 lu = read(fd, buffer, SHA256_DIGEST_LENGTH);284 if (lu != SHA256_DIGEST_LENGTH)285 perror("Error: Cannot read SHA256\n");286 287 if (memcmp(digest, buffer,288 SHA256_DIGEST_LENGTH))289 fprintf(stderr, "Error: SHA256 of the data is not right\n");290 else291 printf("\nSHA256 is correct\n");292 }293 294 throughput = 0;295 if (delta_usec)296 throughput = total * 8.0 / (double)delta_usec / 1000.0;297 getrusage(RUSAGE_THREAD, &ru);298 if (total > 1024*1024) {299 unsigned long total_usec;300 unsigned long mb = total >> 20;301 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +302 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;303 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"304 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches, rcv_mss %u\n",305 total / (1024.0 * 1024.0),306 100.0*total_mmap/total,307 (double)delta_usec / 1000000.0,308 throughput,309 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,310 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,311 (double)total_usec/mb,312 ru.ru_nvcsw,313 tcp_info_get_rcv_mss(fd));314 }315error:316 munmap(buffer, buffer_sz);317 close(fd);318 if (zflg)319 munmap(raddr, chunk_size + map_align);320 pthread_exit(0);321}322 323static void apply_rcvsnd_buf(int fd)324{325 if (rcvbuf && setsockopt(fd, SOL_SOCKET,326 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {327 perror("setsockopt SO_RCVBUF");328 }329 330 if (sndbuf && setsockopt(fd, SOL_SOCKET,331 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {332 perror("setsockopt SO_SNDBUF");333 }334}335 336 337static void setup_sockaddr(int domain, const char *str_addr,338 struct sockaddr_storage *sockaddr)339{340 struct sockaddr_in6 *addr6 = (void *) sockaddr;341 struct sockaddr_in *addr4 = (void *) sockaddr;342 343 switch (domain) {344 case PF_INET:345 memset(addr4, 0, sizeof(*addr4));346 addr4->sin_family = AF_INET;347 addr4->sin_port = htons(cfg_port);348 if (str_addr &&349 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)350 error(1, 0, "ipv4 parse error: %s", str_addr);351 break;352 case PF_INET6:353 memset(addr6, 0, sizeof(*addr6));354 addr6->sin6_family = AF_INET6;355 addr6->sin6_port = htons(cfg_port);356 if (str_addr &&357 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)358 error(1, 0, "ipv6 parse error: %s", str_addr);359 break;360 default:361 error(1, 0, "illegal domain");362 }363}364 365static void do_accept(int fdlisten)366{367 pthread_attr_t attr;368 int rcvlowat;369 370 pthread_attr_init(&attr);371 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);372 373 rcvlowat = chunk_size;374 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,375 &rcvlowat, sizeof(rcvlowat)) == -1) {376 perror("setsockopt SO_RCVLOWAT");377 }378 379 apply_rcvsnd_buf(fdlisten);380 381 while (1) {382 struct sockaddr_in addr;383 socklen_t addrlen = sizeof(addr);384 pthread_t th;385 int fd, res;386 387 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);388 if (fd == -1) {389 perror("accept");390 continue;391 }392 res = pthread_create(&th, &attr, child_thread,393 (void *)(unsigned long)fd);394 if (res) {395 errno = res;396 perror("pthread_create");397 close(fd);398 }399 }400}401 402/* Each thread should reserve a big enough vma to avoid403 * spinlock collisions in ptl locks.404 * This size is 2MB on x86_64, and is exported in /proc/meminfo.405 */406static unsigned long default_huge_page_size(void)407{408 FILE *f = fopen("/proc/meminfo", "r");409 unsigned long hps = 0;410 size_t linelen = 0;411 char *line = NULL;412 413 if (!f)414 return 0;415 while (getline(&line, &linelen, f) > 0) {416 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {417 hps <<= 10;418 break;419 }420 }421 free(line);422 fclose(f);423 return hps;424}425 426static void randomize(void *target, size_t count)427{428 static int urandom = -1;429 ssize_t got;430 431 urandom = open("/dev/urandom", O_RDONLY);432 if (urandom < 0) {433 perror("open /dev/urandom");434 exit(1);435 }436 got = read(urandom, target, count);437 if (got != count) {438 perror("read /dev/urandom");439 exit(1);440 }441}442 443int main(int argc, char *argv[])444{445 unsigned char digest[SHA256_DIGEST_LENGTH];446 struct sockaddr_storage listenaddr, addr;447 unsigned int max_pacing_rate = 0;448 EVP_MD_CTX *ctx = NULL;449 unsigned char *buffer;450 uint64_t total = 0;451 char *host = NULL;452 int fd, c, on = 1;453 size_t buffer_sz;454 int sflg = 0;455 int mss = 0;456 457 while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:C:a:i")) != -1) {458 switch (c) {459 case '4':460 cfg_family = PF_INET;461 cfg_alen = sizeof(struct sockaddr_in);462 break;463 case '6':464 cfg_family = PF_INET6;465 cfg_alen = sizeof(struct sockaddr_in6);466 break;467 case 'p':468 cfg_port = atoi(optarg);469 break;470 case 'H':471 host = optarg;472 break;473 case 's': /* server : listen for incoming connections */474 sflg++;475 break;476 case 'r':477 rcvbuf = atoi(optarg);478 break;479 case 'w':480 sndbuf = atoi(optarg);481 break;482 case 'z':483 zflg = 1;484 break;485 case 'M':486 mss = atoi(optarg);487 break;488 case 'x':489 xflg = 1;490 break;491 case 'k':492 keepflag = 1;493 break;494 case 'P':495 max_pacing_rate = atoi(optarg) ;496 break;497 case 'C':498 chunk_size = atol(optarg);499 break;500 case 'a':501 map_align = atol(optarg);502 break;503 case 'i':504 integrity = 1;505 break;506 default:507 exit(1);508 }509 }510 if (!map_align) {511 map_align = default_huge_page_size();512 /* if really /proc/meminfo is not helping,513 * we use the default x86_64 hugepagesize.514 */515 if (!map_align)516 map_align = 2*1024*1024;517 }518 if (sflg) {519 int fdlisten = socket(cfg_family, SOCK_STREAM, 0);520 521 if (fdlisten == -1) {522 perror("socket");523 exit(1);524 }525 apply_rcvsnd_buf(fdlisten);526 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));527 528 setup_sockaddr(cfg_family, host, &listenaddr);529 530 if (mss &&531 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,532 &mss, sizeof(mss)) == -1) {533 perror("setsockopt TCP_MAXSEG");534 exit(1);535 }536 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {537 perror("bind");538 exit(1);539 }540 if (listen(fdlisten, 128) == -1) {541 perror("listen");542 exit(1);543 }544 do_accept(fdlisten);545 }546 547 buffer = mmap_large_buffer(chunk_size, &buffer_sz);548 if (buffer == (unsigned char *)-1) {549 perror("mmap");550 exit(1);551 }552 553 fd = socket(cfg_family, SOCK_STREAM, 0);554 if (fd == -1) {555 perror("socket");556 exit(1);557 }558 apply_rcvsnd_buf(fd);559 560 setup_sockaddr(cfg_family, host, &addr);561 562 if (mss &&563 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {564 perror("setsockopt TCP_MAXSEG");565 exit(1);566 }567 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {568 perror("connect");569 exit(1);570 }571 if (max_pacing_rate &&572 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,573 &max_pacing_rate, sizeof(max_pacing_rate)) == -1)574 perror("setsockopt SO_MAX_PACING_RATE");575 576 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,577 &on, sizeof(on)) == -1) {578 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");579 zflg = 0;580 }581 if (integrity) {582 randomize(buffer, buffer_sz);583 ctx = EVP_MD_CTX_new();584 if (!ctx) {585 perror("cannot enable SHA computing");586 exit(1);587 }588 EVP_DigestInit_ex(ctx, EVP_sha256(), NULL);589 }590 while (total < FILE_SZ) {591 size_t offset = total % chunk_size;592 int64_t wr = FILE_SZ - total;593 594 if (wr > chunk_size - offset)595 wr = chunk_size - offset;596 /* Note : we just want to fill the pipe with random bytes */597 wr = send(fd, buffer + offset,598 (size_t)wr, zflg ? MSG_ZEROCOPY : 0);599 if (wr <= 0)600 break;601 if (integrity)602 EVP_DigestUpdate(ctx, buffer + offset, wr);603 total += wr;604 }605 if (integrity && total == FILE_SZ) {606 EVP_DigestFinal_ex(ctx, digest, &digest_len);607 send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0);608 }609 close(fd);610 munmap(buffer, buffer_sz);611 return 0;612}613