516 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 Facebook3 *4 * This program is free software; you can redistribute it and/or5 * modify it under the terms of version 2 of the GNU General Public6 * License as published by the Free Software Foundation.7 *8 * Example program for Host Bandwidth Managment9 *10 * This program loads a cgroup skb BPF program to enforce cgroup output11 * (egress) or input (ingress) bandwidth limits.12 *13 * USAGE: hbm [-d] [-l] [-n <id>] [-r <rate>] [-s] [-t <secs>] [-w] [-h] [prog]14 * Where:15 * -d Print BPF trace debug buffer16 * -l Also limit flows doing loopback17 * -n <#> To create cgroup \"/hbm#\" and attach prog18 * Default is /hbm119 * --no_cn Do not return cn notifications20 * -r <rate> Rate limit in Mbps21 * -s Get HBM stats (marked, dropped, etc.)22 * -t <time> Exit after specified seconds (default is 0)23 * -w Work conserving flag. cgroup can increase its bandwidth24 * beyond the rate limit specified while there is available25 * bandwidth. Current implementation assumes there is only26 * NIC (eth0), but can be extended to support multiple NICs.27 * Currrently only supported for egress.28 * -h Print this info29 * prog BPF program file name. Name defaults to hbm_out_kern.o30 */31 32#define _GNU_SOURCE33 34#include <stdio.h>35#include <stdlib.h>36#include <assert.h>37#include <sys/time.h>38#include <unistd.h>39#include <errno.h>40#include <fcntl.h>41#include <linux/unistd.h>42#include <linux/compiler.h>43 44#include <linux/bpf.h>45#include <bpf/bpf.h>46#include <getopt.h>47 48#include "cgroup_helpers.h"49#include "hbm.h"50#include "bpf_util.h"51#include <bpf/libbpf.h>52 53bool outFlag = true;54int minRate = 1000; /* cgroup rate limit in Mbps */55int rate = 1000; /* can grow if rate conserving is enabled */56int dur = 1;57bool stats_flag;58bool loopback_flag;59bool debugFlag;60bool work_conserving_flag;61bool no_cn_flag;62bool edt_flag;63 64static void Usage(void);65static void read_trace_pipe2(void);66static void do_error(char *msg, bool errno_flag);67 68#define TRACEFS "/sys/kernel/tracing/"69 70static struct bpf_program *bpf_prog;71static struct bpf_object *obj;72static int queue_stats_fd;73 74static void read_trace_pipe2(void)75{76 int trace_fd;77 FILE *outf;78 char *outFname = "hbm_out.log";79 80 trace_fd = open(TRACEFS "trace_pipe", O_RDONLY, 0);81 if (trace_fd < 0) {82 printf("Error opening trace_pipe\n");83 return;84 }85 86// Future support of ingress87// if (!outFlag)88// outFname = "hbm_in.log";89 outf = fopen(outFname, "w");90 91 if (outf == NULL)92 printf("Error creating %s\n", outFname);93 94 while (1) {95 static char buf[4097];96 ssize_t sz;97 98 sz = read(trace_fd, buf, sizeof(buf) - 1);99 if (sz > 0) {100 buf[sz] = 0;101 puts(buf);102 if (outf != NULL) {103 fprintf(outf, "%s\n", buf);104 fflush(outf);105 }106 }107 }108}109 110static void do_error(char *msg, bool errno_flag)111{112 if (errno_flag)113 printf("ERROR: %s, errno: %d\n", msg, errno);114 else115 printf("ERROR: %s\n", msg);116 exit(1);117}118 119static int prog_load(char *prog)120{121 struct bpf_program *pos;122 const char *sec_name;123 124 obj = bpf_object__open_file(prog, NULL);125 if (libbpf_get_error(obj)) {126 printf("ERROR: opening BPF object file failed\n");127 return 1;128 }129 130 /* load BPF program */131 if (bpf_object__load(obj)) {132 printf("ERROR: loading BPF object file failed\n");133 goto err;134 }135 136 bpf_object__for_each_program(pos, obj) {137 sec_name = bpf_program__section_name(pos);138 if (sec_name && !strcmp(sec_name, "cgroup_skb/egress")) {139 bpf_prog = pos;140 break;141 }142 }143 if (!bpf_prog) {144 printf("ERROR: finding a prog in obj file failed\n");145 goto err;146 }147 148 queue_stats_fd = bpf_object__find_map_fd_by_name(obj, "queue_stats");149 if (queue_stats_fd < 0) {150 printf("ERROR: finding a map in obj file failed\n");151 goto err;152 }153 154 return 0;155 156err:157 bpf_object__close(obj);158 return 1;159}160 161static int run_bpf_prog(char *prog, int cg_id)162{163 struct hbm_queue_stats qstats = {0};164 char cg_dir[100], cg_pin_path[100];165 struct bpf_link *link = NULL;166 int key = 0;167 int cg1 = 0;168 int rc = 0;169 170 sprintf(cg_dir, "/hbm%d", cg_id);171 rc = prog_load(prog);172 if (rc != 0)173 return rc;174 175 if (setup_cgroup_environment()) {176 printf("ERROR: setting cgroup environment\n");177 goto err;178 }179 cg1 = create_and_get_cgroup(cg_dir);180 if (!cg1) {181 printf("ERROR: create_and_get_cgroup\n");182 goto err;183 }184 if (join_cgroup(cg_dir)) {185 printf("ERROR: join_cgroup\n");186 goto err;187 }188 189 qstats.rate = rate;190 qstats.stats = stats_flag ? 1 : 0;191 qstats.loopback = loopback_flag ? 1 : 0;192 qstats.no_cn = no_cn_flag ? 1 : 0;193 if (bpf_map_update_elem(queue_stats_fd, &key, &qstats, BPF_ANY)) {194 printf("ERROR: Could not update map element\n");195 goto err;196 }197 198 if (!outFlag)199 bpf_program__set_expected_attach_type(bpf_prog, BPF_CGROUP_INET_INGRESS);200 201 link = bpf_program__attach_cgroup(bpf_prog, cg1);202 if (libbpf_get_error(link)) {203 fprintf(stderr, "ERROR: bpf_program__attach_cgroup failed\n");204 goto err;205 }206 207 sprintf(cg_pin_path, "/sys/fs/bpf/hbm%d", cg_id);208 rc = bpf_link__pin(link, cg_pin_path);209 if (rc < 0) {210 printf("ERROR: bpf_link__pin failed: %d\n", rc);211 goto err;212 }213 214 if (work_conserving_flag) {215 struct timeval t0, t_last, t_new;216 FILE *fin;217 unsigned long long last_eth_tx_bytes, new_eth_tx_bytes;218 signed long long last_cg_tx_bytes, new_cg_tx_bytes;219 signed long long delta_time, delta_bytes, delta_rate;220 int delta_ms;221#define DELTA_RATE_CHECK 10000 /* in us */222#define RATE_THRESHOLD 9500000000 /* 9.5 Gbps */223 224 bpf_map_lookup_elem(queue_stats_fd, &key, &qstats);225 if (gettimeofday(&t0, NULL) < 0)226 do_error("gettimeofday failed", true);227 t_last = t0;228 fin = fopen("/sys/class/net/eth0/statistics/tx_bytes", "r");229 if (fscanf(fin, "%llu", &last_eth_tx_bytes) != 1)230 do_error("fscanf fails", false);231 fclose(fin);232 last_cg_tx_bytes = qstats.bytes_total;233 while (true) {234 usleep(DELTA_RATE_CHECK);235 if (gettimeofday(&t_new, NULL) < 0)236 do_error("gettimeofday failed", true);237 delta_ms = (t_new.tv_sec - t0.tv_sec) * 1000 +238 (t_new.tv_usec - t0.tv_usec)/1000;239 if (delta_ms > dur * 1000)240 break;241 delta_time = (t_new.tv_sec - t_last.tv_sec) * 1000000 +242 (t_new.tv_usec - t_last.tv_usec);243 if (delta_time == 0)244 continue;245 t_last = t_new;246 fin = fopen("/sys/class/net/eth0/statistics/tx_bytes",247 "r");248 if (fscanf(fin, "%llu", &new_eth_tx_bytes) != 1)249 do_error("fscanf fails", false);250 fclose(fin);251 printf(" new_eth_tx_bytes:%llu\n",252 new_eth_tx_bytes);253 bpf_map_lookup_elem(queue_stats_fd, &key, &qstats);254 new_cg_tx_bytes = qstats.bytes_total;255 delta_bytes = new_eth_tx_bytes - last_eth_tx_bytes;256 last_eth_tx_bytes = new_eth_tx_bytes;257 delta_rate = (delta_bytes * 8000000) / delta_time;258 printf("%5d - eth_rate:%.1fGbps cg_rate:%.3fGbps",259 delta_ms, delta_rate/1000000000.0,260 rate/1000.0);261 if (delta_rate < RATE_THRESHOLD) {262 /* can increase cgroup rate limit, but first263 * check if we are using the current limit.264 * Currently increasing by 6.25%, unknown265 * if that is the optimal rate.266 */267 int rate_diff100;268 269 delta_bytes = new_cg_tx_bytes -270 last_cg_tx_bytes;271 last_cg_tx_bytes = new_cg_tx_bytes;272 delta_rate = (delta_bytes * 8000000) /273 delta_time;274 printf(" rate:%.3fGbps",275 delta_rate/1000000000.0);276 rate_diff100 = (((long long)rate)*1000000 -277 delta_rate) * 100 /278 (((long long) rate) * 1000000);279 printf(" rdiff:%d", rate_diff100);280 if (rate_diff100 <= 3) {281 rate += (rate >> 4);282 if (rate > RATE_THRESHOLD / 1000000)283 rate = RATE_THRESHOLD / 1000000;284 qstats.rate = rate;285 printf(" INC\n");286 } else {287 printf("\n");288 }289 } else {290 /* Need to decrease cgroup rate limit.291 * Currently decreasing by 12.5%, unknown292 * if that is optimal293 */294 printf(" DEC\n");295 rate -= (rate >> 3);296 if (rate < minRate)297 rate = minRate;298 qstats.rate = rate;299 }300 if (bpf_map_update_elem(queue_stats_fd, &key, &qstats, BPF_ANY))301 do_error("update map element fails", false);302 }303 } else {304 sleep(dur);305 }306 // Get stats!307 if (stats_flag && bpf_map_lookup_elem(queue_stats_fd, &key, &qstats)) {308 char fname[100];309 FILE *fout;310 311 if (!outFlag)312 sprintf(fname, "hbm.%d.in", cg_id);313 else314 sprintf(fname, "hbm.%d.out", cg_id);315 fout = fopen(fname, "w");316 fprintf(fout, "id:%d\n", cg_id);317 fprintf(fout, "ERROR: Could not lookup queue_stats\n");318 fclose(fout);319 } else if (stats_flag && qstats.lastPacketTime >320 qstats.firstPacketTime) {321 long long delta_us = (qstats.lastPacketTime -322 qstats.firstPacketTime)/1000;323 unsigned int rate_mbps = ((qstats.bytes_total -324 qstats.bytes_dropped) * 8 /325 delta_us);326 double percent_pkts, percent_bytes;327 char fname[100];328 FILE *fout;329 int k;330 static const char *returnValNames[] = {331 "DROP_PKT",332 "ALLOW_PKT",333 "DROP_PKT_CWR",334 "ALLOW_PKT_CWR"335 };336#define RET_VAL_COUNT 4337 338// Future support of ingress339// if (!outFlag)340// sprintf(fname, "hbm.%d.in", cg_id);341// else342 sprintf(fname, "hbm.%d.out", cg_id);343 fout = fopen(fname, "w");344 fprintf(fout, "id:%d\n", cg_id);345 fprintf(fout, "rate_mbps:%d\n", rate_mbps);346 fprintf(fout, "duration:%.1f secs\n",347 (qstats.lastPacketTime - qstats.firstPacketTime) /348 1000000000.0);349 fprintf(fout, "packets:%d\n", (int)qstats.pkts_total);350 fprintf(fout, "bytes_MB:%d\n", (int)(qstats.bytes_total /351 1000000));352 fprintf(fout, "pkts_dropped:%d\n", (int)qstats.pkts_dropped);353 fprintf(fout, "bytes_dropped_MB:%d\n",354 (int)(qstats.bytes_dropped /355 1000000));356 // Marked Pkts and Bytes357 percent_pkts = (qstats.pkts_marked * 100.0) /358 (qstats.pkts_total + 1);359 percent_bytes = (qstats.bytes_marked * 100.0) /360 (qstats.bytes_total + 1);361 fprintf(fout, "pkts_marked_percent:%6.2f\n", percent_pkts);362 fprintf(fout, "bytes_marked_percent:%6.2f\n", percent_bytes);363 364 // Dropped Pkts and Bytes365 percent_pkts = (qstats.pkts_dropped * 100.0) /366 (qstats.pkts_total + 1);367 percent_bytes = (qstats.bytes_dropped * 100.0) /368 (qstats.bytes_total + 1);369 fprintf(fout, "pkts_dropped_percent:%6.2f\n", percent_pkts);370 fprintf(fout, "bytes_dropped_percent:%6.2f\n", percent_bytes);371 372 // ECN CE markings373 percent_pkts = (qstats.pkts_ecn_ce * 100.0) /374 (qstats.pkts_total + 1);375 fprintf(fout, "pkts_ecn_ce:%6.2f (%d)\n", percent_pkts,376 (int)qstats.pkts_ecn_ce);377 378 // Average cwnd379 fprintf(fout, "avg cwnd:%d\n",380 (int)(qstats.sum_cwnd / (qstats.sum_cwnd_cnt + 1)));381 // Average rtt382 fprintf(fout, "avg rtt:%d\n",383 (int)(qstats.sum_rtt / (qstats.pkts_total + 1)));384 // Average credit385 if (edt_flag)386 fprintf(fout, "avg credit_ms:%.03f\n",387 (qstats.sum_credit /388 (qstats.pkts_total + 1.0)) / 1000000.0);389 else390 fprintf(fout, "avg credit:%d\n",391 (int)(qstats.sum_credit /392 (1500 * ((int)qstats.pkts_total ) + 1)));393 394 // Return values stats395 for (k = 0; k < RET_VAL_COUNT; k++) {396 percent_pkts = (qstats.returnValCount[k] * 100.0) /397 (qstats.pkts_total + 1);398 fprintf(fout, "%s:%6.2f (%d)\n", returnValNames[k],399 percent_pkts, (int)qstats.returnValCount[k]);400 }401 fclose(fout);402 }403 404 if (debugFlag)405 read_trace_pipe2();406 goto cleanup;407 408err:409 rc = 1;410 411cleanup:412 bpf_link__destroy(link);413 bpf_object__close(obj);414 415 if (cg1 != -1)416 close(cg1);417 418 if (rc != 0)419 cleanup_cgroup_environment();420 return rc;421}422 423static void Usage(void)424{425 printf("This program loads a cgroup skb BPF program to enforce\n"426 "cgroup output (egress) bandwidth limits.\n\n"427 "USAGE: hbm [-o] [-d] [-l] [-n <id>] [--no_cn] [-r <rate>]\n"428 " [-s] [-t <secs>] [-w] [-h] [prog]\n"429 " Where:\n"430 " -o indicates egress direction (default)\n"431 " -d print BPF trace debug buffer\n"432 " --edt use fq's Earliest Departure Time\n"433 " -l also limit flows using loopback\n"434 " -n <#> to create cgroup \"/hbm#\" and attach prog\n"435 " Default is /hbm1\n"436 " --no_cn disable CN notifications\n"437 " -r <rate> Rate in Mbps\n"438 " -s Update HBM stats\n"439 " -t <time> Exit after specified seconds (default is 0)\n"440 " -w Work conserving flag. cgroup can increase\n"441 " bandwidth beyond the rate limit specified\n"442 " while there is available bandwidth. Current\n"443 " implementation assumes there is only eth0\n"444 " but can be extended to support multiple NICs\n"445 " -h print this info\n"446 " prog BPF program file name. Name defaults to\n"447 " hbm_out_kern.o\n");448}449 450int main(int argc, char **argv)451{452 char *prog = "hbm_out_kern.o";453 int k;454 int cg_id = 1;455 char *optstring = "iodln:r:st:wh";456 struct option loptions[] = {457 {"no_cn", 0, NULL, 1},458 {"edt", 0, NULL, 2},459 {NULL, 0, NULL, 0}460 };461 462 while ((k = getopt_long(argc, argv, optstring, loptions, NULL)) != -1) {463 switch (k) {464 case 1:465 no_cn_flag = true;466 break;467 case 2:468 prog = "hbm_edt_kern.o";469 edt_flag = true;470 break;471 case'o':472 break;473 case 'd':474 debugFlag = true;475 break;476 case 'l':477 loopback_flag = true;478 break;479 case 'n':480 cg_id = atoi(optarg);481 break;482 case 'r':483 minRate = atoi(optarg) * 1.024;484 rate = minRate;485 break;486 case 's':487 stats_flag = true;488 break;489 case 't':490 dur = atoi(optarg);491 break;492 case 'w':493 work_conserving_flag = true;494 break;495 case '?':496 if (optopt == 'n' || optopt == 'r' || optopt == 't')497 fprintf(stderr,498 "Option -%c requires an argument.\n\n",499 optopt);500 case 'h':501 default:502 Usage();503 return 0;504 }505 }506 507 if (optind < argc)508 prog = argv[optind];509 printf("HBM prog: %s\n", prog != NULL ? prog : "NULL");510 511 /* Use libbpf 1.0 API mode */512 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);513 514 return run_bpf_prog(prog, cg_id);515}516