129 lines · bash
1#!/bin/bash2#3# Multiqueue: Using pktgen threads for sending on multiple CPUs4# * adding devices to kernel threads which are in the same NUMA node5# * bound devices queue's irq affinity to the threads, 1:1 mapping6# * notice the naming scheme for keeping device names unique7# * nameing scheme: dev@thread_number8# * flow variation via random UDP source port9#10basedir=`dirname $0`11source ${basedir}/functions.sh12root_check_run_with_sudo "$@"13#14# Required param: -i dev in $DEV15source ${basedir}/parameters.sh16 17# Trap EXIT first18trap_exit19 20# Base Config21[ -z "$COUNT" ] && COUNT="20000000" # Zero means indefinitely22[ -z "$CLONE_SKB" ] && CLONE_SKB="0"23 24# Flow variation random source port between min and max25UDP_SRC_MIN=926UDP_SRC_MAX=10927 28node=`get_iface_node $DEV`29irq_array=(`get_iface_irqs $DEV`)30cpu_array=(`get_node_cpus $node`)31 32[ $THREADS -gt ${#irq_array[*]} -o $THREADS -gt ${#cpu_array[*]} ] && \33 err 1 "Thread number $THREADS exceeds: min (${#irq_array[*]},${#cpu_array[*]})"34 35# (example of setting default params in your script)36if [ -z "$DEST_IP" ]; then37 [ -z "$IP6" ] && DEST_IP="198.18.0.42" || DEST_IP="FD00::1"38fi39[ -z "$DST_MAC" ] && DST_MAC="90:e2:ba:ff:ff:ff"40if [ -n "$DEST_IP" ]; then41 validate_addr${IP6} $DEST_IP42 read -r DST_MIN DST_MAX <<< $(parse_addr${IP6} $DEST_IP)43fi44if [ -n "$DST_PORT" ]; then45 read -r UDP_DST_MIN UDP_DST_MAX <<< $(parse_ports $DST_PORT)46 validate_ports $UDP_DST_MIN $UDP_DST_MAX47fi48 49# General cleanup everything since last run50[ -z "$APPEND" ] && pg_ctrl "reset"51 52# Threads are specified with parameter -t value in $THREADS53for ((i = 0; i < $THREADS; i++)); do54 # The device name is extended with @name, using thread number to55 # make then unique, but any name will do.56 # Set the queue's irq affinity to this $thread (processor)57 # if '-f' is designated, offset cpu id58 thread=${cpu_array[$((i+F_THREAD))]}59 dev=${DEV}@${thread}60 echo $thread > /proc/irq/${irq_array[$i]}/smp_affinity_list61 info "irq ${irq_array[$i]} is set affinity to `cat /proc/irq/${irq_array[$i]}/smp_affinity_list`"62 63 # Add remove all other devices and add_device $dev to thread64 [ -z "$APPEND" ] && pg_thread $thread "rem_device_all"65 pg_thread $thread "add_device" $dev66 67 # select queue and bind the queue and $dev in 1:1 relationship68 queue_num=$i69 info "queue number is $queue_num"70 pg_set $dev "queue_map_min $queue_num"71 pg_set $dev "queue_map_max $queue_num"72 73 # Notice config queue to map to cpu (mirrors smp_processor_id())74 # It is beneficial to map IRQ /proc/irq/*/smp_affinity 1:1 to CPU number75 pg_set $dev "flag QUEUE_MAP_CPU"76 77 # Base config of dev78 pg_set $dev "count $COUNT"79 pg_set $dev "clone_skb $CLONE_SKB"80 pg_set $dev "pkt_size $PKT_SIZE"81 pg_set $dev "delay $DELAY"82 83 # Flag example disabling timestamping84 pg_set $dev "flag NO_TIMESTAMP"85 86 # Destination87 pg_set $dev "dst_mac $DST_MAC"88 pg_set $dev "dst${IP6}_min $DST_MIN"89 pg_set $dev "dst${IP6}_max $DST_MAX"90 91 if [ -n "$DST_PORT" ]; then92 # Single destination port or random port range93 pg_set $dev "flag UDPDST_RND"94 pg_set $dev "udp_dst_min $UDP_DST_MIN"95 pg_set $dev "udp_dst_max $UDP_DST_MAX"96 fi97 98 [ ! -z "$UDP_CSUM" ] && pg_set $dev "flag UDPCSUM"99 100 # Setup random UDP port src range101 pg_set $dev "flag UDPSRC_RND"102 pg_set $dev "udp_src_min $UDP_SRC_MIN"103 pg_set $dev "udp_src_max $UDP_SRC_MAX"104done105 106# Run if user hits control-c107function print_result() {108 # Print results109 for ((i = 0; i < $THREADS; i++)); do110 thread=${cpu_array[$((i+F_THREAD))]}111 dev=${DEV}@${thread}112 echo "Device: $dev"113 cat /proc/net/pktgen/$dev | grep -A2 "Result:"114 done115}116# trap keyboard interrupt (Ctrl-C)117trap true SIGINT118 119# start_run120if [ -z "$APPEND" ]; then121 echo "Running... ctrl^C to stop" >&2122 pg_ctrl "start"123 echo "Done" >&2124 125 print_result126else127 echo "Append mode: config done. Do more or use 'pg_ctrl start' to run"128fi129