#!/bin/sh # # SPDX-License-Identifier: BSD-2-Clause # # Copyright (c) 2016-2026 Devin Teske # ############################################################ IDENT(1) # # $Title: netgraph(4) management script for vnet jails $ # $Version: 9.3 $ # ############################################################ INFORMATION # # Use this tool with jail.conf(5) (or rc.conf(5) ``legacy'' configuration) to # manage `vnet' interfaces for jails. Designed to automate the creation of vnet # interface(s) during jail `prestart', return them to the host during jail # `prestop', and destroy said interface(s) during jail `poststop'. # # In jail.conf(5) format: # # ### BEGIN EXCERPT ### # # xxx { # host.hostname = "xxx.yyy"; # path = "/vm/$name"; # # # # # NB: Below 2-lines required # # NB: The number of ngN_$name interfaces should match the number of # # arguments given to `jng bridge $name' in exec.prestart value. # # # vnet; # vnet.interface = ng0_$name, ng1_$name, ...; # # exec.clean; # exec.system_user = "root"; # exec.jail_user = "root"; # # # # # NB: Below lines required # # NB: The number of arguments after `jng bridge $name' should match # # the number of ngN_$name arguments in vnet.interface value. # # NB: Return each ngN_$name to the host in prestop so the kernel does # # not move it during jail removal (BPF race). Destroy in poststop. # # # exec.prestart += "jng bridge $name em0 em1 ..."; # exec.prestop += "ifconfig ng0_$name -vnet $name"; # exec.prestop += "ifconfig ng1_$name -vnet $name"; # exec.poststop += "jng shutdown $name"; # # # Standard recipe # exec.start += "/bin/sh /etc/rc"; # exec.stop = "/bin/sh /etc/rc.shutdown jail"; # exec.consolelog = "/var/log/jail_${name}_console.log"; # mount.devfs; # # # Optional (default off) # #allow.mount; # #allow.set_hostname = 1; # #allow.sysvipc = 1; # #devfs_ruleset = "11"; # rule to unhide bpf for DHCP # } # # ### END EXCERPT ### # # In rc.conf(5) ``legacy'' format (used when /etc/jail.conf does not exist): # # ### BEGIN EXCERPT ### # # jail_enable="YES" # #jail_confwarn="NO" # Optional: disable warning to migrate to jail.conf(5) # jail_list="xxx" # # # # # Global presets for all jails # # # jail_devfs_enable="YES" # mount devfs # # # # # Global options (default off) # # # #jail_mount_enable="YES" # mount /etc/fstab.{name} # #jail_set_hostname_allow="YES" # Allow hostname to change # #jail_sysvipc_allow="YES" # Allow SysV Interprocess Comm. # # # xxx # jail_xxx_hostname="xxx.shxd.cx" # hostname # jail_xxx_rootdir="/vm/xxx" # root directory # jail_xxx_vnet_interfaces="ng0_xxx ng1xxx ..." # vnet interface(s) # jail_xxx_exec_prestart0="jng bridge xxx em0 em1 ..." # bridge interface(s) # jail_xxx_exec_prestop0="ifconfig ng0_xxx -vnet xxx" # return ifnet(s) # jail_xxx_exec_prestop1="ifconfig ng1_xxx -vnet xxx" # jail_xxx_exec_poststop0="jng shutdown xxx" # destroy interface(s) # #jail_xxx_mount_enable="YES" # mount /etc/fstab.xxx # #jail_xxx_devfs_ruleset="11" # rule to unhide bpf for DHCP # # ### END EXCERPT ### # # Note that the legacy rc.conf(5) format is converted to # /var/run/jail.{name}.conf by /etc/rc.d/jail if jail.conf(5) is missing. # # ASIDE: dhclient(8) inside a vnet jail... # # To allow dhclient(8) to work inside a vnet jail, make sure the following # appears in /etc/devfs.rules (which should be created if it doesn't exist): # # [devfsrules_jail=11] # add include $devfsrules_hide_all # add include $devfsrules_unhide_basic # add include $devfsrules_unhide_login # add path 'bpf*' unhide # # And set ether devfs.ruleset="11" (jail.conf(5)) or # jail_{name}_devfs_ruleset="11" (rc.conf(5)). # # NB: While this tool can't create every type of desirable topology, it should # handle most setups, minus some considered exotic or purpose-built. # # Uplink on ng_ether(4) `lower' (jng 7+) keeps the WAN MAC table small: # ng_bridge(4) does not learn on uplink hooks. The first connected hook being # uplink also selects restrictive unknown-unicast: frames for an unknown dest # go only to uplink, not to jail links. Inbound unicast to a jail therefore # requires that jail's MAC to live in the forwarding database (FDB) on the # jail's link. jng 8 pins each eiface MAC with ngctl movehost and sets # maxStaleness so host->staleness cannot catch it. `jng pin NAME' replants # after an accidental move. ng_bridge must not MOVE_HOST from learnMac=0 hooks # or promiscuous TX echo can steal a pinned MAC onto uplink; without that # kernel fix, re-run `jng pin'. # ############################################################ CONFIGURATION # # host->staleness is uint16_t; conf.maxStaleness is uint32_t. # ng_bridge_timeout expires when ++staleness >= maxStaleness. # A threshold above 65535 is unreachable (the counter wraps). # NG_BRIDGE_MAX_STALENESS=4294967295 ############################################################ GLOBALS VERSION='$Version: 9.3 $' pgm="${0##*/}" # Program basename # # Global exit status # SUCCESS=0 FAILURE=1 # # Command-line options # STATS_FMT=text # -j for JSON ############################################################ FUNCTIONS quietly(){ "$@" > /dev/null 2>&1; } die() { local fmt="$1" if [ "$fmt" ]; then shift 1 # fmt printf "%s: $fmt\n" "$pgm" "$@" >&2 fi exit $FAILURE } usage() { local fmt="$1" local optfmt="\t%-5s %s\n" local action usage descr exec >&2 if [ "$fmt" ]; then shift 1 # fmt printf "%s: $fmt\n" "$pgm" "$@" fi printf "Usage: %s [-hv] action [arguments]\n" "$pgm" printf "Options:\n" printf "$optfmt" "-h" "Print this usage statement and exit." printf "$optfmt" "-v" "Print version information and exit." printf "Actions:\n" for action in \ bridge \ graph \ pin \ show \ show1 \ shutdown \ stats \ ; do eval usage=\"\$jng_${action}_usage\" [ "$usage" ] || continue eval descr=\"\$jng_${action}_descr\" printf "\t%s\n\t\t%s\n" "$usage" "$descr" done die } action_usage() { local usage descr action="$1" fmt="$2" shift 1 # action if [ "$fmt" ]; then shift 1 # fmt printf "%s: %s: $fmt\n" "$pgm" "$action" "$@" >&2 fi eval usage=\"\$jng_${action}_usage\" printf "Usage: %s %s\n" "$pgm" "$usage" >&2 eval descr=\"\$jng_${action}_descr\" printf "\t%s\n" "$descr" >&2 die } derive_mac() { local OPTIND=1 OPTARG __flag local __mac_num= __make_pair= while getopts 2n: __flag; do case "$__flag" in 2) __make_pair=1 ;; n) __mac_num=${OPTARG%%[^0-9]*} ;; esac done shift $(( $OPTIND - 1 )) if [ ! "$__mac_num" ]; then eval __mac_num=\${_${iface}_num:--1} __mac_num=$(( $__mac_num + 1 )) eval _${iface}_num=\$__mac_num fi local __iface="$1" __name="$2" __var_to_set="$3" __var_to_set_b="$4" local __iface_devid __new_devid __num __new_devid_b # # Calculate MAC address derived from given iface. # # The formula used is ``NP:SS:SS:II:II:II'' where: # + N denotes 4 bits used as a counter to support branching # each parent interface up to 15 times under the same jail # name (see S below). # + P denotes the special nibble whose value, if one of # 2, 6, A, or E (but usually 2) denotes a privately # administered MAC address (while remaining routable). # + S denotes 16 bits, the sum(1) value of the jail name. # + I denotes bits that are inherited from parent interface. # # The S bits are a CRC-16 checksum of NAME, allowing the jail # to change link numbers in ng_bridge(4) without effecting the # MAC address. Meanwhile, if... # + the jail NAME changes (e.g., it was duplicated and given # a new name with no other changes) # + the underlying network interface changes # + the jail is moved to another host # the MAC address will be recalculated to a new, similarly # unique value preventing conflict. # __iface_devid=$( ifconfig $__iface ether | awk '/ether/,$0=$2' ) # ??:??:??:II:II:II __new_devid=${__iface_devid#??:??:??} # => :II:II:II # => :SS:SS:II:II:II __num=$( set -- $( echo -n "$__name" | sum ) && echo $1 ) __new_devid=$( printf :%02x:%02x \ $(( $__num >> 8 & 255 )) $(( $__num & 255 )) )$__new_devid # => P:SS:SS:II:II:II case "$__iface_devid" in ?2:*) __new_devid=a$__new_devid __new_devid_b=e$__new_devid ;; ?[Ee]:*) __new_devid=2$__new_devid __new_devid_b=6$__new_devid ;; *) __new_devid=2$__new_devid __new_devid_b=e$__new_devid esac # => NP:SS:SS:II:II:II __new_devid=$( printf %x $(( $__mac_num & 15 )) )$__new_devid __new_devid_b=$( printf %x $(( $__mac_num & 15 )) )$__new_devid_b # # Return derivative MAC address(es) # if [ "$__make_pair" ]; then if [ "$__var_to_set" -a "$__var_to_set_b" ]; then eval $__var_to_set=\$__new_devid eval $__var_to_set_b=\$__new_devid_b else echo $__new_devid $__new_devid_b fi else if [ "$__var_to_set" ]; then eval $__var_to_set=\$__new_devid else echo $__new_devid fi fi } mustberoot_to_continue() { [ "$( id -u )" -eq 0 ] || die "Must run as root!" } jng_bridge_has_uplink() { ngctl show "$1:" 2> /dev/null | awk ' $1 ~ /^uplink/ { found = 1; exit } END { exit !found } ' # END-QUOTE } jng_bridge_persist_hosts() { local node="$1" local debug=0 loop=60 stable=1 config eval $( ngctl msg "$node:" getconfig 2> /dev/null | awk ' { if (match($0, /debugLevel=[0-9]+/)) printf "debug=%s ", substr($0, RSTART + 11, RLENGTH - 11) if (match($0, /loopTimeout=[0-9]+/)) printf "loop=%s ", substr($0, RSTART + 12, RLENGTH - 12) if (match($0, /minStableAge=[0-9]+/)) printf "stable=%s ", substr($0, RSTART + 13, RLENGTH - 13) } ' ) config="debugLevel=$debug" config="$config loopTimeout=$loop" config="$config maxStaleness=$NG_BRIDGE_MAX_STALENESS" config="$config minStableAge=$stable" quietly ngctl msg "$node:" setconfig "{ $config }" } jng_pin_mac() { local node="$1" mac="$2" hook="$3" [ "$node" -a "$mac" -a "$hook" ] || return $FAILURE quietly ngctl msg "$node:" movehost "{ addr=$mac hook=\"$hook\" }" } jng_jiface_mac() { local __jiface="$1" __jail="$2" __var_to_set="$3" local __mac __mac=$( ifconfig "$__jiface" ether 2> /dev/null | awk '/ether/ { print $2; exit }' ) if [ ! "$__mac" -a "$__jail" ]; then # # After vnet.interface takes the eiface, it is gone # from the host ifconfig; netgraph node remains. # __mac=$( jexec "$__jail" ifconfig "$__jiface" ether \ 2> /dev/null | awk '/ether/ { print $2; exit }' ) fi eval $__var_to_set=\"\$__mac\" [ "$__mac" ] } jng_pin_jiface() { local jiface="$1" jail="$2" local mac peer peerhook pbridge phook jng_jiface_mac "$jiface" "$jail" mac || return $FAILURE # ether bridge set -- $( ngctl show "$jiface:" 2> /dev/null | awk ' $3 == "bridge" { print $2, $5; exit } ' ) peer="$1" peerhook="$2" [ "$peer" -a "$peerhook" ] || return $FAILURE jng_pin_mac "$peer" "$mac" "$peerhook" || return if jng_bridge_has_uplink "$peer"; then jng_bridge_persist_hosts "$peer" || : persist optional return fi # # Secondary bridge: also pin on the parent that holds uplink # (restrictive unknown-unicast lives there). # set -- $( ngctl show "$peer:" 2> /dev/null | awk ' $3 == "bridge" { print $2, $5; exit } ' ) pbridge="$1" phook="$2" [ "$pbridge" -a "$phook" ] || return $SUCCESS jng_pin_mac "$pbridge" "$mac" "$phook" || return jng_bridge_persist_hosts "$pbridge" || : persist optional } jng_bridge_usage="bridge [-h] [-b BRIDGE_NAME] NAME [!|=]iface0 [[!|=]iface1 ...]" jng_bridge_descr="Create ng0_NAME [ng1_NAME ...]" jng_bridge() { local OPTIND=1 OPTARG flag bridge=bridge while getopts b:h flag; do case "$flag" in b) bridge="$OPTARG" [ "$bridge" ] || action_usage bridge "-b argument cannot be empty" ;; # NOTREACHED *) action_usage bridge # NOTREACHED esac done shift $(( $OPTIND - 1 )) [ $# -gt 0 ] || action_usage bridge "too few arguments" # NOTREACHED local name="$1" [ "${name:-x}" = "${name#*[![:print:]]}" ] || action_usage bridge "invalid bridge name: %s" "$name" # NOTREACHED shift 1 # name mustberoot_to_continue local iface parent jiface jiface_devid local new clone_mac no_derive num quad mtu i=0 for iface in $*; do clone_mac= no_derive= case "$iface" in =*) iface=${iface#=} clone_mac=1 ;; !*) iface=${iface#!} no_derive=1 ;; esac # Make sure the interface doesn't exist already jiface=ng${i}_$name if quietly ngctl msg "$jiface:" getifname; then i=$(( $i + 1 )) continue fi # Bring the interface up ifconfig $iface up || return # Set promiscuous mode and don't overwrite src addr ngctl msg $iface: setpromisc 1 || return ngctl msg $iface: setautosrc 0 || return # Make sure the interface has been bridged # NB: You must connect uplinkX before linkX # NB: see ng_bridge(4) for policy on first connected hook if ! quietly ngctl info ${iface}bridge:; then ngctl mkpeer $iface: bridge lower uplink1 || return ngctl connect $iface: $iface:lower upper link0 || return ngctl name $iface:lower ${iface}bridge || return jng_bridge_persist_hosts ${iface}bridge || : persist optional fi mtu=$( ifconfig $iface | sed -n '1s/^.*mtu //p' ) || return # Optionally create a secondary bridge # NB: This time, you want to only connect linkX (no uplinkX) if [ "$bridge" != "bridge" ] && ! quietly ngctl info "$iface$bridge:" then num=1 while quietly ngctl msg ${iface}bridge: getstats $num do num=$(( $num + 1 )) done ngctl mkpeer $iface:lower bridge link$num link0 || return ngctl name ${iface}bridge:link$num "$iface$bridge" || return fi # Create a new interface to the bridge num=1 while quietly ngctl msg "$iface$bridge:" getstats $num; do num=$(( $num + 1 )) done ngctl mkpeer "$iface$bridge:" eiface link$num ether || return # Rename the new interface while [ ${#jiface} -gt 15 ]; do # OS limitation jiface=${jiface%?} done new=$( ngctl show -n "$iface$bridge:link$num" ) || return new=$( set -- $new; echo $2 ) ngctl name "$iface$bridge:link$num" $jiface || return ifconfig $new name $jiface || return ifconfig $jiface mtu $mtu || return ifconfig $jiface up || return # # Set the MAC address of the new interface using a sensible # algorithm to prevent conflicts on the network. # jiface_devid= if [ "$clone_mac" ]; then jiface_devid=$( ifconfig $iface ether | awk '/ether/,$0=$2' ) elif [ ! "$no_derive" ]; then derive_mac $iface "$name" jiface_devid fi [ "$jiface_devid" ] && quietly ifconfig $jiface ether $jiface_devid jng_pin_jiface "$jiface" "$name" || : pin optional i=$(( $i + 1 )) done # for iface } jng_pin_usage="pin [-h] {-a | NAME ...}" jng_pin_descr="Pin eiface MACs into ng_bridge forwarding database (FDB)" jng_pin() { local OPTIND=1 OPTARG flag local show_all= err=$SUCCESS local name iface jiface while getopts ah flag; do case "$flag" in a) show_all=1 ;; *) action_usage pin # NOTREACHED esac done shift $(( $OPTIND - 1 )) if [ "$show_all" ]; then [ $# -eq 0 ] || action_usage pin "too many arguments" # NOTREACHED for iface in $( ifconfig -l ); do quietly ngctl info ${iface}bridge: || continue jng_bridge_persist_hosts ${iface}bridge || : persist optional done set -- $( jls -q name 2> /dev/null ) [ $# -gt 0 ] || action_usage pin "no jails" # NOTREACHED else [ $# -gt 0 ] || action_usage pin "too few arguments" # NOTREACHED fi mustberoot_to_continue for name in "$@"; do [ "${name:-x}" = "${name#*[![:print:]]}" ] || action_usage pin "invalid name: %s" "$name" # NOTREACHED for jiface in $( jexec "$name" ifconfig -l 2> /dev/null ) do case "$jiface" in ng[0-9]*) jng_pin_jiface "$jiface" "$name" || { echo "$pgm: pin $jiface: failed" >&2 err=$FAILURE } ;; esac done done return $err } jng_graph_usage="graph [-fh] [-T type] [-o output]" jng_graph_descr="Generate network graph (default output is 'jng.svg')" jng_graph() { local OPTIND=1 OPTARG flag local output=jng.svg output_type= force= while getopts fho:T: flag; do case "$flag" in f) force=1 ;; o) output="$OPTARG" ;; T) output_type="$OPTARG" ;; *) action_usage graph # NOTREACHED esac done shift $(( $OPTIND - 1 )) [ $# -eq 0 ] || action_usage graph "too many arguments" # NOTREACHED mustberoot_to_continue if [ -e "$output" -a ! "$force" ]; then echo "$output: Already exists (use '-f' to overwrite)" >&2 return $FAILURE fi if [ ! "$output_type" ]; then local valid suffix valid=$( dot -Txxx 2>&1 ) for suffix in ${valid##*:}; do [ "$output" != "${output%.$suffix}" ] || continue output_type=$suffix break done fi ngctl dot | dot ${output_type:+-T "$output_type"} -o "$output" } jng_show_usage="show [-h]" jng_show_descr="List possible NAME values for 'show NAME'" jng_show1_usage="show [-h] NAME ..." jng_show1_descr="Lists ng0_NAME [ng1_NAME ...]" jng_show2_usage="show [NAME ...]" jng_show2_descr="List NAME values or show interfaces associated with NAME." jng_show() { local OPTIND=1 OPTARG flag local name while getopts h flag; do case "$flag" in *) action_usage show2 # NOTREACHED esac done shift $(( $OPTIND - 1 )) mustberoot_to_continue if [ $# -eq 0 ]; then ngctl ls | awk '$4=="bridge",$0=$2' | xargs -rn1 -Ibridge ngctl show bridge: | awk 'sub(/^ng[[:digit:]]+_/, "", $2), $0 = $2' | sort -u return fi for name in "$@"; do ngctl ls | awk -v name="$name" ' match($2, /^ng[[:digit:]]+_/) && substr($2, RSTART + RLENGTH) == name && $4 == "eiface", $0 = $2 ' | sort done } jng_shutdown_usage="shutdown [-h] NAME ..." jng_shutdown_descr="Shutdown ng0_NAME [ng1_NAME ...]" jng_shutdown() { local OPTIND=1 OPTARG flag while getopts h flag; do case "$flag" in *) action_usage shutdown # NOTREACHED esac done shift $(( $OPTIND -1 )) [ $# -gt 0 ] || action_usage shutdown "too few arguments" # NOTREACHED mustberoot_to_continue local name for name in "$@"; do [ "${name:-x}" = "${name#*[![:print:]]}" ] || action_usage shutdown "invalid name: %s" "$name" # NOTREACHED jng_show "$name" | xargs -rn1 -I jiface ngctl shutdown jiface: done } jng_stats_usage="stats [-hj] {-a | NAME ...}" jng_stats_descr="Show ng_bridge link statistics for NAME interfaces" jng_stats() { local OPTIND=1 OPTARG flag local show_all= local name iface ether= while getopts ahj flag; do case "$flag" in a) show_all=1 ;; j) STATS_FMT=json export pgm : "${HOSTNAME:=$( hostname )}" export HOSTNAME ;; *) action_usage stats # NOTREACHED esac done shift $(( $OPTIND -1 )) if [ "$show_all" ]; then [ $# -eq 0 ] || action_usage stats "too many arguments" # NOTREACHED # Get a list of bridged ng_ether(4) devices for iface in $( ifconfig -l ); do quietly ngctl info ${iface}bridge: || continue ether="$ether $iface" done set -- $ether $( "$0" show ) [ $# -gt 0 ] || action_usage stats "no bridged interfaces" # NOTREACHED else [ $# -gt 0 ] || action_usage stats "too few arguments" # NOTREACHED fi mustberoot_to_continue local now="$( date +%s )" for name in "$@"; do [ "${name:-x}" = "${name#*[![:print:]]}" ] || action_usage stats "invalid name: %s" "$name" # NOTREACHED if ifconfig -l | xargs -n1 2> /dev/null | fgrep -qw "$name" then [ "$STATS_FMT" != "text" ] || echo "${name}bridge:uplink1 [lower]" ngctl msg ${name}bridge: getstats -1 | fmt_stats -n "${name}.lower" -t "$now" [ "$STATS_FMT" != "text" ] || echo "${name}bridge:link0 [upper]" ngctl msg ${name}bridge: getstats 0 | fmt_stats -n "${name}.upper" -t "$now" fi local jiface for jiface in $( jng_show "$name" ); do [ "$STATS_FMT" != "text" ] || echo "$jiface:" ngctl show $jiface: | awk ' $3 == "bridge" && $5 ~ /^link/ { bridge = $2 link = substr($5, 5) system(sprintf("ngctl msg %s: getstats %u", bridge, link)) }' | fmt_stats -n "$jiface" -t "$now" done done } fmt_stats() { local OPTIND=1 OPTARG flag local time= while getopts n:t: flag; do case "$flag" in n) name="$OPTARG" ;; t) time="$OPTARG" ;; *) break esac done shift $(( OPTIND - 1 )) fmt 2 | awk -v fmt="$STATS_FMT" -v name="$name" -v tm="$time" ' function json_add_str(pre, k, s) { return sprintf("%s,\"%s\":\"%s\"", pre, k, s) } function json_add_int(pre, k, i) { return sprintf("%s,\"%s\":%d", pre, k, i) } BEGIN { if (fmt == "json") { if (tm == "") srand() # Time-seed js = json_add_int(js, "epoch", tm != "" ? tm : srand()) js = json_add_str(js, "hostname", ENVIRON["HOSTNAME"]) js = json_add_str(js, "program", ENVIRON["pgm"]) js = json_add_str(js, "name", name) } } /=/ && fl = index($0, "=") { key = substr($0, 0, fl-1) val = substr($0, fl+1) if (fmt == "json") { js = json_add_int(js, key, val) } else { # Multi-line text printf "%20s = %s\n", key, val } } END { if (fmt == "json") { print "{" substr(js, 2) "}" } } ' # END-QUOTE } ############################################################ MAIN # # Command-line arguments # [ $# -gt 0 ] || usage "too few arguments" # NOTREACHED action="$1" [ "$action" ] || usage # NOTREACHED # # Validate action argument # case "$action" in -h) usage ;; # NOTREACHED -v) VERSION="${VERSION#*: }" echo "${VERSION% $}" exit $SUCCESS ;; -*) usage "unknown option: %s" "$action" ;; # NOTREACHED *[!a-zA-Z0-9_-]*) usage 'invalid action "%s"' "$action" ;; # NOTREACHED esac if [ "$BASH_VERSION" ]; then type="$( type -t "jng_$action" )" else type="$( type "jng_$action" 2> /dev/null )" fi || usage 'unknown action "%s"' "$action" # NOTREACHED case "$type" in *function) shift 1 # action eval "jng_$action" \"\$@\" ;; *) usage 'unknown action "%s"' "$action" # NOTREACHED esac ################################################################################ # END ################################################################################