Enhance upper-tester to deploy it as a service

#!/bin/bash

#

# Run one test campaign on the bench and bring everything back here, from the development machine or from the ATS

# host itself (the ATS steps then run locally, and nothing is copied):

#   1. optionally update and rebuild the ATS on the ATS host (--sync, --build);

#   2. optionally deploy the upper tester to the Raspberry Pi (--deploy-ut);

#   3. restart the upper tester service on the Pi;

#   4. start a capture of N2 and N3 (SCTP and GTP-U) on the ATS host;

#   5. run the test case(s) selected in the [EXECUTE] section of the ATS configuration (scripts/run_all.bash);

#   6. stop the capture, and copy back the TITAN logs, the capture and the upper tester journal of the run;

#   7. print the verdicts.

# Logs, capture and journal of the same run end up side by side: logs/$ATS/ and captures/<NAME>.pcap.

#

# Configuration (environment variables, defaults for the current lab):

#   ATS          ATS to run                               (default: AtsNGAP)

#   ATS_HOST     ssh destination of the ATS host           (default: ubuntu@kpournaras-test5g)

#   ATS_DIR      repository on the ATS host                (default: dev/nas.r17, relative to the home directory)

#   PI_HOST      ssh destination of the Raspberry Pi       (required for --deploy-ut and the UT steps)

#   PI_USER      user on the Raspberry Pi                  (default: ue)

#   UT_SERVICE   systemd unit of the upper tester          (default: upper-tester)

#   UT_DIR       install directory of the upper tester     (default: /home/ue/etsi/upper_tester)

#   CAPTURE_IF   capture interface on the ATS host         (default: any)

#   CAPTURE_FILTER  capture filter                         (default: "sctp or udp port 2152")

#   DRY_RUN=1    print the commands instead of running them

#

# Usage: scripts/bench_run.sh [--sync] [--build] [--deploy-ut] [--no-ut] [--name NAME]

#   --name NAME   base name of the capture (default: the first active test case of the [EXECUTE] section)

#   --no-ut       do not touch the upper tester (no restart, no journal)

#

# Requirements, to avoid password prompts (the script still works with them, interactively):

#   - ssh keys: ssh-copy-id $PI_USER@$PI_HOST (and ATS_HOST when run from the development machine);

#   - on the Pi: sudo without password for the restart of the upper tester (upper_tester/deploy/sudoers-upper-tester),

#     and the Pi user in the systemd-journal group to read its journal (sudo usermod -aG systemd-journal $PI_USER);

#   - on the ATS host: sudo for tcpdump (asked for when the capture starts and stops).

set -u

ATS=${ATS:-AtsNGAP}

ATS_HOST=${ATS_HOST:-ubuntu@kpournaras-test5g}

ATS_DIR=${ATS_DIR:-dev/nas.r17}

PI_HOST=${PI_HOST:-}

PI_USER=${PI_USER:-}

UT_SERVICE=${UT_SERVICE:-upper-tester}

UT_DIR=${UT_DIR:-/home/ue/etsi/upper_tester}

CAPTURE_IF=${CAPTURE_IF:-any}

CAPTURE_FILTER=${CAPTURE_FILTER:-sctp or udp port 2152}

DRY_RUN=${DRY_RUN:-0}

SYNC=0

BUILD=0

DEPLOY_UT=0

USE_UT=1

NAME=""

usage() {

    sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'

}

while [[ $# -gt 0 ]]; do

    case "$1" in

        --sync)      SYNC=1; shift ;;

        --build)     BUILD=1; shift ;;

        --deploy-ut) DEPLOY_UT=1; shift ;;

        --no-ut)     USE_UT=0; shift ;;

        --name)      NAME="$2"; shift 2 ;;

        -h|--help)   usage; exit 0 ;;

        *)           echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;

    esac

done

TOP=$(cd "$(dirname "$0")/.." && pwd)

PI=${PI_USER:+$PI_USER@}$PI_HOST

# The ATS host is this machine (e.g. ubuntu@kpournaras-test5g run on kpournaras-test5g): run its steps locally, in

# this repository, and copy nothing (scp of a file onto itself can truncate it)

ATS_HOST_NAME=${ATS_HOST#*@}

if [[ "$ATS_HOST_NAME" == "$(hostname)" || "$ATS_HOST_NAME" == "$(hostname -f 2>/dev/null)" || "$ATS_HOST_NAME" == localhost ]]; then

    ATS_LOCAL=1

    ATS_DIR=$TOP

else

    ATS_LOCAL=0

fi

CAPTURE_TMP=/tmp/bench_run_${ATS}.pcap

PID_TMP=/tmp/bench_run_${ATS}.tcpdump.pid

# Run a command, or print it with DRY_RUN=1

run() {

    if [[ "$DRY_RUN" == "1" ]]; then

        printf '+ %q' "$1"; shift; printf ' %q' "$@"; printf '\n'

    else

        "$@"

    fi

}

step() { echo; echo "=== $*"; }

# Run a shell command on the ATS host: locally, or through ssh (-t for an interactive command, e.g. sudo)

ats() {

    local tty=""

    if [[ "$1" == "-t" ]]; then tty="-t"; shift; fi

    if [[ $ATS_LOCAL == 1 ]]; then

        run bash -c "$1"

    else

        run ssh $tty "$ATS_HOST" "$1"

    fi

}

if [[ $USE_UT == 1 || $DEPLOY_UT == 1 ]] && [[ -z "$PI_HOST" ]] && [[ -z "$PI_USER" ]]; then

    echo "PI_HOST or PI_USER is not set (ssh destination of the Raspberry Pi); use --no-ut to run without the upper tester" >&2

    exit 2

fi

if [[ -z "$NAME" ]]; then

    # First active ("Module.TC_NAME" not commented out) test case of the [EXECUTE] section of the local cfg copy

    NAME=$(awk '/^\[EXECUTE\]/{e=1; next} /^\[/{e=0} e && /^[[:space:]]*[A-Za-z]/{sub(/.*\./, "", $1); print $1; exit}' \

           "$TOP/etc/$ATS/$ATS.cfg" 2>/dev/null)

    NAME=${NAME:-bench_run}

fi

echo "ATS $ATS on $( [[ $ATS_LOCAL == 1 ]] && echo "this host" || echo "$ATS_HOST"):$ATS_DIR, capture captures/$NAME.pcap${PI_HOST:+, upper tester on $PI}"

if [[ $SYNC == 1 ]]; then

    step "Updating the ATS on $ATS_HOST"

    ats "cd $ATS_DIR && git pull --ff-only" || exit 1

fi

if [[ $BUILD == 1 ]]; then

    step "Building $ATS on $ATS_HOST (make -j2: more jobs run out of memory)"

    ats "cd $ATS_DIR && make ATS=$ATS -j2" || exit 1

fi

if [[ $DEPLOY_UT == 1 ]]; then

    step "Deploying the upper tester to $PI:$UT_DIR"

    run rsync -a --delete --exclude __pycache__ --exclude docs "$TOP/upper_tester/" "$PI:$UT_DIR/" || exit 1

fi

if [[ $USE_UT == 1 ]]; then

    step "Restarting the upper tester ($UT_SERVICE) on $PI"

    # -t: sudo can ask for its password, unless upper_tester/deploy/sudoers-upper-tester is installed

    run ssh -t "$PI" "sudo systemctl restart $UT_SERVICE && sleep 2 && systemctl is-active $UT_SERVICE" || exit 1

fi

START=$(date -u '+%Y-%m-%d %H:%M:%S')

step "Starting the capture on $ATS_HOST ($CAPTURE_IF: $CAPTURE_FILTER)"

# sudo -v first, interactively, so that the background tcpdump does not wait for a password

ats -t "sudo -v && (sudo nohup tcpdump -i $CAPTURE_IF -w $CAPTURE_TMP '$CAPTURE_FILTER' >/dev/null 2>&1 & echo \$! > $PID_TMP)" || exit 1

[[ "$DRY_RUN" == "1" ]] || sleep 2

step "Running $ATS on $ATS_HOST"

ats -t "cd $ATS_DIR/scripts && ATS=$ATS ./run_all.bash"

step "Stopping the capture"

ats -t "sudo kill -INT \$(cat $PID_TMP); sleep 1; sudo chmod a+r $CAPTURE_TMP"

mkdir -p "$TOP/logs/$ATS" "$TOP/captures"

if [[ $ATS_LOCAL == 1 ]]; then

    step "Keeping the capture (the logs are already in logs/$ATS)"

    run cp "$CAPTURE_TMP" "$TOP/captures/$NAME.pcap"

else

    step "Copying back the logs and the capture"

    run scp -q "$ATS_HOST:$ATS_DIR/logs/$ATS/*.log" "$TOP/logs/$ATS/"

    run scp -q "$ATS_HOST:$CAPTURE_TMP" "$TOP/captures/$NAME.pcap"

fi

if [[ $USE_UT == 1 ]]; then

    step "Copying back the upper tester journal since $START (UTC)"

    # Without sudo: the Pi user must be in the systemd-journal group (otherwise journalctl shows nothing of the unit)

    JOURNAL_CMD="journalctl -u $UT_SERVICE --since '$START UTC' --no-pager -o short-iso"

    if [[ "$DRY_RUN" == "1" ]]; then

        run ssh "$PI" "$JOURNAL_CMD"

    elif ssh "$PI" "$JOURNAL_CMD" > "$TOP/logs/$ATS/upper_tester.log" && grep -q "ut\." "$TOP/logs/$ATS/upper_tester.log"; then

        echo "logs/$ATS/upper_tester.log: $(wc -l < "$TOP/logs/$ATS/upper_tester.log") lines"

    else

        echo "No upper tester journal: add $PI_USER to the systemd-journal group on the Pi (sudo usermod -aG systemd-journal $PI_USER)"

    fi

fi

step "Verdicts"

if [[ "$DRY_RUN" == "1" ]]; then

    echo "(dry run)"

else

    grep -h "finished. Verdict" "$TOP/logs/$ATS/merged.log" 2>/dev/null | sed -E 's/^.*Test case/Test case/' || echo "No verdict found in logs/$ATS/merged.log"

fi

# Lets the Pi user restart the upper tester without a password (scripts/bench_run.sh).

# Install: sudo install -m 0440 sudoers-upper-tester /etc/sudoers.d/upper-tester && sudo visudo -c

# Adapt the user name (ue) and the unit name (upper-tester) if needed.

ue ALL=(root) NOPASSWD: /usr/bin/systemctl restart upper-tester, /usr/bin/systemctl stop upper-tester, /usr/bin/systemctl start upper-tester

# Upper tester as a systemd service, for the lab Raspberry Pi: package in /home/ue/etsi/upper_tester, user ue.

# (upper-tester.service is the generic variant, installed in /opt/upper-tester.)

# Install:

#   sudo cp upper-tester-lab.service /etc/systemd/system/upper-tester.service

#   sudo systemctl daemon-reload && sudo systemctl enable --now upper-tester

#   systemctl status upper-tester ; journalctl -u upper-tester -f

[Unit]

Description=5G Upper Tester (AT bridge to the RM520N-GL)

After=network-online.target

Wants=network-online.target

[Service]

User=ue

# Access to /dev/ttyUSB* (the user must not need sudo for the modem)

Group=dialout

WorkingDirectory=/home/ue/etsi

ExecStart=/usr/bin/python3 -m upper_tester --config /home/ue/etsi/upper_tester/deploy/upper-tester-lab.toml

Restart=on-failure

RestartSec=3

[Install]

WantedBy=multi-user.target

# Upper tester configuration for the lab Raspberry Pi (upper-tester-lab.service); keys are the command-line options.

# The by-id path stays the same when the modem re-enumerates after AT+CFUN=1,1.

device = "/dev/serial/by-id/usb-Quectel_RM520N-GL_75bfe03e-if02-port0"

baud = 115200

host = "0.0.0.0"

port = 12100

allow = ["10.59.0.0/16"]   # the ATS host (kpournaras-test5g connects from 10.59.4.21)

log_level = "INFO"

profile = "rm520n-gl"