From 11f3daea8627effd0127b28d2d46fb53af49a7fc Mon Sep 17 00:00:00 2001 From: EuAndreh Date: Thu, 30 Mar 2023 14:22:11 -0300 Subject: src/infrastructure/scripts/report.sh: v2 Now instead of working through each commit searially, we build a graph of steps and work on that by emitting make(1) code to be executed. May this code prove to be diffucult to modify or extend, this is a valid commit "checkpoint" to revert to in order to recover the *exact* same behaviour, but simpler code. Simple benchmarks suggest that: - on a single core, the scrit is ~15% slower; - with 2 cores it is as fast as the original; - as more cores are added, the faster it gets, up to $(nproc) when using more cores doesn't change anything. That means that it is a useful optimization for our desktops, but a hinderance for our current server, which only has a single core. --- src/infrastructure/scripts/report.sh | 209 +++++++++++++++++++++++++---------- 1 file changed, 152 insertions(+), 57 deletions(-) (limited to 'src') diff --git a/src/infrastructure/scripts/report.sh b/src/infrastructure/scripts/report.sh index c766d41..0b63ffc 100755 --- a/src/infrastructure/scripts/report.sh +++ b/src/infrastructure/scripts/report.sh @@ -4,7 +4,7 @@ set -eu usage() { cat <<-'EOF' Usage: - report [-C REPO] -o DIRECTORY + report -o DIRECTORY [-K] [-S STEP] report -h EOF } @@ -14,8 +14,9 @@ help() { Options: - -C REPO change to REPO when doing Git operations (default: $PWD) -o DIRECTORY the directory where to place the generated files + -K keep intermediary files + -S STEP which substep of the report to perform (default: top) -h, --help show this message @@ -92,15 +93,19 @@ for flag in "$@"; do esac done -REPO="$PWD" -while getopts 'C:o:h' flag; do +KEEP_FILES=false +STEP=top +while getopts 'o:KS:h' flag; do case "$flag" in - C) - REPO="$OPTARG" - ;; o) OUTDIR="$OPTARG" ;; + K) + KEEP_FILES=true + ;; + S) + STEP="$OPTARG" + ;; h) usage help @@ -119,6 +124,7 @@ if [ -z "${OUTDIR:-}" ]; then exit 2 fi + if [ -r src/infrastructure/config/conf.env ]; then CONF=src/infrastructure/config/conf.env else @@ -129,7 +135,7 @@ fi . "$CONF" -esc() { +escape_html() { sed \ -e 's|&|\&|g' \ -e 's|<|\<|g' \ @@ -138,18 +144,93 @@ esc() { -e "s|'|\'|g" } -mkdir -p "$OUTDIR" -cd "$OUTDIR" -mkdir -p logs data -for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do - git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp - FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)" - mv data/FILENAME-tmp data/"$FILENAME" - git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME" -done -{ +emit_stage0_make() { + git notes list | awk -v OUT="$OUTDIR" '{ + printf "all: %s/stage0/%s\n\n", OUT, $2 + printf "%s/stage0/%s:\n", OUT, $2 + printf "\tgit notes --ref=refs/notes/ci-data show %s > $@\n\n", $2 + }' + printf 'all:\n' + printf '\tprintf "%%s\\n" "$?" | tr " " "\\n"\n' +} + +emit_stage1_make() { + awk -F/ -vCMD="'$0'" -vOUT="$OUTDIR" '{ + printf "all: %s/stage1/%s.mk\n\n", OUT, $(NF) + printf "%s/stage1/%s.mk:\n", OUT, $(NF) + printf "\t%s -o \"%s\" -S stage1-exec < %s/stage0/%s > $@\n", CMD, OUT, OUT, $(NF) + }' + printf 'all:\n' + printf '\tcat -- "%s"/stage1/*.mk\n' "$OUTDIR" +} + +stage1_exec() { + awk -vOUT="$OUTDIR" -vCMD="'$0'" ' + { d[$1] = $2 } + END { + escaped = d["filename"] + gsub(/:/, "\\:", escaped) + + gsub(/"/, "", d["duration"]) + + printf "all: %s/data/%s\n\n", OUT, escaped + printf "%s/data/%s:\n", OUT, escaped + printf "\tln -- %s/stage0/%s $@\n\n\n", OUT, d["sha"] + + printf "all: %s/logs/%s\n\n", OUT, escaped + printf "%s/logs/%s:\n", OUT, escaped + printf "\tgit notes --ref=refs/notes/ci-logs show %s > $@\n\n\n", d["sha"] + + printf "all: %s/msgs/%s\n\n", OUT, escaped + printf "%s/msgs/%s:\n", OUT, escaped + printf "\tif ! git show %s 1>/dev/null 2>&1; then \\\n", d["sha"] + printf "\t\tgit fetch origin %s; \\\n", d["sha"] + printf "\tfi\n" + printf "\tgit log -1 --format=%%B %s > $@\n\n\n", d["sha"] + + printf "all: %s/html/%s\n\n", OUT, escaped + printf "%s/html/%s: %s/msgs/%s\n", OUT, escaped, OUT, escaped + printf "\t%s -o \"%s\" -S html-item \"%s\" \"%s\" \"%s\" \"%s\" \"%s/msgs/%s\" \"%s\" > $@\n\n\n", + CMD, OUT, d["status"], d["sha"], d["filename"], d["duration"], OUT, d["filename"], CGIT_URL + } + ' +} + +emit_html_item() { + STATUS="$1" + SHA="$2" + FILENAME="$3" + DURATION="$4" + MESSAGE_F="$5" + CGIT_URL="$6" + + PASS='✅' # ✅ + WARN='🐌' # 🐌 + FAIL='❌' # ❌ + if [ "$STATUS" = 0 ]; then + if [ "$DURATION" -le 60 ]; then + STATUS_MARKER="$PASS" + else + STATUS_MARKER="$WARN" + fi + else + STATUS_MARKER="$FAIL" + fi + cat <<-EOF +
  • + 
    #
     + $STATUS_MARKER - 
    ${DURATION}s
    + 
    (commit)
    + 
    $FILENAME
     +
    + 
    $(escape_html < "$MESSAGE_F")
     +
    • + EOF +} + +emit_html_index() { cat <<-EOF @@ -212,48 +293,62 @@ done
      EOF + echo "$OUTDIR"/html/* | + tr ' ' '\n' | + LANG=C.UTF-8 sort -r | + xargs cat -- - PASS='✅' # ✅ - WARN='🐌' # 🐌 - FAIL='❌' # ❌ - find data/ -type f | LANG=C.UTF-8 sort -r | while read -r f; do - STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)" - SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)" - FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)" - DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)" - MESSAGE="$({ - git -C "$REPO" log -1 --format=%B "$SHA" || { - git fetch origin "$SHA" - git -C "$REPO" log -1 --format=%B "$SHA" - } - } | esc)" - - if [ "$STATUS" = 0 ]; then - if [ "$DURATION" -le 60 ]; then - STATUS_MARKER="$PASS" - else - STATUS_MARKER="$WARN" - fi - else - STATUS_MARKER="$FAIL" - fi - - cat <<-EOF -
    1. - 
      #
       - $STATUS_MARKER - 
      ${DURATION:-?}s
      - 
      (commit)
      - 
      $FILENAME
       -
      - 
      $MESSAGE
       -
      2. - EOF - done - - cat <<-EOF + cat <<-'EOF'
    EOF -} > index.html +} + + + +case "$STEP" in + top) + for d in stages stage0 stage1 data logs html msgs; do + mkdir -p -- "$OUTDIR/$d" + done + + "$0" -o "$OUTDIR" -S stage0-make | + tee -- "$OUTDIR"/stages/stage0.mk | + make -sf- | + "$0" -o "$OUTDIR" -S stage1-make | + tee -- "$OUTDIR"/stages/stage1.mk | + make -sf- | + tee -- "$OUTDIR"/stages/stage2.mk | + make -sf- + + "$0" -o "$OUTDIR" -S html + + if [ "$KEEP_FILES" = false ]; then + for d in stages stage0 stage1 html msgs; do + rm -rf "${OUTDIR:?}/$d" + done + fi + ;; + stage0-make) + emit_stage0_make + ;; + stage1-make) + emit_stage1_make + ;; + stage1-exec) + stage1_exec + ;; + html-item) + emit_html_item "$@" + ;; + html) + emit_html_index > "$OUTDIR"/index.html + ;; + *) + printf 'Unsupported STEP type: "%s"\n\n' "$STEP" >&2 + usage >&2 + exit 2 + ;; +esac -- cgit v1.2.3