#!/bin/sh set -eu usage() { cat <<-'EOF' Usage: report -o DIRECTORY [-K] [-S STEP] report -h EOF } help() { cat <<-'EOF' Options: -o DIRECTORY the directory where to place the generated files -K keep intermediary files -S STEP which substep of the report to perform (default: top) -h, --help show this message Gather data from Git Notes, and generate an HTML report on CI runs. Two refs with notes are expected: 1. refs/notes/ci-data: contains metadata abount the CI runs, with timestamps, filenames and exit status; 2. refs/notes/ci-logs: contains the content of the log. When reconstructing the CI run, the $FILENAME present in the refs/notes/ci-data ref names the file, and its content comes from refs/notes/ci-logs. On a CI run that generated the numbers from 1 to 10, for a file named 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the expected output on the target directory "public" is: $ tree public/ public/ index.html data/ my-ci-run-2020-01-01-deadbeef.log ... logs/ my-ci-run-2020-01-01-deadbeef.log ... $ cat public/data/my-ci-run-2020-01-01-deadbeef.log 0 deadbeef my-ci-run-2020-01-01-deadbeef.log $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log 1 2 3 4 5 6 7 8 9 10 The generated 'index.html' is a webpage with the list of all known CI runs, their status, a link to the commit and a link to the log file. To enable fetching these refs by default, do so in the git config: $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*' Examples: Generate the report on the 'www' directory: $ report -o www EOF } for flag in "$@"; do case "$flag" in --) break ;; --help) usage help exit ;; *) ;; esac done KEEP_FILES=false STEP=top while getopts 'o:KS:h' flag; do case "$flag" in o) OUTDIR="$OPTARG" ;; K) KEEP_FILES=true ;; S) STEP="$OPTARG" ;; h) usage help exit ;; *) exit 2 ;; esac done shift $((OPTIND - 1)) if [ -z "${OUTDIR:-}" ]; then printf 'Missing -o OUTDIR.\n\n' >&2 usage >&2 exit 2 fi if [ -r src/infrastructure/config/conf.env ]; then CONF=src/infrastructure/config/conf.env else CONF=/etc/conf.env fi # shellcheck source=/dev/null . "$CONF" escape_html() { sed \ -e 's|&|\&|g' \ -e 's|<|\<|g' \ -e 's|>|\>|g' \ -e 's|"|\"|g' \ -e "s|'|\'|g" } emit_stage0_make() { git notes list | awk -v OUT="$OUTDIR" '{ printf "all: %s/stage0/%s\n\n", OUT, $2 printf "%s/stage0/%s:\n", OUT, $2 printf "\tgit notes --ref=refs/notes/ci-data show %s > $@\n\n", $2 }' printf 'all:\n' printf '\tprintf "%%s\\n" "$?" | tr " " "\\n"\n' } emit_stage1_make() { awk -F/ -vCMD="'$0'" -vOUT="$OUTDIR" '{ printf "all: %s/stage1/%s.mk\n\n", OUT, $(NF) printf "%s/stage1/%s.mk:\n", OUT, $(NF) printf "\t%s -o \"%s\" -S stage1-exec < %s/stage0/%s > $@\n", CMD, OUT, OUT, $(NF) }' printf 'all:\n' printf '\tcat -- "%s"/stage1/*.mk\n' "$OUTDIR" } stage1_exec() { awk -vOUT="$OUTDIR" -vCMD="'$0'" ' { d[$1] = $2 } END { escaped = d["filename"] gsub(/:/, "\\:", escaped) gsub(/"/, "", d["duration"]) printf "all: %s/data/%s\n\n", OUT, escaped printf "%s/data/%s:\n", OUT, escaped printf "\tln -- %s/stage0/%s $@\n\n\n", OUT, d["sha"] printf "all: %s/logs/%s\n\n", OUT, escaped printf "%s/logs/%s:\n", OUT, escaped printf "\tgit notes --ref=refs/notes/ci-logs show %s > $@\n\n\n", d["sha"] printf "all: %s/msgs/%s\n\n", OUT, escaped printf "%s/msgs/%s:\n", OUT, escaped printf "\tif ! git show %s 1>/dev/null 2>&1; then \\\n", d["sha"] printf "\t\tgit fetch origin %s; \\\n", d["sha"] printf "\tfi\n" printf "\tgit log -1 --format=%%B %s > $@\n\n\n", d["sha"] printf "all: %s/html/%s\n\n", OUT, escaped printf "%s/html/%s: %s/msgs/%s\n", OUT, escaped, OUT, escaped printf "\t%s -o \"%s\" -S html-item \"%s\" \"%s\" \"%s\" \"%s\" \"%s/msgs/%s\" \"%s\" > $@\n\n\n", CMD, OUT, d["status"], d["sha"], d["filename"], d["duration"], OUT, d["filename"], CGIT_URL } ' } emit_html_item() { STATUS="$1" SHA="$2" FILENAME="$3" DURATION="$4" MESSAGE_F="$5" CGIT_URL="$6" PASS='✅' # ✅ WARN='🐌' # 🐌 FAIL='❌' # ❌ if [ "$STATUS" = 0 ]; then if [ "$DURATION" -le 60 ]; then STATUS_MARKER="$PASS" else STATUS_MARKER="$WARN" fi else STATUS_MARKER="$FAIL" fi cat <<-EOF 
  • #
     $STATUS_MARKER - 
    ${DURATION}s
    (commit)
    $FILENAME
     
    $(escape_html < "$MESSAGE_F")
    • EOF } emit_html_index() { cat <<-EOF $NAME - CI logs

    CI logs for $NAME

      EOF echo "$OUTDIR"/html/* | tr ' ' '\n' | LANG=C.UTF-8 sort -r | xargs cat -- cat <<-'EOF'
    EOF } case "$STEP" in top) for d in stages stage0 stage1 data logs html msgs; do mkdir -p -- "$OUTDIR/$d" done "$0" -o "$OUTDIR" -S stage0-make | tee -- "$OUTDIR"/stages/stage0.mk | make -sf- | "$0" -o "$OUTDIR" -S stage1-make | tee -- "$OUTDIR"/stages/stage1.mk | make -sf- | tee -- "$OUTDIR"/stages/stage2.mk | make -sf- "$0" -o "$OUTDIR" -S html if [ "$KEEP_FILES" = false ]; then for d in stages stage0 stage1 html msgs; do rm -rf "${OUTDIR:?}/$d" done fi ;; stage0-make) emit_stage0_make ;; stage1-make) emit_stage1_make ;; stage1-exec) stage1_exec ;; html-item) emit_html_item "$@" ;; html) emit_html_index > "$OUTDIR"/index.html ;; *) printf 'Unsupported STEP type: "%s"\n\n' "$STEP" >&2 usage >&2 exit 2 ;; esac