From 1a3208d43f837768e18219ca4e79fe31bf748865 Mon Sep 17 00:00:00 2001 From: EuAndreh Date: Fri, 31 Mar 2023 20:12:17 -0300 Subject: Revamp CI: simpler variant of the same functionality --- Makefile | 2 + aux/ci/ci-build.sh | 65 ---------- aux/ci/git-post-receive.sh | 192 ++++++++++++++++++++++++++-- aux/ci/git-pre-push.sh | 22 ---- aux/ci/git-pre-receive.sh | 14 ++ aux/ci/report.sh | 297 ++++++++++++++++++++++++++++++++----------- aux/guix/manifest.scm | 26 ---- aux/guix/pinned-channels.scm | 11 -- aux/guix/with-container.sh | 17 --- aux/lib.sh | 2 + aux/tld.txt | 2 +- aux/workflow/l10n.sh | 2 +- manifest.scm | 25 ++++ 13 files changed, 445 insertions(+), 232 deletions(-) delete mode 100755 aux/ci/ci-build.sh delete mode 100755 aux/ci/git-pre-push.sh create mode 100755 aux/ci/git-pre-receive.sh delete mode 100644 aux/guix/manifest.scm delete mode 100644 aux/guix/pinned-channels.scm delete mode 100755 aux/guix/with-container.sh create mode 100644 manifest.scm diff --git a/Makefile b/Makefile index d038522..b127194 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,8 @@ public: l10n-gen TODOs.md $(manpages) public/makefile.svg -N '$(NAME_UC)' -n '$(NAME)' -m '$(MAILING_LIST)' -o public sh doc/manpages.sh -Ho public $(manpages) +dev: all check public + spellcheck: public sh aux/workflow/assert-spelling.sh -l '$(TRANSLATIONS) en' \ $$(find public -type f -name '*.html') diff --git a/aux/ci/ci-build.sh b/aux/ci/ci-build.sh deleted file mode 100755 index 2ec0102..0000000 --- a/aux/ci/ci-build.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh -set -eux - -PROJECT="$1" -LOGS_DIR="$2" -SHA="$3" -FILENAME="$(date -Is)-$SHA.log" -LOGFILE="$LOGS_DIR/$FILENAME" - -mkdtemp() { - name="$(echo 'mkstemp(template)' | - m4 -D template="${TMPDIR:-/tmp}/m4-tmpname.")" - rm -f "$name" - mkdir "$name" - echo "$name" -} - -{ - echo "Starting CI job at: $(date -Is)" - - finish() { - STATUS="$?" - printf "\n\n>>> exit status was %s\n" "$STATUS" - echo "Finishing CI job at: $(date -Is)" - cd - - NOTE=$(cat <>>\n>>> CI logs added as Git note.\n>>>\n>>> Run status was %s\n>>>\n\n' \ - "$STATUS" - } - trap finish EXIT - - unset GIT_DIR - REMOTE="$PWD" - cd "$(mkdtemp)" - git clone "$REMOTE" . - git config --global user.email git@euandre.org - git config --global user.name 'EuAndreh CI' - - if [ -f aux/guix/with-container.sh ]; then - RUNNER='sh aux/guix/with-container.sh' - else - RUNNER='sh -c' - fi - - $RUNNER 'make clean public dev-check' -} 2>&1 | tee "$LOGFILE" diff --git a/aux/ci/git-post-receive.sh b/aux/ci/git-post-receive.sh index 92bba73..c25649b 100755 --- a/aux/ci/git-post-receive.sh +++ b/aux/ci/git-post-receive.sh @@ -1,22 +1,186 @@ #!/bin/sh +# shellcheck source=/dev/null disable=2317 +. /etc/rc set -eu + +# shellcheck disable=2034 +read -r _oldrev SHA REFNAME + +if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then + exit +fi + + +SKIP_DEPLOY=false for n in $(seq 0 $((GIT_PUSH_OPTION_COUNT - 1))); do - opt="$(eval "echo \$GIT_PUSH_OPTION_$n")" - if [ "$opt" = skip-ci ] || [ "$opt" = ci-skip ]; then - printf "\n'%s' option detected, not running ci-build.sh\n\n" \ - "$opt" - exit 0 - fi + opt="$(eval "printf '%s' \"\$GIT_PUSH_OPTION_$n\"")" + case "$opt" in + ci.skip) + cat <<-EOF + + "$opt" option detected, not running CI. + + EOF + exit + ;; + deploy.skip) + SKIP_DEPLOY=true + ;; + *) + ;; + esac done -# shellcheck disable=2034 -read -r _oldrev SHA _refname -PROJECT="$(basename "$PWD" | cut -d. -f1)" # remove .git suffix -LOGS_DIR="/opt/ci/$PROJECT/logs" -sh "/opt/ci/$PROJECT/ci-build.sh" "$PROJECT" "$LOGS_DIR" "$SHA" ||: +epoch() { + awk 'BEGIN { srand(); print(srand()); }' +} + +now() { + date '+%Y-%m-%dT%H:%M:%S%:z' +} + +NAME="$(basename "$PWD" .git)" +LOGS_DIR=/var/log/ci/"$NAME"/ +HTML_OUTDIR="/srv/www/s/$NAME" +TIMESTAMP="$(now)" +FILENAME="$TIMESTAMP-$SHA.log" +LOGFILE="$LOGS_DIR/$FILENAME" +mkdir -p "$LOGS_DIR" + + +END_MARKER='\033[0m' +LIGHT_BLUE_B='\033[1;36m' +YELLOW='\033[1;33m' + +blue() { + printf "${LIGHT_BLUE_B}%s${END_MARKER}" "$1" +} + +yellow() { + printf "${YELLOW}%s${END_MARKER}" "$1" +} -echo 'To retrigger the build, run:' -echo "cd /srv/http/$PROJECT.git/" -echo "sh /opt/ci/$PROJECT/ci-build.sh" "$PROJECT" "$LOGS_DIR" "$SHA" +info() { + sed "s|^\(.\)|$(blue 'CI'): \1|" +} + + +uuid() { + od -xN20 /dev/urandom | + head -n1 | + awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}' +} + +tmpname() { + printf '%s/uuid-tmpname with spaces.%s' "${TMPDIR:-/tmp}" "$(uuid)" +} + +mkdtemp() { + name="$(tmpname)" + mkdir -- "$name" + printf '%s' "$name" +} + + +{ + cat <<-EOF | info + Starting CI job at: $(now) + EOF + START="$(epoch)" + + duration() { + if [ "$RUN_DURATION" -gt 60 ]; then + cat <<-EOF + $(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds) + EOF + else + cat <<-EOF + Run took $RUN_DURATION seconds. + EOF + fi + } + + finish() { + STATUS="$?" + END="$(epoch)" + RUN_DURATION=$((END - START)) + cat <<-EOF | info + Finishing CI job at: $(now) + Exit status was $STATUS + Re-run with: + \$ $CMD + $(duration) + EOF + + NOTE="$( + cat <<-EOF + See CI logs with: + git notes --ref=refs/notes/ci-logs show $SHA + git notes --ref=refs/notes/ci-data show $SHA + + Exit status: $STATUS + Duration: $RUN_DURATION + EOF + )" + git notes --ref=refs/notes/ci-data add -f -m "$( + cat <<-EOF + status $STATUS + sha $SHA + filename $FILENAME + duration $RUN_DURATION + timestamp $TIMESTAMP + to-prod $TO_PROD + refname $REFNAME + EOF + )" "$SHA" + git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA" + git notes add -f -m "$NOTE" "$SHA" + + { + printf 'Git CI HTML report for %s (%s) started.\n' "$NAME" "$SHA" >&2 + DIR="$(mkdtemp)" + REMOTE="$PWD" + cd "$DIR" + { + git clone "$REMOTE" . + git fetch origin 'refs/notes/*:refs/notes/*' + } 1>/dev/null 2>&1 + sh aux/ci/report.sh -n "$NAME" -o public-ci ||: + sudo -u deployer mkdir -p "$HTML_OUTDIR"/ci/ + sudo -u deployer rsync \ + --chmod=D775,F664 \ + --chown=deployer:deployer \ + --delete \ + -a \ + public-ci/ "$HTML_OUTDIR"/ci/ + rm -rf "$DIR" + printf 'Git CI HTML report for %s (%s) finished.\n' "$NAME" "$SHA" >&2 + } 2>&1 | logger -i -p local0.warn -t git-ci 1>/dev/null 2>&1 & + } + trap finish EXIT + + unset GIT_DIR + + if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then + cat <<-EOF | info + In branch "main", running deploy for $SHA. + EOF + TO_PROD=true + CMD="sudo cicd $NAME $SHA" + else + if [ "$SKIP_DEPLOY" = true ]; then + cat <<-EOF | info + "deploy.skip" option detected, skipping deploy for $SHA. + EOF + else + cat <<-EOF | info + Not on branch "main", skipping deploy for $SHA. + EOF + fi + TO_PROD=false + CMD="sudo cicd -n $NAME $SHA" + fi + $CMD +} 2>&1 | ts -s '%.s' | tee "$LOGFILE" diff --git a/aux/ci/git-pre-push.sh b/aux/ci/git-pre-push.sh deleted file mode 100755 index eaaa7bd..0000000 --- a/aux/ci/git-pre-push.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -set -eux - -TLD="$(cat aux/tld.txt)" -. aux/lib.sh - -PROJECT="$(basename "$PWD")" -LOGS_DIR="/opt/ci/$PROJECT/logs" -REMOTE_GIT_DIR="/srv/http/$PROJECT.git" - -DESCRIPTION="$(mkstemp)" -if [ -f description ] -then - cp description "$DESCRIPTION" -else - git config euandreh.description > "$DESCRIPTION" -fi - -scp "$DESCRIPTION" "$TLD:$REMOTE_GIT_DIR/description" -ssh "$TLD" mkdir -p "$LOGS_DIR" -scp aux/ci/ci-build.sh "$TLD:$(dirname "$LOGS_DIR")/ci-build.sh" -scp aux/ci/git-post-receive.sh "$TLD:$REMOTE_GIT_DIR/hooks/post-receive" diff --git a/aux/ci/git-pre-receive.sh b/aux/ci/git-pre-receive.sh new file mode 100755 index 0000000..199d06e --- /dev/null +++ b/aux/ci/git-pre-receive.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu + +read -r _oldrev SHA _refname +unset GIT_DIR + +if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then + exit +fi + +printf 'Upgrading post-receive hook...' >&2 +git show "$SHA":aux/ci/git-post-receive.sh > hooks/post-receive +chmod +x hooks/post-receive +printf 'done.\n' >&2 diff --git a/aux/ci/report.sh b/aux/ci/report.sh index 6f18f0f..5cffda1 100755 --- a/aux/ci/report.sh +++ b/aux/ci/report.sh @@ -1,17 +1,110 @@ #!/bin/sh set -eu -TLD="$(cat aux/tld.txt)" -. aux/lib.sh +usage() { + cat <<-'EOF' + Usage: + aux/ci/report.sh -n NAME -o OUTDIR + aux/ci/report.sh -h + EOF +} + +help() { + cat <<-'EOF' + + + Options: + -n NAME the name of the project + -o OUTDIR the directory where to place the generated files + -h, --help show this message + + + Gather data from Git Notes, and generate an HTML report on CI runs. + + Two refs with notes are expected: + 1. refs/notes/ci-data: contains metadata abount the CI runs, + with timestamps, filenames and exit status; + 2. refs/notes/ci-logs: contains the content of the log. + + When reconstructing the CI run, the $FILENAME present in + the refs/notes/ci-data ref names the file, and its content comes + from refs/notes/ci-logs. + + On a CI run that generated the numbers from 1 to 10, for a file named + 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the + expected output on the target directory "public" is: + + $ tree public/ + public/ + index.html + data/ + my-ci-run-2020-01-01-deadbeef.log + ... + logs/ + my-ci-run-2020-01-01-deadbeef.log + ... + + $ cat public/data/my-ci-run-2020-01-01-deadbeef.log + 0 deadbeef my-ci-run-2020-01-01-deadbeef.log + + $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + + The generated 'index.html' is a webpage with the list of all known + CI runs, their status, a link to the commit and a link to the + log file. + + To enable fetching these refs by default, do so in the git config: + + $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*' + + + Examples: + + Generate the report on the 'www' directory: + + $ report -o www + EOF +} + + +for flag in "$@"; do + case "$flag" in + --) + break + ;; + --help) + usage + help + exit + ;; + *) + ;; + esac +done -while getopts 'n:o:' flag; do +while getopts 'n:o:h' flag; do case "$flag" in n) - PROJECT="$OPTARG" + NAME="$OPTARG" ;; o) OUTDIR="$OPTARG" ;; + h) + usage + help + exit + ;; *) exit 2 ;; @@ -19,83 +112,137 @@ while getopts 'n:o:' flag; do done shift $((OPTIND - 1)) -assert_arg() { - if [ -z "$1" ]; then - echo "Missing $2" >&2 - exit 2 - fi -} +. aux/lib.sh -assert_arg "${PROJECT:-}" '-n PROJECT' -assert_arg "${OUTDIR:-}" '-o OUTDIR' +eval "$(assert_arg "${NAME:-}" '-n NAME')" +eval "$(assert_arg "${OUTDIR:-}" '-o OUTDIR')" -PASS='✅' -FAIL='❌' -mkdir -p "$OUTDIR/ci-logs" "$OUTDIR/ci-data" +esc() { + sed \ + -e 's|&|\&|g' \ + -e 's|<|\<|g' \ + -e 's|>|\>|g' \ + -e 's|"|\"|g' \ + -e "s|'|\'|g" +} -OUT="$(mkstemp)" -chmod 644 "$OUT" +mkdir -p "$OUTDIR" +cd "$OUTDIR" +mkdir -p logs data -for c in $(git notes list | cut -d\ -f2); do - DATA="$(git notes --ref=refs/notes/ci-data show "$c")" - FILENAME="$(echo "$DATA" | cut -d\ -f2)" - echo "$DATA" > "$OUTDIR/ci-data/$FILENAME" - git notes --ref=refs/notes/ci-logs show "$c" \ - > "$OUTDIR/ci-logs/$FILENAME" +for c in $(git notes list | cut -d' ' -f2); do + git notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp + FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)" + mv data/FILENAME-tmp data/"$FILENAME" + git notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME" done -cat <> "$OUT" - - - - - - - - $PROJECT - CI logs - - - - -
-

- CI logs for - $PROJECT -

-
    -EOF - -for f in $(find "$OUTDIR/ci-data/" -type f | LANG=C.UTF-8 sort -r); do - DATA="$(cat "$f")" - STATUS="$(echo "$DATA" | cut -d\ -f1)" - FILENAME="$(echo "$DATA" | cut -d\ -f2)" - - if [ "$STATUS" = 0 ]; then - STATUS_MARKER="$PASS" - else - STATUS_MARKER="$FAIL" - fi - - cat <> "$OUT" -
  1. - $STATUS_MARKER 
    $FILENAME
     -
    2. -EOF -done +{ + cat <<-EOF + + + + + + + $NAME - CI logs + + + +
    +

    + CI logs for + $NAME +

    +
      + EOF + + + PASS='✅' # ✅ + WARN='🐌' # 🐌 + FAIL='❌' # ❌ + find data/ -type f | LANG=C.UTF-8 sort -r | while read -r f; do + STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)" + SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)" + FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)" + DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)" + MESSAGE="$({ + git log -1 --format=%B "$SHA" || { + git fetch origin "$SHA" + git log -1 --format=%B "$SHA" + } + } | esc)" + + if [ "$STATUS" = 0 ]; then + if [ "$DURATION" -le 60 ]; then + STATUS_MARKER="$PASS" + else + STATUS_MARKER="$WARN" + fi + else + STATUS_MARKER="$FAIL" + fi -cat <> "$OUT" -
    -
    - - -EOF + cat <<-EOF +
  2. + 
    #
     + $STATUS_MARKER - 
    ${DURATION:-?}s
    + 
    (commit)
    + 
    $FILENAME
     + 
    (data)
    +
    + 
    $MESSAGE
     +
    3. + EOF + done -mv "$OUT" "$OUTDIR/ci.html" + cat <<-EOF +
+
+ + + EOF +} > index.html diff --git a/aux/guix/manifest.scm b/aux/guix/manifest.scm deleted file mode 100644 index d4da520..0000000 --- a/aux/guix/manifest.scm +++ /dev/null @@ -1,26 +0,0 @@ -(specifications->manifest - (map symbol->string - '(bash - coreutils - findutils - diffutils - grep - sed - m4 - git - gawk - make - makefile2graph - graphviz - shellcheck - pandoc - gettext - po4a-text - mdpo-patched - hunspell - hunspell-dict-en-utf8 - hunspell-dict-pt-utf8 - hunspell-dict-fr-utf8 - hunspell-dict-eo-utf8 - - posix-c99))) diff --git a/aux/guix/pinned-channels.scm b/aux/guix/pinned-channels.scm deleted file mode 100644 index 504406f..0000000 --- a/aux/guix/pinned-channels.scm +++ /dev/null @@ -1,11 +0,0 @@ -(cons* - (channel - (name 'xyz-euandreh) - (url "git://euandreh.xyz/package-repository") - (branch "main") - (introduction - (make-channel-introduction - "d749e053e6db365069cb9b2ef47a78b06f9e7361" - (openpgp-fingerprint - "5BDA E9B8 B2F6 C6BC BB0D 6CE5 81F9 0EC3 CD35 6060")))) - %default-channels) diff --git a/aux/guix/with-container.sh b/aux/guix/with-container.sh deleted file mode 100755 index 6e19691..0000000 --- a/aux/guix/with-container.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -set -eu - -if [ -z "${1:-}" ]; then - guix time-machine --fallback -C aux/guix/pinned-channels.scm -- \ - environment --fallback -m aux/guix/manifest.scm -elif [ "$1" = '-p' ]; then - guix time-machine --fallback -C aux/guix/pinned-channels.scm -- \ - environment --fallback -m aux/guix/manifest.scm --pure -C -elif [ "$1" = '-C' ]; then - guix time-machine --fallback -C aux/guix/pinned-channels.scm -- \ - pack -f docker -S/bin=bin -m aux/guix/manifest.scm -v3 -else - guix time-machine --fallback -C aux/guix/pinned-channels.scm -- \ - environment --fallback -m aux/guix/manifest.scm --pure -C \ - -- sh -c "$@" -fi diff --git a/aux/lib.sh b/aux/lib.sh index e345bd6..2276681 100644 --- a/aux/lib.sh +++ b/aux/lib.sh @@ -1,5 +1,7 @@ #!/bin/sh +TLD="$(cat aux/tld.txt)" + assert_arg() { if [ -z "$1" ]; then printf 'Missing %s.\n\n' "$2" >&2 diff --git a/aux/tld.txt b/aux/tld.txt index 0cb8b8b..fd7ea0f 100644 --- a/aux/tld.txt +++ b/aux/tld.txt @@ -1 +1 @@ -euandreh.xyz +euandre.org diff --git a/aux/workflow/l10n.sh b/aux/workflow/l10n.sh index cf687aa..1002adc 100755 --- a/aux/workflow/l10n.sh +++ b/aux/workflow/l10n.sh @@ -49,7 +49,7 @@ for from_f in "$@"; do md2po --include-codeblocks --quiet --save \ --po-filepath "$pofile" < "$from_f" po2md --pofiles "$pofile" --save "$to_f" \ - --quiet --wrapwidth 999 < "$from_f" + --quiet < "$from_f" ;; *) echo "Unsupported file format: $from_f" >&2 diff --git a/manifest.scm b/manifest.scm new file mode 100644 index 0000000..a537d79 --- /dev/null +++ b/manifest.scm @@ -0,0 +1,25 @@ +(specifications->manifest + (map + symbol->string + '(bash + coreutils + findutils + diffutils + grep + sed + git + gawk + make + makefile2graph + graphviz + pandoc + shellcheck + gettext + po4a + mdpo + hunspell + hunspell-dict-en-utf8 + hunspell-dict-pt-utf8 + hunspell-dict-fr-utf8 + hunspell-dict-eo-utf8 + hunspell-dict-es-utf8))) -- cgit v1.2.3