From 08588f9907299b1a927e281d5c65b46b7cefa427 Mon Sep 17 00:00:00 2001 From: EuAndreh Date: Tue, 4 Apr 2023 08:33:41 -0300 Subject: Revamp v2/ --- v2/aux/ci/git-post-receive.sh | 186 +++++++++++++++++++++++++++++++ v2/aux/ci/git-pre-receive.sh | 14 +++ v2/aux/ci/report.sh | 250 ++++++++++++++++++++++++++++++++++++++++++ v2/aux/domain.txt | 1 + v2/aux/lib.sh | 37 +++++++ v2/aux/po4a-cfg.sh | 109 ++++++++++++++++++ v2/aux/workflow/TODOs.sh | 1 - v2/aux/workflow/favicon.html | 1 - v2/aux/workflow/preamble.md | 1 - v2/aux/workflow/style.css | 1 - 10 files changed, 597 insertions(+), 4 deletions(-) create mode 100755 v2/aux/ci/git-post-receive.sh create mode 100755 v2/aux/ci/git-pre-receive.sh create mode 100755 v2/aux/ci/report.sh create mode 100644 v2/aux/domain.txt create mode 100644 v2/aux/lib.sh create mode 100755 v2/aux/po4a-cfg.sh delete mode 120000 v2/aux/workflow/TODOs.sh delete mode 100644 v2/aux/workflow/favicon.html delete mode 120000 v2/aux/workflow/preamble.md delete mode 120000 v2/aux/workflow/style.css (limited to 'v2/aux') diff --git a/v2/aux/ci/git-post-receive.sh b/v2/aux/ci/git-post-receive.sh new file mode 100755 index 0000000..76adccf --- /dev/null +++ b/v2/aux/ci/git-post-receive.sh @@ -0,0 +1,186 @@ +#!/bin/sh +# shellcheck source=/dev/null disable=2317 +. /etc/rc +set -eu + + +# shellcheck disable=2034 +read -r _oldrev SHA REFNAME + +if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then + exit +fi + + +SKIP_DEPLOY=false +for n in $(seq 0 $((GIT_PUSH_OPTION_COUNT - 1))); do + opt="$(eval "printf '%s' \"\$GIT_PUSH_OPTION_$n\"")" + case "$opt" in + ci.skip) + cat <<-EOF + + "$opt" option detected, not running CI. + + EOF + exit + ;; + deploy.skip) + SKIP_DEPLOY=true + ;; + *) + ;; + esac +done + + +epoch() { + awk 'BEGIN { srand(); print(srand()); }' +} + +now() { + date '+%Y-%m-%dT%H:%M:%S%:z' +} + +NAME="$(basename "$PWD" .git)" +LOGS_DIR=/var/log/ci/"$NAME"/ +HTML_OUTDIR="/srv/www/s/$NAME" +TIMESTAMP="$(now)" +FILENAME="$TIMESTAMP-$SHA.log" +LOGFILE="$LOGS_DIR/$FILENAME" +mkdir -p "$LOGS_DIR" + + +END_MARKER='\033[0m' +LIGHT_BLUE_B='\033[1;36m' +YELLOW='\033[1;33m' + +blue() { + printf "${LIGHT_BLUE_B}%s${END_MARKER}" "$1" +} + +yellow() { + printf "${YELLOW}%s${END_MARKER}" "$1" +} + +info() { + sed "s|^\(.\)|$(blue 'CI'): \1|" +} + + +uuid() { + od -xN20 /dev/urandom | + head -n1 | + awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}' +} + +tmpname() { + printf '%s/uuid-tmpname with spaces.%s' "${TMPDIR:-/tmp}" "$(uuid)" +} + +mkdtemp() { + name="$(tmpname)" + mkdir -- "$name" + printf '%s' "$name" +} + + +{ + cat <<-EOF | info + Starting CI job at: $(now) + EOF + START="$(epoch)" + + duration() { + if [ "$RUN_DURATION" -gt 60 ]; then + cat <<-EOF + $(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds) + EOF + else + cat <<-EOF + Run took $RUN_DURATION seconds. + EOF + fi + } + + finish() { + STATUS="$?" + END="$(epoch)" + RUN_DURATION=$((END - START)) + cat <<-EOF | info + Finishing CI job at: $(now) + Exit status was $STATUS + Re-run with: + \$ $CMD + $(duration) + EOF + + NOTE="$( + cat <<-EOF + See CI logs with: + git notes --ref=refs/notes/ci-logs show $SHA + git notes --ref=refs/notes/ci-data show $SHA + + Exit status: $STATUS + Duration: $RUN_DURATION + EOF + )" + git notes --ref=refs/notes/ci-data add -f -m "$( + cat <<-EOF + status $STATUS + sha $SHA + filename $FILENAME + duration $RUN_DURATION + timestamp $TIMESTAMP + to-prod $TO_PROD + refname $REFNAME + EOF + )" "$SHA" + git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA" + git notes add -f -m "$NOTE" "$SHA" + + { + printf 'Git CI HTML report for %s (%s) started.\n' "$NAME" "$SHA" >&2 + DIR="$(mkdtemp)" + REMOTE="$PWD" + cd "$DIR" + { + git clone "$REMOTE" . + git fetch origin 'refs/notes/*:refs/notes/*' + } 1>/dev/null 2>&1 + sh aux/ci/report.sh -n "$NAME" -o public-ci + sudo -u deployer mkdir -p "$HTML_OUTDIR"/ci/ + sudo -u deployer rsync \ + --chmod=D775,F664 \ + --chown=deployer:deployer \ + --delete \ + -a \ + public-ci/ "$HTML_OUTDIR"/ci/ + rm -rf "$DIR" + printf 'Git CI HTML report for %s (%s) finished.\n' "$NAME" "$SHA" >&2 + } 2>&1 | logger -i -p local0.warn -t git-ci 1>/dev/null 2>&1 & + } + trap finish EXIT + + unset GIT_DIR + + if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then + cat <<-EOF | info + In branch "main", running deploy for $SHA. + EOF + TO_PROD=true + CMD="sudo cicd $NAME $SHA" + else + if [ "$SKIP_DEPLOY" = true ]; then + cat <<-EOF | info + "deploy.skip" option detected, skipping deploy for $SHA. + EOF + else + cat <<-EOF | info + Not on branch "main", skipping deploy for $SHA. + EOF + fi + TO_PROD=false + CMD="sudo cicd -n $NAME $SHA" + fi + $CMD +} 2>&1 | ts -s '%.s' | tee "$LOGFILE" diff --git a/v2/aux/ci/git-pre-receive.sh b/v2/aux/ci/git-pre-receive.sh new file mode 100755 index 0000000..199d06e --- /dev/null +++ b/v2/aux/ci/git-pre-receive.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu + +read -r _oldrev SHA _refname +unset GIT_DIR + +if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then + exit +fi + +printf 'Upgrading post-receive hook...' >&2 +git show "$SHA":aux/ci/git-post-receive.sh > hooks/post-receive +chmod +x hooks/post-receive +printf 'done.\n' >&2 diff --git a/v2/aux/ci/report.sh b/v2/aux/ci/report.sh new file mode 100755 index 0000000..0a0a0ae --- /dev/null +++ b/v2/aux/ci/report.sh @@ -0,0 +1,250 @@ +#!/bin/sh +set -eu + +usage() { + cat <<-'EOF' + Usage: + aux/ci/report.sh -o OUTDIR + aux/ci/report.sh -h + EOF +} + +help() { + cat <<-'EOF' + + + Options: + -o OUTDIR the directory where to place the generated files + -h, --help show this message + + + Gather data from Git Notes, and generate an HTML report on CI runs. + + Two refs with notes are expected: + 1. refs/notes/ci-data: contains metadata abount the CI runs, + with timestamps, filenames and exit status; + 2. refs/notes/ci-logs: contains the content of the log. + + When reconstructing the CI run, the $FILENAME present in + the refs/notes/ci-data ref names the file, and its content comes + from refs/notes/ci-logs. + + On a CI run that generated the numbers from 1 to 10, for a file named + 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, ran for + 15 seconds and was deployed to production, the expected output on the + target directory "public" is: + + $ tree public/ + public/ + index.html + data/ + 2020-01-01T01:00:00-deadbeef.log + ... + logs/ + 2020-01-01T01:00:00-deadbeef.log + ... + + $ cat public/data/2020-01-01T01:00:00-deadbeef.log + status 0 + sha deadbeef + filename deadbeef 2020-01-01T01:00:00-deadbeef.log + duration 15 + timestamp 2020-01-01T01:00:00 + to-prod true + refname refs/heads/main + + $ cat public/logs/2020-01-01T01:00:00-deadbeef.log + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + + The generated 'index.html' is a webpage with the list of all known + CI runs, their status, a link to the commit and a link to the + log and data files. + + To enable fetching these refs by default, do so in the git config: + + $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*' + + + Examples: + + Generate the report on the 'www' directory: + + $ sh aux/ci/report.sh -o www + EOF +} + + +for flag in "$@"; do + case "$flag" in + --) + break + ;; + --help) + usage + help + exit + ;; + *) + ;; + esac +done + +while getopts 'o:h' flag; do + case "$flag" in + o) + OUTDIR="$OPTARG" + ;; + h) + usage + help + exit + ;; + *) + exit 2 + ;; + esac +done +shift $((OPTIND - 1)) + +. aux/lib.sh + +eval "$(assert_arg "${OUTDIR:-}" '-o OUTDIR')" + + +esc() { + sed \ + -e 's|&|\&|g' \ + -e 's|<|\<|g' \ + -e 's|>|\>|g' \ + -e 's|"|\"|g' \ + -e "s|'|\'|g" +} + +mkdir -p "$OUTDIR" +cd "$OUTDIR" +mkdir -p logs data + +for c in $(git notes list | cut -d' ' -f2); do + git notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp + FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)" + mv data/FILENAME-tmp data/"$FILENAME" + git notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME" +done + +{ + cat <<-EOF + + + + + + + $NAME - CI logs + + + +
+

+ CI logs for + $NAME +

+
    + EOF + + + PASS='✅' # ✅ + WARN='🐌' # 🐌 + FAIL='❌' # ❌ + find data/ -type f | LANG=C.UTF-8 sort -r | while read -r f; do + STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)" + SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)" + FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)" + DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)" + MESSAGE="$({ + git log -1 --format=%B "$SHA" || { + git fetch origin "$SHA" + git log -1 --format=%B "$SHA" + } + } | esc)" + + if [ "$STATUS" = 0 ]; then + if [ "$DURATION" -le 60 ]; then + STATUS_MARKER="$PASS" + else + STATUS_MARKER="$WARN" + fi + else + STATUS_MARKER="$FAIL" + fi + + cat <<-EOF +
  1. + 
    #
     + $STATUS_MARKER - 
    ${DURATION:-?}s
    + 
    (commit)
    + 
    $FILENAME
     + 
    (data)
    +
    + 
    $MESSAGE
     +
    2. + EOF + done + + cat <<-EOF +
+
+ + + EOF +} > index.html diff --git a/v2/aux/domain.txt b/v2/aux/domain.txt new file mode 100644 index 0000000..fd7ea0f --- /dev/null +++ b/v2/aux/domain.txt @@ -0,0 +1 @@ +euandre.org diff --git a/v2/aux/lib.sh b/v2/aux/lib.sh new file mode 100644 index 0000000..93a37eb --- /dev/null +++ b/v2/aux/lib.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +DOMAIN="$(cat aux/domain.txt)" +NAME="$(basename "$PWD")" +export DOMAIN NAME + +assert_arg() { + if [ -z "$1" ]; then + printf 'Missing %s.\n\n' "$2" >&2 + cat <<-'EOF' + usage >&2 + exit 2 + EOF + fi +} + +uuid() { + od -xN20 /dev/urandom | + head -n1 | + awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}' +} + +tmpname() { + printf '%s/uuid-tmpname-without-spaces.%s' "${TMPDIR:-/tmp}" "$(uuid)" +} + +mkstemp() { + name="$(tmpname)" + touch -- "$name" + printf '%s' "$name" +} + +mkdtemp() { + name="$(tmpname)" + mkdir -p -- "$name" + printf '%s' "$name" +} diff --git a/v2/aux/po4a-cfg.sh b/v2/aux/po4a-cfg.sh new file mode 100755 index 0000000..b20e303 --- /dev/null +++ b/v2/aux/po4a-cfg.sh @@ -0,0 +1,109 @@ +#!/bin/sh +set -eu + +usage() { + cat <<-'EOF' + Usage: + aux/po4a-cfg.sh > po/po4a.cfg + aux/po4a-cfg.sh -h + EOF +} + +help() { + cat <<-'EOF' + + + Options: + -h, --help show this message + + + Discover translatable files in the repository (via + git-ls-files(1)) that have '.en.' or '/en/' in their name and + emit the configuration file to be used with po4a(1). + + + Examples: + + Setup i18n on a new repository: + + $ mkdir po + $ touch po/pt.po + $ touch po/"$(basename "$PWD")".pot + $ aux/po4a-cfg.sh > po/po4a.cfg + $ po4a po/po4a.cfg + + + Conditionally update the configuration in a Makefile: + + po/po4a.cfg: ALWAYS + @sh aux/po4a-cfg.sh | ifnew $@ + po4a $@ + EOF +} + + +for flag in "$@"; do + case "$flag" in + --) + break + ;; + --help) + usage + help + exit + ;; + *) + ;; + esac +done + +while getopts 'h' flag; do + case "$flag" in + h) + usage + help + exit + ;; + *) + usage >&2 + exit 2 + ;; + esac +done +shift $((OPTIND - 1)) + +. aux/lib.sh + + +guess_type() { + case "$1" in + *.md) + echo text + ;; + *.[1-9]*.in) + echo man + ;; + *.html) + echo xhtml + ;; + *) + echo text + ;; + esac +} + + +cat <<-'EOF' + [options] --keep 0 --master-charset UTF-8 --localized-charset UTF-8 + + [po_directory] po + +EOF + +git ls-files | grep -F '.en.' | while read -r file; do + TYPE="$(guess_type "$file")" + # shellcheck disable=2016 + VAR_FILE="$(printf '%s' "$file" | sed 's|\.en\.|.$lang.|')" + # shellcheck disable=2016 + printf '[type: %s] %s $lang:%s\n' "$TYPE" "$file" "$VAR_FILE" +done diff --git a/v2/aux/workflow/TODOs.sh b/v2/aux/workflow/TODOs.sh deleted file mode 120000 index aaa6c50..0000000 --- a/v2/aux/workflow/TODOs.sh +++ /dev/null @@ -1 +0,0 @@ -../../../aux/workflow/TODOs.sh \ No newline at end of file diff --git a/v2/aux/workflow/favicon.html b/v2/aux/workflow/favicon.html deleted file mode 100644 index 6f1bac8..0000000 --- a/v2/aux/workflow/favicon.html +++ /dev/null @@ -1 +0,0 @@ - diff --git a/v2/aux/workflow/preamble.md b/v2/aux/workflow/preamble.md deleted file mode 120000 index 739f3ea..0000000 --- a/v2/aux/workflow/preamble.md +++ /dev/null @@ -1 +0,0 @@ -../../../aux/workflow/preamble.md \ No newline at end of file diff --git a/v2/aux/workflow/style.css b/v2/aux/workflow/style.css deleted file mode 120000 index ffff132..0000000 --- a/v2/aux/workflow/style.css +++ /dev/null @@ -1 +0,0 @@ -../../../aux/workflow/style.css \ No newline at end of file -- cgit v1.2.3