diff options
| -rw-r--r-- | deps.mk | 2 | ||||
| -rwxr-xr-x | src/cicd | 399 | ||||
| -rwxr-xr-x | src/cicd-run | 136 | ||||
| -rwxr-xr-x | src/cireport | 260 |
4 files changed, 399 insertions, 398 deletions
@@ -1,6 +1,4 @@ sources.sh = \ src/cicd \ src/cicd-post-receive \ - src/cicd-run \ - src/cireport \ @@ -52,3 +52,402 @@ for repo in "$@"; do EOF printf 'Enqueued with id %s for %s\n' "$id" "$repo" done + +exit +#!/bin/sh +set -eu + + +usage() { + cat <<-'EOF' + Usage: cicdd + EOF +} + + +N=`nproc` + +if [ ! -e "$CLONE" ]; then + git clone "$REPO" "$CLONE" +fi + +cd "$CLONE" +git fetch +git checkout "$VERSION" + +MANIFEST_OPT= +if [ -f manifest.scm ]; then + MANIFEST_OPT='-m manifest.scm' +fi +guix shell -Cv3 $MANIFEST_OPT -- make -j$N all check + + + +NAME="$(basename "$PWD" .git)" +LOGS_DIR=/var/log/ci/"$NAME"/ +TIMESTAMP="$(now)" +FILENAME="$TIMESTAMP-$SHA.log" +LOGFILE="$LOGS_DIR/$FILENAME" +mkdir -p "$LOGS_DIR" + + +info() { + pre "$(color -c lightblueb 'CI')" +} + + + +{ + cat <<-EOF | info + Starting CI job at: $(now) + EOF + START="$(epoch)" + + duration() { + if [ "$RUN_DURATION" -gt 60 ]; then + cat <<-EOF + $(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds) + EOF + else + cat <<-EOF + Run took $RUN_DURATION seconds. + EOF + fi + } + + finish() { + STATUS="$?" + END="$(epoch)" + RUN_DURATION=$((END - START)) + cat <<-EOF | info + Finishing CI job at: $(now) + Exit status was $STATUS + Re-run with: + \$ $CMD + $(duration) + EOF + + NOTE="$( + cat <<-EOF + See CI logs with: + git notes --ref=refs/notes/ci-logs show $SHA + git notes --ref=refs/notes/ci-data show $SHA + + Exit status: $STATUS + Duration: $RUN_DURATION + EOF + )" + git notes --ref=refs/notes/ci-data add -f -m "$( + cat <<-EOF + status $STATUS + sha $SHA + filename $FILENAME + duration $RUN_DURATION + timestamp $TIMESTAMP + to-prod $TO_PROD + refname $REFNAME + EOF + )" "$SHA" + git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA" + git notes add -f -m "$NOTE" "$SHA" + + { + printf 'Git CI HTML report for %s (%s) started.\n' "$NAME" "$SHA" >&2 + DIR="$(mkdtemp)" + report -o "$DIR" + sudo -u deployer rsync \ + --chmod=D775,F664 \ + --chown=deployer:deployer \ + --delete \ + -a \ + "$DIR"/ "$HTML_OUTDIR_CI"/ + rm -rf "$DIR" + printf 'Git CI HTML report for %s (%s) finished.\n' "$NAME" "$SHA" >&2 + } 2>&1 | logger -i -p local0.warn -t git-ci 1>/dev/null 2>&1 & + } + trap finish EXIT + + unset GIT_DIR + + if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then + cat <<-EOF | info + In branch "main", running deploy for $SHA. + EOF + TO_PROD=true + CMD="sudo reconfigure $SHA" + else + if [ "$SKIP_DEPLOY" = true ]; then + cat <<-EOF | info + "deploy.skip" option detected, skipping deploy for $SHA. + EOF + else + cat <<-EOF | info + Not on branch "main", skipping deploy for $SHA. + EOF + fi + TO_PROD=false + CMD="sudo reconfigure -n $SHA" + fi + $CMD +} 2>&1 | ts -s '%.s' | tee "$LOGFILE" +exit +#!/bin/sh +set -eu + +usage() { + cat <<-'EOF' + Usage: + report [-C REPO] -o DIRECTORY + report -h + EOF +} + +help() { + cat <<-'EOF' + + + Options: + -C REPO change to REPO when doing Git operations (default: $PWD) + -o DIRECTORY the directory where to place the generated files + -h, --help show this message + + + Gather data from Git Notes, and generate an HTML report on CI runs. + + Two refs with notes are expected: + 1. refs/notes/ci-data: contains metadata abount the CI runs, + with timestamps, filenames and exit status; + 2. refs/notes/ci-logs: contains the content of the log. + + When reconstructing the CI run, the $FILENAME present in + the refs/notes/ci-data ref names the file, and its content comes + from refs/notes/ci-logs. + + On a CI run that generated the numbers from 1 to 10, for a file named + 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the + expected output on the target directory "public" is: + + $ tree public/ + public/ + index.html + data/ + my-ci-run-2020-01-01-deadbeef.log + ... + logs/ + my-ci-run-2020-01-01-deadbeef.log + ... + + $ cat public/data/my-ci-run-2020-01-01-deadbeef.log + 0 deadbeef my-ci-run-2020-01-01-deadbeef.log + + $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + + The generated 'index.html' is a webpage with the list of all known + CI runs, their status, a link to the commit and a link to the + log file. + + To enable fetching these refs by default, do so in the git config: + + $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*' + + + Examples: + + Generate the report on the 'www' directory: + + $ report -o www + EOF +} + + +for flag in "$@"; do + case "$flag" in + --) + break + ;; + --help) + usage + help + exit + ;; + *) + ;; + esac +done + +REPO="$PWD" +while getopts 'C:o:h' flag; do + case "$flag" in + C) + REPO="$OPTARG" + ;; + o) + OUTDIR="$OPTARG" + ;; + h) + usage + help + exit + ;; + *) + exit 2 + ;; + esac +done +shift $((OPTIND - 1)) + +if [ -z "${OUTDIR:-}" ]; then + printf 'Missing -o OUTDIR.\n\n' >&2 + usage >&2 + exit 2 +fi + +if [ -r src/infrastructure/config/conf.env ]; then + CONF=src/infrastructure/config/conf.env +else + CONF=/etc/conf.env +fi + +# shellcheck source=/dev/null +. "$CONF" + + +esc() { + sed \ + -e 's|&|\&|g' \ + -e 's|<|\<|g' \ + -e 's|>|\>|g' \ + -e 's|"|\"|g' \ + -e "s|'|\'|g" +} + +mkdir -p "$OUTDIR" +cd "$OUTDIR" +mkdir -p logs data + +for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do + git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp + FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)" + mv data/FILENAME-tmp data/"$FILENAME" + git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME" +done + +{ + cat <<-EOF + <!DOCTYPE html> + <html lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <meta name="description" content="CI logs for $NAME" /> + <link rel="icon" type="image/svg+xml" href="/favicon.svg" /> + <title>$NAME - CI logs</title> + <style> + body { + max-width: 800px; + margin: 0 auto; + } + + code { + display: block; + margin: 1em 0em 3em 3em; + overflow: auto; + } + + pre { + display: inline; + } + + ol { + list-style-type: disc; + } + + pre, code { + background-color: #ddd; + } + + @media(prefers-color-scheme: dark) { + :root { + color: white; + background-color: black; + } + + a { + color: hsl(211, 100%, 60%); + } + + a:visited { + color: hsl(242, 100%, 80%); + } + + pre, code { + background-color: #222; + } + } + </style> + </head> + <body> + <main> + <h1> + CI logs for + <a href="$HOMEPAGE">$NAME</a> + </h1> + <ol> + EOF + + + PASS='✅' # ✅ + WARN='🐌' # 🐌 + FAIL='❌' # ❌ + find data/ -type f | LANG=C.UTF-8 sort -r | while read -r f; do + STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)" + SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)" + FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)" + DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)" + MESSAGE="$({ + git -C "$REPO" log -1 --format=%B "$SHA" || { + git fetch origin "$SHA" + git -C "$REPO" log -1 --format=%B "$SHA" + } + } | esc)" + + if [ "$STATUS" = 0 ]; then + if [ "$DURATION" -le 60 ]; then + STATUS_MARKER="$PASS" + else + STATUS_MARKER="$WARN" + fi + else + STATUS_MARKER="$FAIL" + fi + + cat <<-EOF + <li id="$FILENAME"> + <a href="#$FILENAME"><pre>#</pre></a> + $STATUS_MARKER - <pre>${DURATION:-?}s</pre> + <pre>(<a href="${CGIT_URL}${SHA}">commit</a>)</pre> + <a href="logs/$FILENAME"><pre>$FILENAME</pre></a> + <pre>(<a href="data/$FILENAME">data</a>)</pre> + <br /> + <code><pre>$MESSAGE</pre></code> + </li> + EOF + done + + cat <<-EOF + </ol> + </main> + </body> + </html> + EOF +} > index.html diff --git a/src/cicd-run b/src/cicd-run deleted file mode 100755 index 66e5f8c..0000000 --- a/src/cicd-run +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/sh -set -eu - - -usage() { - cat <<-'EOF' - Usage: cicdd - EOF -} - - -N=`nproc` - -if [ ! -e "$CLONE" ]; then - git clone "$REPO" "$CLONE" -fi - -cd "$CLONE" -git fetch -git checkout "$VERSION" - -MANIFEST_OPT= -if [ -f manifest.scm ]; then - MANIFEST_OPT='-m manifest.scm' -fi -guix shell -Cv3 $MANIFEST_OPT -- make -j$N all check - - - -NAME="$(basename "$PWD" .git)" -LOGS_DIR=/var/log/ci/"$NAME"/ -TIMESTAMP="$(now)" -FILENAME="$TIMESTAMP-$SHA.log" -LOGFILE="$LOGS_DIR/$FILENAME" -mkdir -p "$LOGS_DIR" - - -info() { - pre "$(color -c lightblueb 'CI')" -} - - - -{ - cat <<-EOF | info - Starting CI job at: $(now) - EOF - START="$(epoch)" - - duration() { - if [ "$RUN_DURATION" -gt 60 ]; then - cat <<-EOF - $(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds) - EOF - else - cat <<-EOF - Run took $RUN_DURATION seconds. - EOF - fi - } - - finish() { - STATUS="$?" - END="$(epoch)" - RUN_DURATION=$((END - START)) - cat <<-EOF | info - Finishing CI job at: $(now) - Exit status was $STATUS - Re-run with: - \$ $CMD - $(duration) - EOF - - NOTE="$( - cat <<-EOF - See CI logs with: - git notes --ref=refs/notes/ci-logs show $SHA - git notes --ref=refs/notes/ci-data show $SHA - - Exit status: $STATUS - Duration: $RUN_DURATION - EOF - )" - git notes --ref=refs/notes/ci-data add -f -m "$( - cat <<-EOF - status $STATUS - sha $SHA - filename $FILENAME - duration $RUN_DURATION - timestamp $TIMESTAMP - to-prod $TO_PROD - refname $REFNAME - EOF - )" "$SHA" - git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA" - git notes add -f -m "$NOTE" "$SHA" - - { - printf 'Git CI HTML report for %s (%s) started.\n' "$NAME" "$SHA" >&2 - DIR="$(mkdtemp)" - report -o "$DIR" - sudo -u deployer rsync \ - --chmod=D775,F664 \ - --chown=deployer:deployer \ - --delete \ - -a \ - "$DIR"/ "$HTML_OUTDIR_CI"/ - rm -rf "$DIR" - printf 'Git CI HTML report for %s (%s) finished.\n' "$NAME" "$SHA" >&2 - } 2>&1 | logger -i -p local0.warn -t git-ci 1>/dev/null 2>&1 & - } - trap finish EXIT - - unset GIT_DIR - - if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then - cat <<-EOF | info - In branch "main", running deploy for $SHA. - EOF - TO_PROD=true - CMD="sudo reconfigure $SHA" - else - if [ "$SKIP_DEPLOY" = true ]; then - cat <<-EOF | info - "deploy.skip" option detected, skipping deploy for $SHA. - EOF - else - cat <<-EOF | info - Not on branch "main", skipping deploy for $SHA. - EOF - fi - TO_PROD=false - CMD="sudo reconfigure -n $SHA" - fi - $CMD -} 2>&1 | ts -s '%.s' | tee "$LOGFILE" diff --git a/src/cireport b/src/cireport deleted file mode 100755 index e14e40a..0000000 --- a/src/cireport +++ /dev/null @@ -1,260 +0,0 @@ -#!/bin/sh -set -eu - -usage() { - cat <<-'EOF' - Usage: - report [-C REPO] -o DIRECTORY - report -h - EOF -} - -help() { - cat <<-'EOF' - - - Options: - -C REPO change to REPO when doing Git operations (default: $PWD) - -o DIRECTORY the directory where to place the generated files - -h, --help show this message - - - Gather data from Git Notes, and generate an HTML report on CI runs. - - Two refs with notes are expected: - 1. refs/notes/ci-data: contains metadata abount the CI runs, - with timestamps, filenames and exit status; - 2. refs/notes/ci-logs: contains the content of the log. - - When reconstructing the CI run, the $FILENAME present in - the refs/notes/ci-data ref names the file, and its content comes - from refs/notes/ci-logs. - - On a CI run that generated the numbers from 1 to 10, for a file named - 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the - expected output on the target directory "public" is: - - $ tree public/ - public/ - index.html - data/ - my-ci-run-2020-01-01-deadbeef.log - ... - logs/ - my-ci-run-2020-01-01-deadbeef.log - ... - - $ cat public/data/my-ci-run-2020-01-01-deadbeef.log - 0 deadbeef my-ci-run-2020-01-01-deadbeef.log - - $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - - The generated 'index.html' is a webpage with the list of all known - CI runs, their status, a link to the commit and a link to the - log file. - - To enable fetching these refs by default, do so in the git config: - - $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*' - - - Examples: - - Generate the report on the 'www' directory: - - $ report -o www - EOF -} - - -for flag in "$@"; do - case "$flag" in - --) - break - ;; - --help) - usage - help - exit - ;; - *) - ;; - esac -done - -REPO="$PWD" -while getopts 'C:o:h' flag; do - case "$flag" in - C) - REPO="$OPTARG" - ;; - o) - OUTDIR="$OPTARG" - ;; - h) - usage - help - exit - ;; - *) - exit 2 - ;; - esac -done -shift $((OPTIND - 1)) - -if [ -z "${OUTDIR:-}" ]; then - printf 'Missing -o OUTDIR.\n\n' >&2 - usage >&2 - exit 2 -fi - -if [ -r src/infrastructure/config/conf.env ]; then - CONF=src/infrastructure/config/conf.env -else - CONF=/etc/conf.env -fi - -# shellcheck source=/dev/null -. "$CONF" - - -esc() { - sed \ - -e 's|&|\&|g' \ - -e 's|<|\<|g' \ - -e 's|>|\>|g' \ - -e 's|"|\"|g' \ - -e "s|'|\'|g" -} - -mkdir -p "$OUTDIR" -cd "$OUTDIR" -mkdir -p logs data - -for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do - git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp - FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)" - mv data/FILENAME-tmp data/"$FILENAME" - git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME" -done - -{ - cat <<-EOF - <!DOCTYPE html> - <html lang="en"> - <head> - <meta charset="UTF-8" /> - <meta name="viewport" content="width=device-width, initial-scale=1" /> - <meta name="description" content="CI logs for $NAME" /> - <link rel="icon" type="image/svg+xml" href="/favicon.svg" /> - <title>$NAME - CI logs</title> - <style> - body { - max-width: 800px; - margin: 0 auto; - } - - code { - display: block; - margin: 1em 0em 3em 3em; - overflow: auto; - } - - pre { - display: inline; - } - - ol { - list-style-type: disc; - } - - pre, code { - background-color: #ddd; - } - - @media(prefers-color-scheme: dark) { - :root { - color: white; - background-color: black; - } - - a { - color: hsl(211, 100%, 60%); - } - - a:visited { - color: hsl(242, 100%, 80%); - } - - pre, code { - background-color: #222; - } - } - </style> - </head> - <body> - <main> - <h1> - CI logs for - <a href="$HOMEPAGE">$NAME</a> - </h1> - <ol> - EOF - - - PASS='✅' # ✅ - WARN='🐌' # 🐌 - FAIL='❌' # ❌ - find data/ -type f | LANG=C.UTF-8 sort -r | while read -r f; do - STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)" - SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)" - FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)" - DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)" - MESSAGE="$({ - git -C "$REPO" log -1 --format=%B "$SHA" || { - git fetch origin "$SHA" - git -C "$REPO" log -1 --format=%B "$SHA" - } - } | esc)" - - if [ "$STATUS" = 0 ]; then - if [ "$DURATION" -le 60 ]; then - STATUS_MARKER="$PASS" - else - STATUS_MARKER="$WARN" - fi - else - STATUS_MARKER="$FAIL" - fi - - cat <<-EOF - <li id="$FILENAME"> - <a href="#$FILENAME"><pre>#</pre></a> - $STATUS_MARKER - <pre>${DURATION:-?}s</pre> - <pre>(<a href="${CGIT_URL}${SHA}">commit</a>)</pre> - <a href="logs/$FILENAME"><pre>$FILENAME</pre></a> - <pre>(<a href="data/$FILENAME">data</a>)</pre> - <br /> - <code><pre>$MESSAGE</pre></code> - </li> - EOF - done - - cat <<-EOF - </ol> - </main> - </body> - </html> - EOF -} > index.html |