From 63d46fe31f3ae7b7eed1a2c801448160fe9e77d8 Mon Sep 17 00:00:00 2001 From: EuAndreh Date: Mon, 10 Mar 2025 14:19:04 -0300 Subject: cat src/cicd-run src/cireport >> src/cicd --- deps.mk | 2 - src/cicd | 399 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/cicd-run | 136 -------------------- src/cireport | 260 -------------------------------------- 4 files changed, 399 insertions(+), 398 deletions(-) delete mode 100755 src/cicd-run delete mode 100755 src/cireport diff --git a/deps.mk b/deps.mk index e852d20..0a4e047 100644 --- a/deps.mk +++ b/deps.mk @@ -1,6 +1,4 @@ sources.sh = \ src/cicd \ src/cicd-post-receive \ - src/cicd-run \ - src/cireport \ diff --git a/src/cicd b/src/cicd index d508b93..bcf32f1 100755 --- a/src/cicd +++ b/src/cicd @@ -52,3 +52,402 @@ for repo in "$@"; do EOF printf 'Enqueued with id %s for %s\n' "$id" "$repo" done + +exit +#!/bin/sh +set -eu + + +usage() { + cat <<-'EOF' + Usage: cicdd + EOF +} + + +N=`nproc` + +if [ ! -e "$CLONE" ]; then + git clone "$REPO" "$CLONE" +fi + +cd "$CLONE" +git fetch +git checkout "$VERSION" + +MANIFEST_OPT= +if [ -f manifest.scm ]; then + MANIFEST_OPT='-m manifest.scm' +fi +guix shell -Cv3 $MANIFEST_OPT -- make -j$N all check + + + +NAME="$(basename "$PWD" .git)" +LOGS_DIR=/var/log/ci/"$NAME"/ +TIMESTAMP="$(now)" +FILENAME="$TIMESTAMP-$SHA.log" +LOGFILE="$LOGS_DIR/$FILENAME" +mkdir -p "$LOGS_DIR" + + +info() { + pre "$(color -c lightblueb 'CI')" +} + + + +{ + cat <<-EOF | info + Starting CI job at: $(now) + EOF + START="$(epoch)" + + duration() { + if [ "$RUN_DURATION" -gt 60 ]; then + cat <<-EOF + $(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds) + EOF + else + cat <<-EOF + Run took $RUN_DURATION seconds. + EOF + fi + } + + finish() { + STATUS="$?" + END="$(epoch)" + RUN_DURATION=$((END - START)) + cat <<-EOF | info + Finishing CI job at: $(now) + Exit status was $STATUS + Re-run with: + \$ $CMD + $(duration) + EOF + + NOTE="$( + cat <<-EOF + See CI logs with: + git notes --ref=refs/notes/ci-logs show $SHA + git notes --ref=refs/notes/ci-data show $SHA + + Exit status: $STATUS + Duration: $RUN_DURATION + EOF + )" + git notes --ref=refs/notes/ci-data add -f -m "$( + cat <<-EOF + status $STATUS + sha $SHA + filename $FILENAME + duration $RUN_DURATION + timestamp $TIMESTAMP + to-prod $TO_PROD + refname $REFNAME + EOF + )" "$SHA" + git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA" + git notes add -f -m "$NOTE" "$SHA" + + { + printf 'Git CI HTML report for %s (%s) started.\n' "$NAME" "$SHA" >&2 + DIR="$(mkdtemp)" + report -o "$DIR" + sudo -u deployer rsync \ + --chmod=D775,F664 \ + --chown=deployer:deployer \ + --delete \ + -a \ + "$DIR"/ "$HTML_OUTDIR_CI"/ + rm -rf "$DIR" + printf 'Git CI HTML report for %s (%s) finished.\n' "$NAME" "$SHA" >&2 + } 2>&1 | logger -i -p local0.warn -t git-ci 1>/dev/null 2>&1 & + } + trap finish EXIT + + unset GIT_DIR + + if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then + cat <<-EOF | info + In branch "main", running deploy for $SHA. + EOF + TO_PROD=true + CMD="sudo reconfigure $SHA" + else + if [ "$SKIP_DEPLOY" = true ]; then + cat <<-EOF | info + "deploy.skip" option detected, skipping deploy for $SHA. + EOF + else + cat <<-EOF | info + Not on branch "main", skipping deploy for $SHA. + EOF + fi + TO_PROD=false + CMD="sudo reconfigure -n $SHA" + fi + $CMD +} 2>&1 | ts -s '%.s' | tee "$LOGFILE" +exit +#!/bin/sh +set -eu + +usage() { + cat <<-'EOF' + Usage: + report [-C REPO] -o DIRECTORY + report -h + EOF +} + +help() { + cat <<-'EOF' + + + Options: + -C REPO change to REPO when doing Git operations (default: $PWD) + -o DIRECTORY the directory where to place the generated files + -h, --help show this message + + + Gather data from Git Notes, and generate an HTML report on CI runs. + + Two refs with notes are expected: + 1. refs/notes/ci-data: contains metadata abount the CI runs, + with timestamps, filenames and exit status; + 2. refs/notes/ci-logs: contains the content of the log. + + When reconstructing the CI run, the $FILENAME present in + the refs/notes/ci-data ref names the file, and its content comes + from refs/notes/ci-logs. + + On a CI run that generated the numbers from 1 to 10, for a file named + 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the + expected output on the target directory "public" is: + + $ tree public/ + public/ + index.html + data/ + my-ci-run-2020-01-01-deadbeef.log + ... + logs/ + my-ci-run-2020-01-01-deadbeef.log + ... + + $ cat public/data/my-ci-run-2020-01-01-deadbeef.log + 0 deadbeef my-ci-run-2020-01-01-deadbeef.log + + $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + + The generated 'index.html' is a webpage with the list of all known + CI runs, their status, a link to the commit and a link to the + log file. + + To enable fetching these refs by default, do so in the git config: + + $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*' + + + Examples: + + Generate the report on the 'www' directory: + + $ report -o www + EOF +} + + +for flag in "$@"; do + case "$flag" in + --) + break + ;; + --help) + usage + help + exit + ;; + *) + ;; + esac +done + +REPO="$PWD" +while getopts 'C:o:h' flag; do + case "$flag" in + C) + REPO="$OPTARG" + ;; + o) + OUTDIR="$OPTARG" + ;; + h) + usage + help + exit + ;; + *) + exit 2 + ;; + esac +done +shift $((OPTIND - 1)) + +if [ -z "${OUTDIR:-}" ]; then + printf 'Missing -o OUTDIR.\n\n' >&2 + usage >&2 + exit 2 +fi + +if [ -r src/infrastructure/config/conf.env ]; then + CONF=src/infrastructure/config/conf.env +else + CONF=/etc/conf.env +fi + +# shellcheck source=/dev/null +. "$CONF" + + +esc() { + sed \ + -e 's|&|\&|g' \ + -e 's|<|\<|g' \ + -e 's|>|\>|g' \ + -e 's|"|\"|g' \ + -e "s|'|\'|g" +} + +mkdir -p "$OUTDIR" +cd "$OUTDIR" +mkdir -p logs data + +for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do + git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp + FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)" + mv data/FILENAME-tmp data/"$FILENAME" + git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME" +done + +{ + cat <<-EOF + + + + + + + + $NAME - CI logs + + + +
+

+ CI logs for + $NAME +

+
    + EOF + + + PASS='✅' # ✅ + WARN='🐌' # 🐌 + FAIL='❌' # ❌ + find data/ -type f | LANG=C.UTF-8 sort -r | while read -r f; do + STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)" + SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)" + FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)" + DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)" + MESSAGE="$({ + git -C "$REPO" log -1 --format=%B "$SHA" || { + git fetch origin "$SHA" + git -C "$REPO" log -1 --format=%B "$SHA" + } + } | esc)" + + if [ "$STATUS" = 0 ]; then + if [ "$DURATION" -le 60 ]; then + STATUS_MARKER="$PASS" + else + STATUS_MARKER="$WARN" + fi + else + STATUS_MARKER="$FAIL" + fi + + cat <<-EOF +
  1. + 
    #
     + $STATUS_MARKER - 
    ${DURATION:-?}s
    + 
    (commit)
    + 
    $FILENAME
     + 
    (data)
    +
    + 
    $MESSAGE
     +
    2. + EOF + done + + cat <<-EOF +
+
+ + + EOF +} > index.html diff --git a/src/cicd-run b/src/cicd-run deleted file mode 100755 index 66e5f8c..0000000 --- a/src/cicd-run +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/sh -set -eu - - -usage() { - cat <<-'EOF' - Usage: cicdd - EOF -} - - -N=`nproc` - -if [ ! -e "$CLONE" ]; then - git clone "$REPO" "$CLONE" -fi - -cd "$CLONE" -git fetch -git checkout "$VERSION" - -MANIFEST_OPT= -if [ -f manifest.scm ]; then - MANIFEST_OPT='-m manifest.scm' -fi -guix shell -Cv3 $MANIFEST_OPT -- make -j$N all check - - - -NAME="$(basename "$PWD" .git)" -LOGS_DIR=/var/log/ci/"$NAME"/ -TIMESTAMP="$(now)" -FILENAME="$TIMESTAMP-$SHA.log" -LOGFILE="$LOGS_DIR/$FILENAME" -mkdir -p "$LOGS_DIR" - - -info() { - pre "$(color -c lightblueb 'CI')" -} - - - -{ - cat <<-EOF | info - Starting CI job at: $(now) - EOF - START="$(epoch)" - - duration() { - if [ "$RUN_DURATION" -gt 60 ]; then - cat <<-EOF - $(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds) - EOF - else - cat <<-EOF - Run took $RUN_DURATION seconds. - EOF - fi - } - - finish() { - STATUS="$?" - END="$(epoch)" - RUN_DURATION=$((END - START)) - cat <<-EOF | info - Finishing CI job at: $(now) - Exit status was $STATUS - Re-run with: - \$ $CMD - $(duration) - EOF - - NOTE="$( - cat <<-EOF - See CI logs with: - git notes --ref=refs/notes/ci-logs show $SHA - git notes --ref=refs/notes/ci-data show $SHA - - Exit status: $STATUS - Duration: $RUN_DURATION - EOF - )" - git notes --ref=refs/notes/ci-data add -f -m "$( - cat <<-EOF - status $STATUS - sha $SHA - filename $FILENAME - duration $RUN_DURATION - timestamp $TIMESTAMP - to-prod $TO_PROD - refname $REFNAME - EOF - )" "$SHA" - git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA" - git notes add -f -m "$NOTE" "$SHA" - - { - printf 'Git CI HTML report for %s (%s) started.\n' "$NAME" "$SHA" >&2 - DIR="$(mkdtemp)" - report -o "$DIR" - sudo -u deployer rsync \ - --chmod=D775,F664 \ - --chown=deployer:deployer \ - --delete \ - -a \ - "$DIR"/ "$HTML_OUTDIR_CI"/ - rm -rf "$DIR" - printf 'Git CI HTML report for %s (%s) finished.\n' "$NAME" "$SHA" >&2 - } 2>&1 | logger -i -p local0.warn -t git-ci 1>/dev/null 2>&1 & - } - trap finish EXIT - - unset GIT_DIR - - if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then - cat <<-EOF | info - In branch "main", running deploy for $SHA. - EOF - TO_PROD=true - CMD="sudo reconfigure $SHA" - else - if [ "$SKIP_DEPLOY" = true ]; then - cat <<-EOF | info - "deploy.skip" option detected, skipping deploy for $SHA. - EOF - else - cat <<-EOF | info - Not on branch "main", skipping deploy for $SHA. - EOF - fi - TO_PROD=false - CMD="sudo reconfigure -n $SHA" - fi - $CMD -} 2>&1 | ts -s '%.s' | tee "$LOGFILE" diff --git a/src/cireport b/src/cireport deleted file mode 100755 index e14e40a..0000000 --- a/src/cireport +++ /dev/null @@ -1,260 +0,0 @@ -#!/bin/sh -set -eu - -usage() { - cat <<-'EOF' - Usage: - report [-C REPO] -o DIRECTORY - report -h - EOF -} - -help() { - cat <<-'EOF' - - - Options: - -C REPO change to REPO when doing Git operations (default: $PWD) - -o DIRECTORY the directory where to place the generated files - -h, --help show this message - - - Gather data from Git Notes, and generate an HTML report on CI runs. - - Two refs with notes are expected: - 1. refs/notes/ci-data: contains metadata abount the CI runs, - with timestamps, filenames and exit status; - 2. refs/notes/ci-logs: contains the content of the log. - - When reconstructing the CI run, the $FILENAME present in - the refs/notes/ci-data ref names the file, and its content comes - from refs/notes/ci-logs. - - On a CI run that generated the numbers from 1 to 10, for a file named - 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the - expected output on the target directory "public" is: - - $ tree public/ - public/ - index.html - data/ - my-ci-run-2020-01-01-deadbeef.log - ... - logs/ - my-ci-run-2020-01-01-deadbeef.log - ... - - $ cat public/data/my-ci-run-2020-01-01-deadbeef.log - 0 deadbeef my-ci-run-2020-01-01-deadbeef.log - - $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - - The generated 'index.html' is a webpage with the list of all known - CI runs, their status, a link to the commit and a link to the - log file. - - To enable fetching these refs by default, do so in the git config: - - $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*' - - - Examples: - - Generate the report on the 'www' directory: - - $ report -o www - EOF -} - - -for flag in "$@"; do - case "$flag" in - --) - break - ;; - --help) - usage - help - exit - ;; - *) - ;; - esac -done - -REPO="$PWD" -while getopts 'C:o:h' flag; do - case "$flag" in - C) - REPO="$OPTARG" - ;; - o) - OUTDIR="$OPTARG" - ;; - h) - usage - help - exit - ;; - *) - exit 2 - ;; - esac -done -shift $((OPTIND - 1)) - -if [ -z "${OUTDIR:-}" ]; then - printf 'Missing -o OUTDIR.\n\n' >&2 - usage >&2 - exit 2 -fi - -if [ -r src/infrastructure/config/conf.env ]; then - CONF=src/infrastructure/config/conf.env -else - CONF=/etc/conf.env -fi - -# shellcheck source=/dev/null -. "$CONF" - - -esc() { - sed \ - -e 's|&|\&|g' \ - -e 's|<|\<|g' \ - -e 's|>|\>|g' \ - -e 's|"|\"|g' \ - -e "s|'|\'|g" -} - -mkdir -p "$OUTDIR" -cd "$OUTDIR" -mkdir -p logs data - -for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do - git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp - FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)" - mv data/FILENAME-tmp data/"$FILENAME" - git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME" -done - -{ - cat <<-EOF - - - - - - - - $NAME - CI logs - - - -
-

- CI logs for - $NAME -

-
    - EOF - - - PASS='✅' # ✅ - WARN='🐌' # 🐌 - FAIL='❌' # ❌ - find data/ -type f | LANG=C.UTF-8 sort -r | while read -r f; do - STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)" - SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)" - FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)" - DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)" - MESSAGE="$({ - git -C "$REPO" log -1 --format=%B "$SHA" || { - git fetch origin "$SHA" - git -C "$REPO" log -1 --format=%B "$SHA" - } - } | esc)" - - if [ "$STATUS" = 0 ]; then - if [ "$DURATION" -le 60 ]; then - STATUS_MARKER="$PASS" - else - STATUS_MARKER="$WARN" - fi - else - STATUS_MARKER="$FAIL" - fi - - cat <<-EOF -
  1. - 
    #
     - $STATUS_MARKER - 
    ${DURATION:-?}s
    - 
    (commit)
    - 
    $FILENAME
     - 
    (data)
    -
    - 
    $MESSAGE
     -
    2. - EOF - done - - cat <<-EOF -
-
- - - EOF -} > index.html -- cgit v1.2.3