aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorEuAndreh <eu@euandre.org>2023-03-11 08:53:30 -0300
committerEuAndreh <eu@euandre.org>2023-03-11 19:36:12 -0300
commit786d6958a5733b28ec089685df7c0eac980c5f2a (patch)
tree9afb19eda6db66f68f5a914aa4fa38fde8b0d9b5
parentchannels.scm: Remove "nonguix" channel (diff)
downloadserver-786d6958a5733b28ec089685df7c0eac980c5f2a.tar.gz
server-786d6958a5733b28ec089685df7c0eac980c5f2a.tar.xz
Copy files back
-rwxr-xr-xsrc/infrastructure/ci/git-post-receive.sh171
-rwxr-xr-xsrc/infrastructure/ci/git-pre-receive.sh14
-rw-r--r--src/infrastructure/config/gitconfig7
-rw-r--r--src/infrastructure/config/init.scm6
-rw-r--r--src/infrastructure/config/profile.sh5
-rw-r--r--src/infrastructure/config/rc.sh75
-rw-r--r--src/infrastructure/config/ssh.conf6
-rw-r--r--src/infrastructure/guix/system.scm6
-rw-r--r--src/infrastructure/keys/andreh.pub (renamed from src/infrastructure/keys/SSH/EuAndreh.pub)0
-rwxr-xr-xsrc/infrastructure/scripts/backup.sh135
-rwxr-xr-xsrc/infrastructure/scripts/cronjob.sh159
-rwxr-xr-xsrc/infrastructure/scripts/deploy.sh71
-rwxr-xr-xsrc/infrastructure/scripts/gc.sh146
-rwxr-xr-xsrc/infrastructure/scripts/r.sh77
-rwxr-xr-xsrc/infrastructure/scripts/reconfigure.sh134
-rwxr-xr-xsrc/infrastructure/scripts/report.sh221
16 files changed, 1231 insertions, 2 deletions
diff --git a/src/infrastructure/ci/git-post-receive.sh b/src/infrastructure/ci/git-post-receive.sh
new file mode 100755
index 0000000..68d4da2
--- /dev/null
+++ b/src/infrastructure/ci/git-post-receive.sh
@@ -0,0 +1,171 @@
+#!/bin/sh
+# shellcheck source=/dev/null disable=2317
+. /etc/rc
+set -eu
+
+
+# shellcheck disable=2034
+read -r _oldrev SHA REFNAME
+
+if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then
+ exit
+fi
+
+
+SKIP_DEPLOY=false
+for n in $(seq 0 $((GIT_PUSH_OPTION_COUNT - 1))); do
+ opt="$(eval "printf '%s' \"\$GIT_PUSH_OPTION_$n\"")"
+ case "$opt" in
+ ci.skip)
+ cat <<-EOF
+
+ "$opt" option detected, not running CI.
+
+ EOF
+ exit
+ ;;
+ deploy.skip)
+ SKIP_DEPLOY=true
+ ;;
+ *)
+ ;;
+ esac
+done
+
+
+now() {
+ date '+%Y-%m-%dT%H:%M:%S%:z'
+}
+
+LOGS_DIR=/var/log/ci/servers/
+TIMESTAMP="$(now)"
+FILENAME="$TIMESTAMP-$SHA.log"
+LOGFILE="$LOGS_DIR/$FILENAME"
+mkdir -p "$LOGS_DIR"
+
+
+END_MARKER='\033[0m'
+LIGHT_BLUE_B='\033[1;36m'
+YELLOW='\033[1;33m'
+
+blue() {
+ printf "${LIGHT_BLUE_B}%s${END_MARKER}" "$1"
+}
+
+yellow() {
+ printf "${YELLOW}%s${END_MARKER}" "$1"
+}
+
+info() {
+ sed "s|^\(.\)|$(blue 'CI'): \1|"
+}
+
+
+uuid() {
+ od -xN20 /dev/urandom |
+ head -n1 |
+ awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}'
+}
+
+tmpname() {
+ printf '%s/uuid-tmpname with spaces.%s' "${TMPDIR:-/tmp}" "$(uuid)"
+}
+
+mkdtemp() {
+ name="$(tmpname)"
+ mkdir -- "$name"
+ printf '%s' "$name"
+}
+
+
+{
+ cat <<-EOF | info
+ Starting CI job at: $(now)
+ EOF
+ START="$(date +%s)"
+
+ duration() {
+ if [ "$RUN_DURATION" -gt 60 ]; then
+ cat <<-EOF
+ $(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds)
+ EOF
+ else
+ cat <<-EOF
+ Run took $RUN_DURATION seconds.
+ EOF
+ fi
+ }
+
+ finish() {
+ STATUS="$?"
+ END="$(date +%s)"
+ RUN_DURATION=$((END - START))
+ cat <<-EOF | info
+ Finishing CI job at: $(now)
+ Exit status was $STATUS
+ Re-run with:
+ \$ $CMD
+ $(duration)
+ EOF
+
+ NOTE="$(
+ cat <<-EOF
+ See CI logs with:
+ git notes --ref=refs/notes/ci-logs show $SHA
+ git notes --ref=refs/notes/ci-data show $SHA
+
+ Exit status: $STATUS
+ Duration: $RUN_DURATION
+ EOF
+ )"
+ git notes --ref=refs/notes/ci-data add -f -m "$(
+ cat <<-EOF
+ status $STATUS
+ sha $SHA
+ filename $FILENAME
+ duration $RUN_DURATION
+ timestamp $TIMESTAMP
+ to-prod $TO_PROD
+ refname $REFNAME
+ EOF
+ )" "$SHA"
+ git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA"
+ git notes add -f -m "$NOTE" "$SHA"
+
+ {
+ DIR="$(mkdtemp)"
+ report -o "$DIR"
+ sudo -u deployer rsync \
+ --chmod=D775,F664 \
+ --chown=deployer:deployer \
+ --delete \
+ -a \
+ "$DIR"/ /srv/www/dev/ci/
+ rm -rf "$DIR"
+ } 1>/dev/null 2>&1 &
+ }
+ trap finish EXIT
+
+ unset GIT_DIR
+
+ if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then
+ cat <<-EOF | info
+ In branch "main", running deploy for $SHA.
+ EOF
+ TO_PROD=true
+ CMD="sudo reconfigure $SHA"
+ else
+ if [ "$SKIP_DEPLOY" = true ]; then
+ cat <<-EOF | info
+ "deploy.skip" option detected, skipping deploy for $SHA.
+ EOF
+ else
+ cat <<-EOF | info
+ Not on branch "main", skipping deploy for $SHA.
+ EOF
+ fi
+ TO_PROD=false
+ CMD="sudo reconfigure -n $SHA"
+ fi
+ $CMD
+} 2>&1 | ts -s '%.s' | tee "$LOGFILE"
diff --git a/src/infrastructure/ci/git-pre-receive.sh b/src/infrastructure/ci/git-pre-receive.sh
new file mode 100755
index 0000000..8cd83ee
--- /dev/null
+++ b/src/infrastructure/ci/git-pre-receive.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+
+read -r _oldrev SHA _refname
+unset GIT_DIR
+
+if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then
+ exit
+fi
+
+printf 'Upgrading post-receive hook...' >&2
+git show "$SHA":src/infrastructure/ci/git-post-receive.sh > hooks/post-receive
+chmod +x hooks/post-receive
+printf 'done.\n' >&2
diff --git a/src/infrastructure/config/gitconfig b/src/infrastructure/config/gitconfig
new file mode 100644
index 0000000..915ee72
--- /dev/null
+++ b/src/infrastructure/config/gitconfig
@@ -0,0 +1,7 @@
+[init]
+ defaultBranch = main
+[user]
+ email = ci@euandre.org
+ name = "euandre.org CI"
+[advice]
+ detachedHead = false
diff --git a/src/infrastructure/config/init.scm b/src/infrastructure/config/init.scm
new file mode 100644
index 0000000..9e962e8
--- /dev/null
+++ b/src/infrastructure/config/init.scm
@@ -0,0 +1,6 @@
+(use-modules
+ (ice-9 colorized)
+ (ice-9 readline))
+
+(activate-colorized)
+(activate-readline)
diff --git a/src/infrastructure/config/profile.sh b/src/infrastructure/config/profile.sh
new file mode 100644
index 0000000..1dca8b2
--- /dev/null
+++ b/src/infrastructure/config/profile.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# shellcheck source=/dev/null
+. /etc/rc
+ln -fs .profile .bashrc
diff --git a/src/infrastructure/config/rc.sh b/src/infrastructure/config/rc.sh
new file mode 100644
index 0000000..c92df73
--- /dev/null
+++ b/src/infrastructure/config/rc.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+
+# shellcheck source=/dev/null
+. /etc/profile
+
+export XDG_PREFIX=~/.usr
+export XDG_CACHE_HOME="$XDG_PREFIX"/var/cache
+export XDG_CONFIG_HOME="$XDG_PREFIX"/etc
+export XDG_DATA_HOME="$XDG_PREFIX"/share
+export XDG_STATE_HOME="$XDG_PREFIX"/state
+export XDG_LOG_HOME="$XDG_PREFIX"/var/log
+
+mkdir -p \
+ "$XDG_CONFIG_HOME" \
+ "$XDG_CACHE_HOME" \
+ "$XDG_DATA_HOME" \
+ "$XDG_LOG_HOME" \
+ "$XDG_STATE_HOME"/ssh/conn
+
+
+GUIX_PROFILE="$XDG_CONFIG_HOME"/guix/current
+if [ -r "$GUIX_PROFILE"/etc/profile ]; then
+ # shellcheck source=/dev/null
+ . "$GUIX_PROFILE"/etc/profile
+fi
+
+export ENV=~/.profile
+export HISTSIZE=-1
+export HISTCONTROL=ignorespace:ignoredups
+export EDITOR=vi
+export VISUAL="$EDITOR"
+export PAGER='less -R'
+
+export EXINIT='
+ " set number
+ " set autoindent
+ set ruler
+ set showmode
+ set showmatch
+'
+
+export HISTFILE="$XDG_STATE_HOME"/bash-history
+export LESSHISTFILE="$XDG_STATE_HOME"/lesshst
+export RLWRAP_HOME="$XDG_CACHE_HOME"/rlwrap
+export GUILE_HISTORY="$XDG_STATE_HOME"/guile-history
+
+HOSTNAME="$(hostname)"
+export BORG_REPO="20931@hk-s020.rsync.net:borg/$HOSTNAME"
+export BORG_REMOTE_PATH='borg1'
+export BORG_PASSCOMMAND='cat /opt/secrets/borg-passphrase.txt'
+
+export GIT_CONFIG_GLOBAL=/etc/gitconfig
+
+unalias -a
+alias l='ls -lahF --color'
+alias grep='grep --color=auto'
+alias diff='diff --color=auto'
+alias watch='watch --color '
+alias man='MANWIDTH=$((COLUMNS > 80 ? 80 : COLUMNS)) man'
+alias less='less -R'
+alias tree='tree -aC'
+alias mv='mv -i'
+alias e='vi'
+
+alias sqlite='rlwrap sqlite3'
+alias guile='guile -l /etc/init.scm'
+
+error_marker() {
+ STATUS=$?
+ if [ "$STATUS" != 0 ]; then
+ printf ' (!! %s !!) ' "$STATUS"
+ fi
+}
+export PS1='`error_marker`\T \w/
+\u@\H\$ '
diff --git a/src/infrastructure/config/ssh.conf b/src/infrastructure/config/ssh.conf
new file mode 100644
index 0000000..ca41df0
--- /dev/null
+++ b/src/infrastructure/config/ssh.conf
@@ -0,0 +1,6 @@
+Host *
+ ServerAliveInterval 30
+ ServerAliveCountMax 20
+ ControlMaster auto
+ ControlPath ${XDG_STATE_HOME}/ssh/conn/%r@%h:%p
+ ControlPersist 1h
diff --git a/src/infrastructure/guix/system.scm b/src/infrastructure/guix/system.scm
index 7891f74..322d5b6 100644
--- a/src/infrastructure/guix/system.scm
+++ b/src/infrastructure/guix/system.scm
@@ -1,6 +1,6 @@
(use-modules
((guix licenses) #:prefix license:)
- ((srfi srfi-1) #:prefix srfi-1:)
+ ((srfi srfi-1) #:prefix s1:)
((xyz euandreh heredoc) #:prefix heredoc:)
(gnu)
(gnu build linux-container)
@@ -17,6 +17,7 @@
(gnu packages tls)
(gnu packages version-control)
(gnu system setuid)
+ (guix build utils)
(guix build-system gnu)
(guix build-system trivial)
(guix download)
@@ -26,9 +27,10 @@
(guix records)
(guix utils))
(use-package-modules
- lua
+ ssh
web)
(use-service-modules
+ admin
certbot
cgit
mcron
diff --git a/src/infrastructure/keys/SSH/EuAndreh.pub b/src/infrastructure/keys/andreh.pub
index bfd5e6f..bfd5e6f 100644
--- a/src/infrastructure/keys/SSH/EuAndreh.pub
+++ b/src/infrastructure/keys/andreh.pub
diff --git a/src/infrastructure/scripts/backup.sh b/src/infrastructure/scripts/backup.sh
new file mode 100755
index 0000000..47cc76c
--- /dev/null
+++ b/src/infrastructure/scripts/backup.sh
@@ -0,0 +1,135 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ cat <<-'EOF'
+ Usage:
+ backup [-q] [-C COMMENT] [ARCHIVE_TAG]
+ backup -h
+ EOF
+}
+
+help() {
+ cat <<-'EOF'
+
+
+ Options:
+ -q disable verbose mode, useful for batch sessions
+ -C COMMENT the comment text to be attached to the archive
+ -h, --help show this message
+
+ ARCHIVE_TAG the tag used to create the new
+ backup (default: "manual")
+
+
+ The repository is expected to have been create with:
+
+ $ borg init -e repokey-blake2
+
+ The following environment variables are expected to be exported:
+
+ $BORG_PASSCOMMAND
+ $BORG_REPO
+ $BORG_REMOTE_PATH
+
+ Password-less SSH access is required, usually done via adding
+ /root/.ssh/id_rsa.pub to the ssh remote's
+ $THE_REMOTE:.ssh/authorized_keys
+
+ Root permission is also required.
+
+
+ Examples:
+
+ Run backup from cronjob:
+
+ $ backup -q cronjob
+
+
+ Create backup with a comment, a tag, and verbose mode active:
+
+ $ backup -C 'The backup has a comment'
+ EOF
+}
+
+
+for flag in "$@"; do
+ case "$flag" in
+ --)
+ break
+ ;;
+ --help)
+ usage
+ help
+ exit
+ ;;
+ *)
+ ;;
+ esac
+done
+
+VERBOSE_FLAGS='--verbose --progress'
+COMMENT=' '
+while getopts 'qC:h' flag; do
+ case "$flag" in
+ q)
+ VERBOSE_FLAGS=''
+ ;;
+ C)
+ COMMENT="$OPTARG"
+ ;;
+ h)
+ usage
+ help
+ exit
+ ;;
+ *)
+ usage >&2
+ exit 2
+ ;;
+ esac
+done
+shift $((OPTIND - 1))
+
+ARCHIVE_TAG="${1:-manual}"
+
+
+if [ "$(id -un)" != 'root' ]; then
+ printf 'This script must be run as root.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+
+run() {
+ set -x
+ # shellcheck disable=2086
+ sudo -i borg create \
+ $VERBOSE_FLAGS \
+ --comment "$COMMENT" \
+ --stats \
+ --compression lzma,9 \
+ "$BORG_REPO::$(hostname)-{now}-$ARCHIVE_TAG" \
+ /mnt/production/ \
+ /root/ \
+ /home/ \
+ /etc/ \
+ /var/ \
+ /opt/ \
+ /srv/
+ STATUS=$?
+ set +x
+
+ if [ "$STATUS" = 0 ]; then
+ return 0
+ elif [ "$STATUS" = 1 ]; then
+ printf 'WARNING, but no ERROR.\n' >&2
+ return 0
+ else
+ return "$STATUS"
+ fi
+}
+
+run
+
+sudo -i borg check --verify-data --verbose "$BORG_REPO"
diff --git a/src/infrastructure/scripts/cronjob.sh b/src/infrastructure/scripts/cronjob.sh
new file mode 100755
index 0000000..4823ac1
--- /dev/null
+++ b/src/infrastructure/scripts/cronjob.sh
@@ -0,0 +1,159 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ cat <<-'EOF'
+ Usage:
+ cronjob COMMAND...
+ cronjob -h
+ EOF
+}
+
+help() {
+ cat <<-'EOF'
+
+
+ Options:
+ -h, --help show this message
+
+ COMMAND the command to be executed
+
+
+ Execute the given command, and send the output to email, with
+ special treatment to the status code. It kills the job it it
+ lasts more than one hour.
+
+ It load the appropriate files, so that the actual cron
+ declaration is smaller.
+
+
+ Examples:
+
+ Run a backup:
+
+ $ cronjob backup -q cron
+ EOF
+}
+
+
+for flag in "$@"; do
+ case "$flag" in
+ --)
+ break
+ ;;
+ --help)
+ usage
+ help
+ exit
+ ;;
+ *)
+ ;;
+ esac
+done
+
+while getopts 'h' flag; do
+ case "$flag" in
+ h)
+ usage
+ help
+ exit
+ ;;
+ *)
+ usage >&2
+ exit 2
+ ;;
+ esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${1:-}" ]; then
+ printf 'Missing COMMAND.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+if [ "$(id -un)" != 'root' ]; then
+ printf 'This script must be run as root.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+
+
+now() {
+ date '+%Y-%m-%dT%H:%M:%S%:z'
+}
+
+
+uuid() {
+ od -xN20 /dev/random |
+ head -n1 |
+ awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}'
+}
+
+mkstemp() {
+ name="${TMPDIR:-/tmp}/uuid-tmpname with spaces.$(uuid)"
+ touch "$name"
+ printf '%s' "$name"
+}
+
+pre() {
+ sed -u "s|^|[$CMD]: |"
+}
+
+duration() {
+ minutes=$((${1} / 60))
+ seconds=$((${1} % 60))
+ printf '%sm%ss' "$minutes" "$seconds"
+}
+
+
+CMD="$*"
+HOSTNAME="$(hostname)"
+FROM="cronjob@$HOSTNAME"
+ONE_HOUR='3600'
+STATUS_F="$(mkstemp)"
+OUT="$(mkstemp)"
+
+email() {
+ {
+ cat <<-EOF
+ Content-Type: text/plain; charset=UTF-8
+ Content-Transfer-Encoding: 8bit
+ From: $FROM
+ To: root@localhost
+ Subject: (exit status: $(cat "$STATUS_F")) - $HOSTNAME: $CMD
+
+ EOF
+ cat "$OUT"
+ } | sendmail -t -f "$FROM"
+ rm -f "$OUT" "$STATUS_F"
+}
+trap email EXIT
+
+{
+ cat <<-EOF
+ Running commad: $*
+ Starting at: $(now)
+
+ EOF
+
+ START="$(date +%s)"
+ STATUS=0
+ timeout "$ONE_HOUR" "$@" || STATUS=$?
+ printf '%s' "$STATUS" > "$STATUS_F"
+ END="$(date +%s)"
+ DURATION_SECONDS=$((END - START))
+
+ cat <<-EOF
+
+ Finished at: $(now)
+ Duration: $(duration "$DURATION_SECONDS")
+ EOF
+} 2>&1 | pre | ts '%Y-%m-%dT%H:%M:%S' | tee "$OUT" >> /var/log/cronjobs.log
diff --git a/src/infrastructure/scripts/deploy.sh b/src/infrastructure/scripts/deploy.sh
new file mode 100755
index 0000000..65a50c1
--- /dev/null
+++ b/src/infrastructure/scripts/deploy.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ cat <<-'EOF'
+ Usage:
+ deploy
+ deploy -h
+ EOF
+}
+
+help() {
+ cat <<-'EOF'
+
+ Options:
+ -h, --help show this message
+
+
+ Do a blue/green deployment of the relevant service. It makes
+ sure that the new service is up and running before shutting
+ down the old one.
+
+
+ Examples:
+
+ Just do the deploy:
+
+ $ deploy
+ EOF
+}
+
+
+for flag in "$@"; do
+ case "$flag" in
+ --)
+ break
+ ;;
+ --help)
+ usage
+ help
+ exit
+ ;;
+ *)
+ ;;
+ esac
+done
+
+while getopts 'h' flag; do
+ case "$flag" in
+ h)
+ usage
+ help
+ exit
+ ;;
+ *)
+ usage >&2
+ exit 2
+ ;;
+ esac
+done
+shift $((OPTIND - 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+ printf 'This script must be run as root.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+
+: sudo herd restart a-service
diff --git a/src/infrastructure/scripts/gc.sh b/src/infrastructure/scripts/gc.sh
new file mode 100755
index 0000000..0eca4be
--- /dev/null
+++ b/src/infrastructure/scripts/gc.sh
@@ -0,0 +1,146 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ cat <<-'EOF'
+ Usage:
+ gc [TYPE]
+ gc -h
+ EOF
+}
+
+help() {
+ cat <<-'EOF'
+
+
+ Options:
+ -h, --help show this message
+
+ TYPE what to do GC on (default: all):
+ - guix
+ - deploy
+ - trash
+ - tmpdir
+ - logs
+
+
+ GC the server, deleting old, unusable data, in order to free
+ disk space system-wide.
+
+
+ Examples:
+
+ Just run it, for all:
+
+ $ gc
+
+
+ Cleanup tmpdir:
+
+ $ gc tmpdir
+ EOF
+}
+
+
+for flag in "$@"; do
+ case "$flag" in
+ --)
+ break
+ ;;
+ --help)
+ usage
+ help
+ exit
+ ;;
+ *)
+ ;;
+ esac
+done
+
+while getopts 'h' flag; do
+ case "$flag" in
+ h)
+ usage
+ help
+ exit
+ ;;
+ *)
+ usage >&2
+ exit 2
+ ;;
+ esac
+done
+shift $((OPTIND - 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+ printf 'This script must be run as root.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+
+disk() {
+ df -h / /mnt/backup/ |
+ tail -n +2 |
+ awk '{ printf "%s\t%s/%s\t%s\n", $4, $3, $2, $6 }'
+}
+
+today() {
+ date '+%Y-%m-%d'
+}
+
+gc_guix() {
+ sudo -i guix system delete-generations
+ sudo -i guix gc -d
+}
+
+gc_deploy() {
+ sudo -u deployer find /opt/deploy \
+ ! -path /opt/deploy -prune \
+ -type d \
+ -not -name "$(today)*" \
+ -exec rm -rf "{}" ';'
+}
+
+gc_trash() {
+ yes | sudo -i trash-empty
+}
+
+gc_tmpdir() {
+ find "${TMPDIR:-/tmp}" -atime +10 -exec rm -vf "{}" ';'
+}
+
+gc_logs() {
+ find /var/log/ci/ -atime +10 -exec rm -vf "{}" ';'
+}
+
+
+gc_all() {
+ gc_guix
+ gc_deploy
+ gc_trash
+ gc_tmpdir
+ gc_logs
+}
+
+
+TYPE="${1:-all}"
+CMD=gc_"$TYPE"
+if ! command -v "$CMD" >/dev/null; then
+ printf 'Invalid TYPE: "%s".\n\n' "$TYPE" >&2
+ usage >&2
+ exit 2
+fi
+
+BEFORE="$(disk)"
+set -x
+"$CMD"
+set +x
+AFTER="$(disk)"
+
+cat <<-EOF
+ Disk space:
+ before: $BEFORE
+ after: $AFTER
+EOF
diff --git a/src/infrastructure/scripts/r.sh b/src/infrastructure/scripts/r.sh
new file mode 100755
index 0000000..8e74576
--- /dev/null
+++ b/src/infrastructure/scripts/r.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ cat <<-'EOF'
+ Usage:
+ r COMMAND...
+ r -h
+ EOF
+}
+
+help() {
+ cat <<-'EOF'
+
+
+ Options:
+ -h, --help show this message
+
+ COMMAND the command to be executed
+
+
+ Execute the given command, with a proper login environment
+ loaded.
+
+
+ Examples:
+
+ Run a backup via SSH:
+
+ $ ssh euandre.org r backup -q cron
+ EOF
+}
+
+
+for flag in "$@"; do
+ case "$flag" in
+ --)
+ break
+ ;;
+ --help)
+ usage
+ help
+ exit
+ ;;
+ *)
+ ;;
+ esac
+done
+
+while getopts 'h' flag; do
+ case "$flag" in
+ h)
+ usage
+ help
+ exit
+ ;;
+ *)
+ usage >&2
+ exit 2
+ ;;
+ esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${1:-}" ]; then
+ printf 'Missing COMMAND.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+exec "$@"
diff --git a/src/infrastructure/scripts/reconfigure.sh b/src/infrastructure/scripts/reconfigure.sh
new file mode 100755
index 0000000..c76ea3e
--- /dev/null
+++ b/src/infrastructure/scripts/reconfigure.sh
@@ -0,0 +1,134 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ cat <<-'EOF'
+ Usage:
+ reconfigure [-n] [-U] [SHA]
+ reconfigure -h
+ EOF
+}
+
+help() {
+ cat <<-'EOF'
+
+
+ Options:
+ -n build the system, but don't switch to it (dry-run)
+ -U pull the latest channels before reconfiguring
+ -h, --help show this message
+
+ SHA the repository SHA to checkout (default: main)
+
+
+ Run a "guix system reconfigure" as root via "sudo -i". If a -U
+ flag is given, perform a "guix pull" (in root profile) prior to
+ the reconfigure. The user must be able to become the "deployer"
+ user, either via "sudo reconfigure" or by being member of the
+ "become-deployer" group.
+
+
+ Examples:
+
+ Reconfigure the system:
+
+ $ reconfigure
+
+
+ Build the system on a custom SHA, but don't switch to it:
+
+ $ reconfigure -n 916dafc092f797349a54515756f2c8e477326511
+
+
+ Update and upgrade:
+
+ $ reconfigure -U
+ EOF
+}
+
+
+for flag in "$@"; do
+ case "$flag" in
+ --)
+ break
+ ;;
+ --help)
+ usage
+ help
+ exit
+ ;;
+ *)
+ ;;
+ esac
+done
+
+UPDATE=false
+DRY_RUN=false
+while getopts 'nUh' flag; do
+ case "$flag" in
+ n)
+ DRY_RUN=true
+ ;;
+ U)
+ UPDATE=true
+ ;;
+ h)
+ usage
+ help
+ exit
+ ;;
+ *)
+ usage >&2
+ exit 2
+ ;;
+ esac
+done
+shift $((OPTIND - 1))
+
+SHA="${1:-main}"
+REPO='/srv/git/servers.git'
+NOW="$(date '+%Y-%m-%dT%H:%M:%S%:z')"
+NOW_DIR=/opt/deploy/"$NOW"
+NPROC=$(($(nproc) * 2 + 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+ printf 'This script must be run as root.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+
+if [ "$UPDATE" = true ] && [ "$DRY_RUN" = false ]; then
+ sudo -i guix pull -v3
+fi
+
+set -x
+sudo -u deployer git clone --depth=1 "file://$REPO" "$NOW_DIR"
+sudo -u deployer rm -f /opt/deploy/current
+sudo -u deployer ln -s "$NOW_DIR" /opt/deploy/current
+cd /opt/deploy/current
+sudo -u deployer git fetch --depth=1 "file://$REPO" "$SHA"
+sudo -u deployer --preserve-env=GIT_CONFIG_GLOBAL git checkout "$SHA"
+guix system describe
+
+if [ "$DRY_RUN" = true ]; then
+ sudo -i guix system -c$NPROC -v3 build "$PWD"/src/infrastructure/guix/system.scm
+else
+ # COMMENT: pre-receive is always running the previous version!
+ # The same is true for the reconfigure script itself.
+ sudo cp description "$REPO"/description
+ sudo cp src/infrastructure/ci/git-pre-receive.sh "$REPO"/hooks/pre-receive
+ sudo cp src/infrastructure/guix/channels.scm /etc/guix/
+ sudo cp src/infrastructure/guix/system.scm /etc/guix/
+
+ sudo -i guix system -c$NPROC -v3 reconfigure /etc/guix/system.scm
+
+ deploy
+fi
diff --git a/src/infrastructure/scripts/report.sh b/src/infrastructure/scripts/report.sh
new file mode 100755
index 0000000..8b3d3e3
--- /dev/null
+++ b/src/infrastructure/scripts/report.sh
@@ -0,0 +1,221 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ cat <<-'EOF'
+ Usage:
+ report [-C REPO] -o DIRECTORY
+ report -h
+ EOF
+}
+
+help() {
+ cat <<-'EOF'
+
+ Options:
+ -C REPO change to REPO when doing Git operations (default: $PWD)
+ -o DIRECTORY the directory where to place the generated files
+ -h, --help show this message
+
+
+ Gather data from Git Notes, and generate an HTML report on CI runs.
+
+ Two refs with notes are expected:
+ 1. refs/notes/ci-data: contains metadata abount the CI runs,
+ with timestamps, filenames and exit status;
+ 2. refs/notes/ci-logs: contains the content of the log.
+
+ When reconstructing the CI run, the $FILENAME present in
+ the refs/notes/ci-data ref names the file, and its content comes
+ from refs/notes/ci-logs.
+
+ On a CI run that generated the numbers from 1 to 10, for a file named
+ 'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the
+ expected output on the target directory "public" is:
+
+ $ tree public/
+ public/
+ index.html
+ data/
+ my-ci-run-2020-01-01-deadbeef.log
+ ...
+ logs/
+ my-ci-run-2020-01-01-deadbeef.log
+ ...
+
+ $ cat public/data/my-ci-run-2020-01-01-deadbeef.log
+ 0 deadbeef my-ci-run-2020-01-01-deadbeef.log
+
+ $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log
+ 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+ 10
+
+ The generated 'index.html' is a webpage with the list of all known
+ CI runs, their status, a link to the commit and a link to the
+ log file.
+
+ To enable fetching these refs by default, do so in the git config:
+
+ $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*'
+
+
+ Examples:
+
+ Generate the report on the 'www' directory:
+
+ $ report -o www
+ EOF
+}
+
+
+for flag in "$@"; do
+ case "$flag" in
+ --)
+ break
+ ;;
+ --help)
+ usage
+ help
+ exit
+ ;;
+ *)
+ ;;
+ esac
+done
+
+REPO="$PWD"
+while getopts 'C:o:h' flag; do
+ case "$flag" in
+ C)
+ REPO="$OPTARG"
+ ;;
+ o)
+ OUTDIR="$OPTARG"
+ ;;
+ h)
+ usage
+ help
+ exit
+ ;;
+ *)
+ exit 2
+ ;;
+ esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${OUTDIR:-}" ]; then
+ printf 'Missing -o OUTDIR.\n\n' >&2
+ usage >&2
+ exit 2
+fi
+
+
+esc() {
+ sed \
+ -e 's|&|\&amp;|g' \
+ -e 's|<|\&lt;|g' \
+ -e 's|>|\&gt;|g' \
+ -e 's|"|\&quot;|g' \
+ -e "s|'|\&#39;|g"
+}
+
+mkdir -p "$OUTDIR"
+cd "$OUTDIR"
+mkdir -p logs data
+
+for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do
+ git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp
+ FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)"
+ mv data/FILENAME-tmp data/"$FILENAME"
+ git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME"
+done
+
+{
+ cat <<-EOF
+ <!DOCTYPE html>
+ <html lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
+ <meta name="description" content="CI logs for servers" />
+ <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
+ <title>servers - CI logs</title>
+ <style>
+ body {
+ max-width: 800px;
+ margin: 0 auto 0 auto;
+ }
+
+ code {
+ display: block;
+ margin: 1em 0em 3em 3em;
+ overflow: auto;
+ }
+
+ pre {
+ display: inline;
+ }
+
+ ol {
+ list-style-type: disc;
+ }
+ </style>
+ </head>
+ <body>
+ <main>
+ <h1>
+ CI logs for
+ <a href="https://euandre.org/git/servers/">servers</a>
+ </h1>
+ <ol>
+ EOF
+
+
+ PASS='&#x2705;' # ✅
+ WARN='&#x1F40C;' # 🐌
+ FAIL='&#x274C;' # ❌
+ for f in $(find data/ -type f | LANG=C.UTF-8 sort -r); do
+ STATUS="$( grep '^status ' "$f" | cut -d' ' -f2- | esc)"
+ SHA="$( grep '^sha ' "$f" | cut -d' ' -f2- | esc)"
+ FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)"
+ DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)"
+ MESSAGE="$(git -C "$REPO" log -1 --format=%B "$SHA" | esc)"
+
+ if [ "$STATUS" = 0 ]; then
+ if [ "$DURATION" -le 60 ]; then
+ STATUS_MARKER="$PASS"
+ else
+ STATUS_MARKER="$WARN"
+ fi
+ else
+ STATUS_MARKER="$FAIL"
+ fi
+
+ cat <<-EOF
+ <li id="$FILENAME">
+ <a href="#$FILENAME"><pre>#</pre></a>
+ $STATUS_MARKER - <pre>${DURATION:-?}s</pre>
+ <pre>(<a href="https://euandre.org/git/servers/commit/?id=$SHA">commit</a>)</pre>
+ <a href="logs/$FILENAME"><pre>$FILENAME</pre></a>
+ <br />
+ <code><pre>$MESSAGE</pre></code>
+ </li>
+ EOF
+ done
+
+ cat <<-EOF
+ </ol>
+ </main>
+ </body>
+ </html>
+ EOF
+} > index.html