Copy files back

16 files changed, 1231 insertions, 2 deletions

diff --git a/src/infrastructure/ci/git-post-receive.sh b/src/infrastructure/ci/git-post-receive.sh
new file mode 100755
index 0000000..68d4da2
--- /dev/null
+++ b/src/infrastructure/ci/git-post-receive.sh
@@ -0,0 +1,171 @@
+#!/bin/sh
+# shellcheck source=/dev/null disable=2317
+. /etc/rc
+set -eu
+
+
+# shellcheck disable=2034
+read -r _oldrev SHA REFNAME
+
+if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then
+	exit
+fi
+
+
+SKIP_DEPLOY=false
+for n in $(seq 0 $((GIT_PUSH_OPTION_COUNT - 1))); do
+	opt="$(eval "printf '%s' \"\$GIT_PUSH_OPTION_$n\"")"
+	case "$opt" in
+		ci.skip)
+			cat <<-EOF
+
+				"$opt" option detected, not running CI.
+
+			EOF
+			exit
+			;;
+		deploy.skip)
+			SKIP_DEPLOY=true
+			;;
+		*)
+			;;
+	esac
+done
+
+
+now() {
+	date '+%Y-%m-%dT%H:%M:%S%:z'
+}
+
+LOGS_DIR=/var/log/ci/servers/
+TIMESTAMP="$(now)"
+FILENAME="$TIMESTAMP-$SHA.log"
+LOGFILE="$LOGS_DIR/$FILENAME"
+mkdir -p "$LOGS_DIR"
+
+
+END_MARKER='\033[0m'
+LIGHT_BLUE_B='\033[1;36m'
+YELLOW='\033[1;33m'
+
+blue() {
+	printf "${LIGHT_BLUE_B}%s${END_MARKER}" "$1"
+}
+
+yellow() {
+	printf "${YELLOW}%s${END_MARKER}" "$1"
+}
+
+info() {
+	sed "s|^\(.\)|$(blue 'CI'): \1|"
+}
+
+
+uuid() {
+	od -xN20 /dev/urandom |
+		head -n1 |
+		awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}'
+}
+
+tmpname() {
+	printf '%s/uuid-tmpname with spaces.%s' "${TMPDIR:-/tmp}" "$(uuid)"
+}
+
+mkdtemp() {
+	name="$(tmpname)"
+	mkdir -- "$name"
+	printf '%s' "$name"
+}
+
+
+{
+	cat <<-EOF | info
+		Starting CI job at: $(now)
+	EOF
+	START="$(date +%s)"
+
+	duration() {
+		if [ "$RUN_DURATION" -gt 60 ]; then
+			cat <<-EOF
+				$(yellow 'WARNING'): run took more than 1 minute! ($RUN_DURATION seconds)
+			EOF
+		else
+			cat <<-EOF
+				Run took $RUN_DURATION seconds.
+			EOF
+		fi
+	}
+
+	finish() {
+		STATUS="$?"
+		END="$(date +%s)"
+		RUN_DURATION=$((END - START))
+		cat <<-EOF | info
+			Finishing CI job at: $(now)
+			Exit status was $STATUS
+			Re-run with:
+			  \$ $CMD
+			$(duration)
+		EOF
+
+		NOTE="$(
+			cat <<-EOF
+				See CI logs with:
+				  git notes --ref=refs/notes/ci-logs show $SHA
+				  git notes --ref=refs/notes/ci-data show $SHA
+
+				Exit status: $STATUS
+				Duration: $RUN_DURATION
+			EOF
+		)"
+		git notes --ref=refs/notes/ci-data add -f -m "$(
+			cat <<-EOF
+				status $STATUS
+				sha $SHA
+				filename $FILENAME
+				duration $RUN_DURATION
+				timestamp $TIMESTAMP
+				to-prod $TO_PROD
+				refname $REFNAME
+			EOF
+		)" "$SHA"
+		git notes --ref=refs/notes/ci-logs add -f -F "$LOGFILE" "$SHA"
+		git notes add -f -m "$NOTE" "$SHA"
+
+		{
+			DIR="$(mkdtemp)"
+			report -o "$DIR"
+			sudo -u deployer rsync \
+				--chmod=D775,F664         \
+				--chown=deployer:deployer \
+				--delete                  \
+				-a                        \
+				"$DIR"/ /srv/www/dev/ci/
+			rm -rf "$DIR"
+		} 1>/dev/null 2>&1 &
+	}
+	trap finish EXIT
+
+	unset GIT_DIR
+
+	if [ "$REFNAME" = 'refs/heads/main' ] && [ "$SKIP_DEPLOY" = false ]; then
+		cat <<-EOF | info
+			In branch "main", running deploy for $SHA.
+		EOF
+		TO_PROD=true
+		CMD="sudo reconfigure $SHA"
+	else
+		if [ "$SKIP_DEPLOY" = true ]; then
+			cat <<-EOF | info
+				"deploy.skip" option detected, skipping deploy for $SHA.
+			EOF
+		else
+			cat <<-EOF | info
+				Not on branch "main", skipping deploy for $SHA.
+			EOF
+		fi
+		TO_PROD=false
+		CMD="sudo reconfigure -n $SHA"
+	fi
+	$CMD
+} 2>&1 | ts -s '%.s' | tee "$LOGFILE"
diff --git a/src/infrastructure/ci/git-pre-receive.sh b/src/infrastructure/ci/git-pre-receive.sh
new file mode 100755
index 0000000..8cd83ee
--- /dev/null
+++ b/src/infrastructure/ci/git-pre-receive.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+
+read -r _oldrev SHA _refname
+unset GIT_DIR
+
+if [ "$SHA" = '0000000000000000000000000000000000000000' ]; then
+	exit
+fi
+
+printf 'Upgrading post-receive hook...' >&2
+git show "$SHA":src/infrastructure/ci/git-post-receive.sh > hooks/post-receive
+chmod +x hooks/post-receive
+printf 'done.\n' >&2
diff --git a/src/infrastructure/config/gitconfig b/src/infrastructure/config/gitconfig
new file mode 100644
index 0000000..915ee72
--- /dev/null
+++ b/src/infrastructure/config/gitconfig
@@ -0,0 +1,7 @@
+[init]
+  defaultBranch = main
+[user]
+  email = ci@euandre.org
+  name = "euandre.org CI"
+[advice]
+  detachedHead = false
diff --git a/src/infrastructure/config/init.scm b/src/infrastructure/config/init.scm
new file mode 100644
index 0000000..9e962e8
--- /dev/null
+++ b/src/infrastructure/config/init.scm
@@ -0,0 +1,6 @@
+(use-modules
+  (ice-9 colorized)
+  (ice-9 readline))
+
+(activate-colorized)
+(activate-readline)
diff --git a/src/infrastructure/config/profile.sh b/src/infrastructure/config/profile.sh
new file mode 100644
index 0000000..1dca8b2
--- /dev/null
+++ b/src/infrastructure/config/profile.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# shellcheck source=/dev/null
+. /etc/rc
+ln -fs .profile .bashrc
diff --git a/src/infrastructure/config/rc.sh b/src/infrastructure/config/rc.sh
new file mode 100644
index 0000000..c92df73
--- /dev/null
+++ b/src/infrastructure/config/rc.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+
+# shellcheck source=/dev/null
+. /etc/profile
+
+export XDG_PREFIX=~/.usr
+export XDG_CACHE_HOME="$XDG_PREFIX"/var/cache
+export XDG_CONFIG_HOME="$XDG_PREFIX"/etc
+export XDG_DATA_HOME="$XDG_PREFIX"/share
+export XDG_STATE_HOME="$XDG_PREFIX"/state
+export XDG_LOG_HOME="$XDG_PREFIX"/var/log
+
+mkdir -p \
+	"$XDG_CONFIG_HOME" \
+	"$XDG_CACHE_HOME"  \
+	"$XDG_DATA_HOME"   \
+	"$XDG_LOG_HOME"    \
+	"$XDG_STATE_HOME"/ssh/conn
+
+
+GUIX_PROFILE="$XDG_CONFIG_HOME"/guix/current
+if [ -r "$GUIX_PROFILE"/etc/profile ]; then
+	# shellcheck source=/dev/null
+	. "$GUIX_PROFILE"/etc/profile
+fi
+
+export ENV=~/.profile
+export HISTSIZE=-1
+export HISTCONTROL=ignorespace:ignoredups
+export EDITOR=vi
+export VISUAL="$EDITOR"
+export PAGER='less -R'
+
+export EXINIT='
+	" set number
+	" set autoindent
+	set ruler
+	set showmode
+	set showmatch
+'
+
+export HISTFILE="$XDG_STATE_HOME"/bash-history
+export LESSHISTFILE="$XDG_STATE_HOME"/lesshst
+export RLWRAP_HOME="$XDG_CACHE_HOME"/rlwrap
+export GUILE_HISTORY="$XDG_STATE_HOME"/guile-history
+
+HOSTNAME="$(hostname)"
+export BORG_REPO="20931@hk-s020.rsync.net:borg/$HOSTNAME"
+export BORG_REMOTE_PATH='borg1'
+export BORG_PASSCOMMAND='cat /opt/secrets/borg-passphrase.txt'
+
+export GIT_CONFIG_GLOBAL=/etc/gitconfig
+
+unalias -a
+alias l='ls -lahF --color'
+alias grep='grep --color=auto'
+alias diff='diff --color=auto'
+alias watch='watch --color '
+alias man='MANWIDTH=$((COLUMNS > 80 ? 80 : COLUMNS)) man'
+alias less='less -R'
+alias tree='tree -aC'
+alias mv='mv -i'
+alias e='vi'
+
+alias sqlite='rlwrap sqlite3'
+alias guile='guile -l /etc/init.scm'
+
+error_marker() {
+	STATUS=$?
+	if [ "$STATUS" != 0 ]; then
+		printf ' (!! %s !!) ' "$STATUS"
+	fi
+}
+export PS1='`error_marker`\T \w/
+\u@\H\$ '
diff --git a/src/infrastructure/config/ssh.conf b/src/infrastructure/config/ssh.conf
new file mode 100644
index 0000000..ca41df0
--- /dev/null
+++ b/src/infrastructure/config/ssh.conf
@@ -0,0 +1,6 @@
+Host *
+  ServerAliveInterval 30
+  ServerAliveCountMax 20
+  ControlMaster auto
+  ControlPath ${XDG_STATE_HOME}/ssh/conn/%r@%h:%p
+  ControlPersist 1h
diff --git a/src/infrastructure/guix/system.scm b/src/infrastructure/guix/system.scm
index 7891f74..322d5b6 100644
--- a/src/infrastructure/guix/system.scm
+++ b/src/infrastructure/guix/system.scm
@@ -1,6 +1,6 @@
 (use-modules
   ((guix licenses) #:prefix license:)
-  ((srfi srfi-1) #:prefix srfi-1:)
+  ((srfi srfi-1) #:prefix s1:)
   ((xyz euandreh heredoc) #:prefix heredoc:)
   (gnu)
   (gnu build linux-container)
@@ -17,6 +17,7 @@
   (gnu packages tls)
   (gnu packages version-control)
   (gnu system setuid)
+  (guix build utils)
   (guix build-system gnu)
   (guix build-system trivial)
   (guix download)
@@ -26,9 +27,10 @@
   (guix records)
   (guix utils))
 (use-package-modules
-  lua
+  ssh
   web)
 (use-service-modules
+  admin
   certbot
   cgit
   mcron
diff --git a/src/infrastructure/keys/SSH/EuAndreh.pub b/src/infrastructure/keys/andreh.pub
index bfd5e6f..bfd5e6f 100644
--- a/src/infrastructure/keys/SSH/EuAndreh.pub
+++ b/src/infrastructure/keys/andreh.pub
diff --git a/src/infrastructure/scripts/backup.sh b/src/infrastructure/scripts/backup.sh
new file mode 100755
index 0000000..47cc76c
--- /dev/null
+++ b/src/infrastructure/scripts/backup.sh
@@ -0,0 +1,135 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  backup [-q] [-C COMMENT] [ARCHIVE_TAG]
+		  backup -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -q            disable verbose mode, useful for batch sessions
+		  -C COMMENT    the comment text to be attached to the archive
+		  -h, --help    show this message
+
+		  ARCHIVE_TAG    the tag used to create the new
+		                 backup (default: "manual")
+
+
+		The repository is expected to have been create with:
+
+		  $ borg init -e repokey-blake2
+
+		The following environment variables are expected to be exported:
+
+		  $BORG_PASSCOMMAND
+		  $BORG_REPO
+		  $BORG_REMOTE_PATH
+
+		Password-less SSH access is required, usually done via adding
+		/root/.ssh/id_rsa.pub to the ssh remote's
+		$THE_REMOTE:.ssh/authorized_keys
+
+		Root permission is also required.
+
+
+		Examples:
+
+		  Run backup from cronjob:
+
+		    $ backup -q cronjob
+
+
+		  Create backup with a comment, a tag, and verbose mode active:
+
+		    $ backup -C 'The backup has a comment'
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+VERBOSE_FLAGS='--verbose --progress'
+COMMENT=' '
+while getopts 'qC:h' flag; do
+	case "$flag" in
+		q)
+			VERBOSE_FLAGS=''
+			;;
+		C)
+			COMMENT="$OPTARG"
+			;;
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+ARCHIVE_TAG="${1:-manual}"
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+run() {
+	set -x
+	# shellcheck disable=2086
+	sudo -i borg create \
+		$VERBOSE_FLAGS                               \
+		--comment "$COMMENT"                         \
+		--stats                                      \
+		--compression lzma,9                         \
+		"$BORG_REPO::$(hostname)-{now}-$ARCHIVE_TAG" \
+		/mnt/production/                             \
+		/root/                                       \
+		/home/                                       \
+		/etc/                                        \
+		/var/                                        \
+		/opt/                                        \
+		/srv/
+	STATUS=$?
+	set +x
+
+	if [ "$STATUS" = 0 ]; then
+		return 0
+	elif [ "$STATUS" = 1 ]; then
+		printf 'WARNING, but no ERROR.\n' >&2
+		return 0
+	else
+		return "$STATUS"
+	fi
+}
+
+run
+
+sudo -i borg check --verify-data --verbose "$BORG_REPO"
diff --git a/src/infrastructure/scripts/cronjob.sh b/src/infrastructure/scripts/cronjob.sh
new file mode 100755
index 0000000..4823ac1
--- /dev/null
+++ b/src/infrastructure/scripts/cronjob.sh
@@ -0,0 +1,159 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  cronjob COMMAND...
+		  cronjob -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -h, --help    show this message
+
+		  COMMAND       the command to be executed
+
+
+		Execute the given command, and send the output to email, with
+		special treatment to the status code.  It kills the job it it
+		lasts more than one hour.
+
+		It load the appropriate files, so that the actual cron
+		declaration is smaller.
+
+
+		Examples:
+
+		  Run a backup:
+
+		    $ cronjob backup -q cron
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${1:-}" ]; then
+	printf 'Missing COMMAND.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+
+
+now() {
+	date '+%Y-%m-%dT%H:%M:%S%:z'
+}
+
+
+uuid() {
+	od -xN20 /dev/random |
+		head -n1 |
+		awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}'
+}
+
+mkstemp() {
+	name="${TMPDIR:-/tmp}/uuid-tmpname with spaces.$(uuid)"
+	touch "$name"
+	printf '%s' "$name"
+}
+
+pre() {
+	sed -u "s|^|[$CMD]: |"
+}
+
+duration() {
+	minutes=$((${1} / 60))
+	seconds=$((${1} % 60))
+	printf '%sm%ss' "$minutes" "$seconds"
+}
+
+
+CMD="$*"
+HOSTNAME="$(hostname)"
+FROM="cronjob@$HOSTNAME"
+ONE_HOUR='3600'
+STATUS_F="$(mkstemp)"
+OUT="$(mkstemp)"
+
+email() {
+	{
+		cat <<-EOF
+			Content-Type: text/plain; charset=UTF-8
+			Content-Transfer-Encoding: 8bit
+			From: $FROM
+			To: root@localhost
+			Subject: (exit status: $(cat "$STATUS_F")) - $HOSTNAME: $CMD
+
+		EOF
+		cat "$OUT"
+	} | sendmail -t -f "$FROM"
+	rm -f "$OUT" "$STATUS_F"
+}
+trap email EXIT
+
+{
+	cat <<-EOF
+		Running commad: $*
+		Starting at: $(now)
+
+	EOF
+
+	START="$(date +%s)"
+	STATUS=0
+	timeout "$ONE_HOUR" "$@" || STATUS=$?
+	printf '%s' "$STATUS" > "$STATUS_F"
+	END="$(date +%s)"
+	DURATION_SECONDS=$((END - START))
+
+	cat <<-EOF
+
+		Finished at: $(now)
+		Duration: $(duration "$DURATION_SECONDS")
+	EOF
+} 2>&1 | pre | ts '%Y-%m-%dT%H:%M:%S' | tee "$OUT" >> /var/log/cronjobs.log
diff --git a/src/infrastructure/scripts/deploy.sh b/src/infrastructure/scripts/deploy.sh
new file mode 100755
index 0000000..65a50c1
--- /dev/null
+++ b/src/infrastructure/scripts/deploy.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  deploy
+		  deploy -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+		Options:
+		  -h, --help    show this message
+
+
+		Do a blue/green deployment of the relevant service.  It makes
+		sure that the new service is up and running before shutting
+		down the old one.
+
+
+		Examples:
+
+		  Just do the deploy:
+
+		    $ deploy
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+: sudo herd restart a-service
diff --git a/src/infrastructure/scripts/gc.sh b/src/infrastructure/scripts/gc.sh
new file mode 100755
index 0000000..0eca4be
--- /dev/null
+++ b/src/infrastructure/scripts/gc.sh
@@ -0,0 +1,146 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  gc [TYPE]
+		  gc -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -h, --help    show this message
+
+		  TYPE          what to do GC on (default: all):
+		                - guix
+		                - deploy
+		                - trash
+		                - tmpdir
+		                - logs
+
+
+		GC the server, deleting old, unusable data, in order to free
+		disk space system-wide.
+
+
+		Examples:
+
+		  Just run it, for all:
+
+		    $ gc
+
+
+		  Cleanup tmpdir:
+
+		    $ gc tmpdir
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+disk() {
+	df -h / /mnt/backup/ |
+		tail -n +2 |
+		awk '{ printf "%s\t%s/%s\t%s\n", $4, $3, $2, $6 }'
+}
+
+today() {
+	date '+%Y-%m-%d'
+}
+
+gc_guix() {
+	sudo -i guix system delete-generations
+	sudo -i guix gc -d
+}
+
+gc_deploy() {
+	sudo -u deployer find /opt/deploy \
+		! -path /opt/deploy -prune \
+		-type d \
+		-not -name "$(today)*" \
+		-exec rm -rf "{}" ';'
+}
+
+gc_trash() {
+	yes | sudo -i trash-empty
+}
+
+gc_tmpdir() {
+	find "${TMPDIR:-/tmp}"    -atime +10 -exec rm -vf "{}" ';'
+}
+
+gc_logs() {
+	find /var/log/ci/ -atime +10 -exec rm -vf "{}" ';'
+}
+
+
+gc_all() {
+	gc_guix
+	gc_deploy
+	gc_trash
+	gc_tmpdir
+	gc_logs
+}
+
+
+TYPE="${1:-all}"
+CMD=gc_"$TYPE"
+if ! command -v "$CMD" >/dev/null; then
+	printf 'Invalid TYPE: "%s".\n\n' "$TYPE" >&2
+	usage >&2
+	exit 2
+fi
+
+BEFORE="$(disk)"
+set -x
+"$CMD"
+set +x
+AFTER="$(disk)"
+
+cat <<-EOF
+	Disk space:
+	  before: $BEFORE
+	  after:  $AFTER
+EOF
diff --git a/src/infrastructure/scripts/r.sh b/src/infrastructure/scripts/r.sh
new file mode 100755
index 0000000..8e74576
--- /dev/null
+++ b/src/infrastructure/scripts/r.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  r COMMAND...
+		  r -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -h, --help    show this message
+
+		  COMMAND       the command to be executed
+
+
+		Execute the given command, with a proper login environment
+		loaded.
+
+
+		Examples:
+
+		  Run a backup via SSH:
+
+		    $ ssh euandre.org r backup -q cron
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${1:-}" ]; then
+	printf 'Missing COMMAND.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+exec "$@"
diff --git a/src/infrastructure/scripts/reconfigure.sh b/src/infrastructure/scripts/reconfigure.sh
new file mode 100755
index 0000000..c76ea3e
--- /dev/null
+++ b/src/infrastructure/scripts/reconfigure.sh
@@ -0,0 +1,134 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  reconfigure [-n] [-U] [SHA]
+		  reconfigure -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -n            build the system, but don't switch to it (dry-run)
+		  -U            pull the latest channels before reconfiguring
+		  -h, --help    show this message
+
+		  SHA           the repository SHA to checkout (default: main)
+
+
+		Run a "guix system reconfigure" as root via "sudo -i".  If a -U
+		flag is given, perform a "guix pull" (in root profile) prior to
+		the reconfigure.  The user must be able to become the "deployer"
+		user, either via "sudo reconfigure" or by being member of the
+		"become-deployer" group.
+
+
+		Examples:
+
+		  Reconfigure the system:
+
+		    $ reconfigure
+
+
+		  Build the system on a custom SHA, but don't switch to it:
+
+		    $ reconfigure -n 916dafc092f797349a54515756f2c8e477326511
+
+
+		  Update and upgrade:
+
+		    $ reconfigure -U
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+UPDATE=false
+DRY_RUN=false
+while getopts 'nUh' flag; do
+	case "$flag" in
+		n)
+			DRY_RUN=true
+			;;
+		U)
+			UPDATE=true
+			;;
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+SHA="${1:-main}"
+REPO='/srv/git/servers.git'
+NOW="$(date '+%Y-%m-%dT%H:%M:%S%:z')"
+NOW_DIR=/opt/deploy/"$NOW"
+NPROC=$(($(nproc) * 2 + 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+
+if [ "$UPDATE" = true ] && [ "$DRY_RUN" = false ]; then
+	sudo -i guix pull -v3
+fi
+
+set -x
+sudo -u deployer git clone --depth=1 "file://$REPO" "$NOW_DIR"
+sudo -u deployer rm -f /opt/deploy/current
+sudo -u deployer ln -s "$NOW_DIR" /opt/deploy/current
+cd /opt/deploy/current
+sudo -u deployer git fetch --depth=1 "file://$REPO" "$SHA"
+sudo -u deployer --preserve-env=GIT_CONFIG_GLOBAL git checkout "$SHA"
+guix system describe
+
+if [ "$DRY_RUN" = true ]; then
+	sudo -i guix system -c$NPROC -v3 build "$PWD"/src/infrastructure/guix/system.scm
+else
+	# COMMENT: pre-receive is always running the previous version!
+	#          The same is true for the reconfigure script itself.
+	sudo cp description                              "$REPO"/description
+	sudo cp src/infrastructure/ci/git-pre-receive.sh "$REPO"/hooks/pre-receive
+	sudo cp src/infrastructure/guix/channels.scm /etc/guix/
+	sudo cp src/infrastructure/guix/system.scm   /etc/guix/
+
+	sudo -i guix system -c$NPROC -v3 reconfigure /etc/guix/system.scm
+
+	deploy
+fi
diff --git a/src/infrastructure/scripts/report.sh b/src/infrastructure/scripts/report.sh
new file mode 100755
index 0000000..8b3d3e3
--- /dev/null
+++ b/src/infrastructure/scripts/report.sh
@@ -0,0 +1,221 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  report [-C REPO] -o DIRECTORY
+		  report -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+		Options:
+		  -C REPO         change to REPO when doing Git operations (default: $PWD)
+		  -o DIRECTORY    the directory where to place the generated files
+		  -h, --help      show this message
+
+
+		Gather data from Git Notes, and generate an HTML report on CI runs.
+
+		Two refs with notes are expected:
+		1. refs/notes/ci-data: contains metadata abount the CI runs,
+		   with timestamps, filenames and exit status;
+		2. refs/notes/ci-logs: contains the content of the log.
+
+		When reconstructing the CI run, the $FILENAME present in
+		the refs/notes/ci-data ref names the file, and its content comes
+		from refs/notes/ci-logs.
+
+		On a CI run that generated the numbers from 1 to 10, for a file named
+		'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the
+		expected output on the target directory "public" is:
+
+		  $ tree public/
+		  public/
+		    index.html
+		    data/
+		      my-ci-run-2020-01-01-deadbeef.log
+		      ...
+		    logs/
+		      my-ci-run-2020-01-01-deadbeef.log
+		      ...
+
+		  $ cat public/data/my-ci-run-2020-01-01-deadbeef.log
+		  0 deadbeef my-ci-run-2020-01-01-deadbeef.log
+
+		  $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log
+		  1
+		  2
+		  3
+		  4
+		  5
+		  6
+		  7
+		  8
+		  9
+		  10
+
+		The generated 'index.html' is a webpage with the list of all known
+		CI runs, their status, a link to the commit and a link to the
+		log file.
+
+		To enable fetching these refs by default, do so in the git config:
+
+		  $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*'
+
+
+		Examples:
+
+		  Generate the report on the 'www' directory:
+
+		    $ report -o www
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+REPO="$PWD"
+while getopts 'C:o:h' flag; do
+	case "$flag" in
+		C)
+			REPO="$OPTARG"
+			;;
+		o)
+			OUTDIR="$OPTARG"
+			;;
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${OUTDIR:-}" ]; then
+	printf 'Missing -o OUTDIR.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+esc() {
+	sed \
+		-e 's|&|\&amp;|g'  \
+		-e 's|<|\&lt;|g'   \
+		-e 's|>|\&gt;|g'   \
+		-e 's|"|\&quot;|g' \
+		-e "s|'|\&#39;|g"
+}
+
+mkdir -p "$OUTDIR"
+cd "$OUTDIR"
+mkdir -p logs data
+
+for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do
+	git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp
+	FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)"
+	mv data/FILENAME-tmp data/"$FILENAME"
+	git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME"
+done
+
+{
+	cat <<-EOF
+		<!DOCTYPE html>
+		<html lang="en">
+		  <head>
+		    <meta charset="UTF-8" />
+		    <meta name="viewport"    content="width=device-width, initial-scale=1" />
+		    <meta name="description" content="CI logs for servers" />
+		    <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
+		    <title>servers - CI logs</title>
+		    <style>
+		      body {
+		        max-width: 800px;
+		        margin: 0 auto 0 auto;
+		      }
+
+		      code {
+		        display: block;
+		        margin: 1em 0em 3em 3em;
+		        overflow: auto;
+		      }
+
+		      pre {
+		        display: inline;
+		      }
+
+		      ol {
+		        list-style-type: disc;
+		      }
+		    </style>
+		  </head>
+		  <body>
+		    <main>
+		      <h1>
+		        CI logs for
+		        <a href="https://euandre.org/git/servers/">servers</a>
+		      </h1>
+		      <ol>
+	EOF
+
+
+	PASS='&#x2705;'  # ✅
+	WARN='&#x1F40C;' # 🐌
+	FAIL='&#x274C;'  # ❌
+	for f in $(find data/ -type f | LANG=C.UTF-8 sort -r); do
+		STATUS="$(  grep '^status '   "$f" | cut -d' ' -f2- | esc)"
+		SHA="$(     grep '^sha '      "$f" | cut -d' ' -f2- | esc)"
+		FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)"
+		DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)"
+		MESSAGE="$(git -C "$REPO" log -1 --format=%B "$SHA" | esc)"
+
+		if [ "$STATUS" = 0 ]; then
+			if [ "$DURATION" -le 60 ]; then
+				STATUS_MARKER="$PASS"
+			else
+				STATUS_MARKER="$WARN"
+			fi
+		else
+			STATUS_MARKER="$FAIL"
+		fi
+
+		cat <<-EOF
+			        <li id="$FILENAME">
+			          <a href="#$FILENAME"><pre>#</pre></a>
+			          $STATUS_MARKER - <pre>${DURATION:-?}s</pre>
+			          <pre>(<a href="https://euandre.org/git/servers/commit/?id=$SHA">commit</a>)</pre>
+			          <a href="logs/$FILENAME"><pre>$FILENAME</pre></a>
+			          <br />
+			          <code><pre>$MESSAGE</pre></code>
+			        </li>
+		EOF
+	done
+
+	cat <<-EOF
+		      </ol>
+		    </main>
+		  </body>
+		</html>
+	EOF
+} > index.html