Copy files back

7 files changed, 943 insertions, 0 deletions

diff --git a/src/infrastructure/scripts/backup.sh b/src/infrastructure/scripts/backup.sh
new file mode 100755
index 0000000..47cc76c
--- /dev/null
+++ b/src/infrastructure/scripts/backup.sh
@@ -0,0 +1,135 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  backup [-q] [-C COMMENT] [ARCHIVE_TAG]
+		  backup -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -q            disable verbose mode, useful for batch sessions
+		  -C COMMENT    the comment text to be attached to the archive
+		  -h, --help    show this message
+
+		  ARCHIVE_TAG    the tag used to create the new
+		                 backup (default: "manual")
+
+
+		The repository is expected to have been create with:
+
+		  $ borg init -e repokey-blake2
+
+		The following environment variables are expected to be exported:
+
+		  $BORG_PASSCOMMAND
+		  $BORG_REPO
+		  $BORG_REMOTE_PATH
+
+		Password-less SSH access is required, usually done via adding
+		/root/.ssh/id_rsa.pub to the ssh remote's
+		$THE_REMOTE:.ssh/authorized_keys
+
+		Root permission is also required.
+
+
+		Examples:
+
+		  Run backup from cronjob:
+
+		    $ backup -q cronjob
+
+
+		  Create backup with a comment, a tag, and verbose mode active:
+
+		    $ backup -C 'The backup has a comment'
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+VERBOSE_FLAGS='--verbose --progress'
+COMMENT=' '
+while getopts 'qC:h' flag; do
+	case "$flag" in
+		q)
+			VERBOSE_FLAGS=''
+			;;
+		C)
+			COMMENT="$OPTARG"
+			;;
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+ARCHIVE_TAG="${1:-manual}"
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+run() {
+	set -x
+	# shellcheck disable=2086
+	sudo -i borg create \
+		$VERBOSE_FLAGS                               \
+		--comment "$COMMENT"                         \
+		--stats                                      \
+		--compression lzma,9                         \
+		"$BORG_REPO::$(hostname)-{now}-$ARCHIVE_TAG" \
+		/mnt/production/                             \
+		/root/                                       \
+		/home/                                       \
+		/etc/                                        \
+		/var/                                        \
+		/opt/                                        \
+		/srv/
+	STATUS=$?
+	set +x
+
+	if [ "$STATUS" = 0 ]; then
+		return 0
+	elif [ "$STATUS" = 1 ]; then
+		printf 'WARNING, but no ERROR.\n' >&2
+		return 0
+	else
+		return "$STATUS"
+	fi
+}
+
+run
+
+sudo -i borg check --verify-data --verbose "$BORG_REPO"
diff --git a/src/infrastructure/scripts/cronjob.sh b/src/infrastructure/scripts/cronjob.sh
new file mode 100755
index 0000000..4823ac1
--- /dev/null
+++ b/src/infrastructure/scripts/cronjob.sh
@@ -0,0 +1,159 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  cronjob COMMAND...
+		  cronjob -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -h, --help    show this message
+
+		  COMMAND       the command to be executed
+
+
+		Execute the given command, and send the output to email, with
+		special treatment to the status code.  It kills the job it it
+		lasts more than one hour.
+
+		It load the appropriate files, so that the actual cron
+		declaration is smaller.
+
+
+		Examples:
+
+		  Run a backup:
+
+		    $ cronjob backup -q cron
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${1:-}" ]; then
+	printf 'Missing COMMAND.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+
+
+now() {
+	date '+%Y-%m-%dT%H:%M:%S%:z'
+}
+
+
+uuid() {
+	od -xN20 /dev/random |
+		head -n1 |
+		awk '{OFS="-"; print $2$3,$4,$5,$6,$7$8$9}'
+}
+
+mkstemp() {
+	name="${TMPDIR:-/tmp}/uuid-tmpname with spaces.$(uuid)"
+	touch "$name"
+	printf '%s' "$name"
+}
+
+pre() {
+	sed -u "s|^|[$CMD]: |"
+}
+
+duration() {
+	minutes=$((${1} / 60))
+	seconds=$((${1} % 60))
+	printf '%sm%ss' "$minutes" "$seconds"
+}
+
+
+CMD="$*"
+HOSTNAME="$(hostname)"
+FROM="cronjob@$HOSTNAME"
+ONE_HOUR='3600'
+STATUS_F="$(mkstemp)"
+OUT="$(mkstemp)"
+
+email() {
+	{
+		cat <<-EOF
+			Content-Type: text/plain; charset=UTF-8
+			Content-Transfer-Encoding: 8bit
+			From: $FROM
+			To: root@localhost
+			Subject: (exit status: $(cat "$STATUS_F")) - $HOSTNAME: $CMD
+
+		EOF
+		cat "$OUT"
+	} | sendmail -t -f "$FROM"
+	rm -f "$OUT" "$STATUS_F"
+}
+trap email EXIT
+
+{
+	cat <<-EOF
+		Running commad: $*
+		Starting at: $(now)
+
+	EOF
+
+	START="$(date +%s)"
+	STATUS=0
+	timeout "$ONE_HOUR" "$@" || STATUS=$?
+	printf '%s' "$STATUS" > "$STATUS_F"
+	END="$(date +%s)"
+	DURATION_SECONDS=$((END - START))
+
+	cat <<-EOF
+
+		Finished at: $(now)
+		Duration: $(duration "$DURATION_SECONDS")
+	EOF
+} 2>&1 | pre | ts '%Y-%m-%dT%H:%M:%S' | tee "$OUT" >> /var/log/cronjobs.log
diff --git a/src/infrastructure/scripts/deploy.sh b/src/infrastructure/scripts/deploy.sh
new file mode 100755
index 0000000..65a50c1
--- /dev/null
+++ b/src/infrastructure/scripts/deploy.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  deploy
+		  deploy -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+		Options:
+		  -h, --help    show this message
+
+
+		Do a blue/green deployment of the relevant service.  It makes
+		sure that the new service is up and running before shutting
+		down the old one.
+
+
+		Examples:
+
+		  Just do the deploy:
+
+		    $ deploy
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+: sudo herd restart a-service
diff --git a/src/infrastructure/scripts/gc.sh b/src/infrastructure/scripts/gc.sh
new file mode 100755
index 0000000..0eca4be
--- /dev/null
+++ b/src/infrastructure/scripts/gc.sh
@@ -0,0 +1,146 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  gc [TYPE]
+		  gc -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -h, --help    show this message
+
+		  TYPE          what to do GC on (default: all):
+		                - guix
+		                - deploy
+		                - trash
+		                - tmpdir
+		                - logs
+
+
+		GC the server, deleting old, unusable data, in order to free
+		disk space system-wide.
+
+
+		Examples:
+
+		  Just run it, for all:
+
+		    $ gc
+
+
+		  Cleanup tmpdir:
+
+		    $ gc tmpdir
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+disk() {
+	df -h / /mnt/backup/ |
+		tail -n +2 |
+		awk '{ printf "%s\t%s/%s\t%s\n", $4, $3, $2, $6 }'
+}
+
+today() {
+	date '+%Y-%m-%d'
+}
+
+gc_guix() {
+	sudo -i guix system delete-generations
+	sudo -i guix gc -d
+}
+
+gc_deploy() {
+	sudo -u deployer find /opt/deploy \
+		! -path /opt/deploy -prune \
+		-type d \
+		-not -name "$(today)*" \
+		-exec rm -rf "{}" ';'
+}
+
+gc_trash() {
+	yes | sudo -i trash-empty
+}
+
+gc_tmpdir() {
+	find "${TMPDIR:-/tmp}"    -atime +10 -exec rm -vf "{}" ';'
+}
+
+gc_logs() {
+	find /var/log/ci/ -atime +10 -exec rm -vf "{}" ';'
+}
+
+
+gc_all() {
+	gc_guix
+	gc_deploy
+	gc_trash
+	gc_tmpdir
+	gc_logs
+}
+
+
+TYPE="${1:-all}"
+CMD=gc_"$TYPE"
+if ! command -v "$CMD" >/dev/null; then
+	printf 'Invalid TYPE: "%s".\n\n' "$TYPE" >&2
+	usage >&2
+	exit 2
+fi
+
+BEFORE="$(disk)"
+set -x
+"$CMD"
+set +x
+AFTER="$(disk)"
+
+cat <<-EOF
+	Disk space:
+	  before: $BEFORE
+	  after:  $AFTER
+EOF
diff --git a/src/infrastructure/scripts/r.sh b/src/infrastructure/scripts/r.sh
new file mode 100755
index 0000000..8e74576
--- /dev/null
+++ b/src/infrastructure/scripts/r.sh
@@ -0,0 +1,77 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  r COMMAND...
+		  r -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -h, --help    show this message
+
+		  COMMAND       the command to be executed
+
+
+		Execute the given command, with a proper login environment
+		loaded.
+
+
+		Examples:
+
+		  Run a backup via SSH:
+
+		    $ ssh euandre.org r backup -q cron
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+while getopts 'h' flag; do
+	case "$flag" in
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${1:-}" ]; then
+	printf 'Missing COMMAND.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+exec "$@"
diff --git a/src/infrastructure/scripts/reconfigure.sh b/src/infrastructure/scripts/reconfigure.sh
new file mode 100755
index 0000000..c76ea3e
--- /dev/null
+++ b/src/infrastructure/scripts/reconfigure.sh
@@ -0,0 +1,134 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  reconfigure [-n] [-U] [SHA]
+		  reconfigure -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+
+		Options:
+		  -n            build the system, but don't switch to it (dry-run)
+		  -U            pull the latest channels before reconfiguring
+		  -h, --help    show this message
+
+		  SHA           the repository SHA to checkout (default: main)
+
+
+		Run a "guix system reconfigure" as root via "sudo -i".  If a -U
+		flag is given, perform a "guix pull" (in root profile) prior to
+		the reconfigure.  The user must be able to become the "deployer"
+		user, either via "sudo reconfigure" or by being member of the
+		"become-deployer" group.
+
+
+		Examples:
+
+		  Reconfigure the system:
+
+		    $ reconfigure
+
+
+		  Build the system on a custom SHA, but don't switch to it:
+
+		    $ reconfigure -n 916dafc092f797349a54515756f2c8e477326511
+
+
+		  Update and upgrade:
+
+		    $ reconfigure -U
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+UPDATE=false
+DRY_RUN=false
+while getopts 'nUh' flag; do
+	case "$flag" in
+		n)
+			DRY_RUN=true
+			;;
+		U)
+			UPDATE=true
+			;;
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			usage >&2
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+SHA="${1:-main}"
+REPO='/srv/git/servers.git'
+NOW="$(date '+%Y-%m-%dT%H:%M:%S%:z')"
+NOW_DIR=/opt/deploy/"$NOW"
+NPROC=$(($(nproc) * 2 + 1))
+
+
+if [ "$(id -un)" != 'root' ]; then
+	printf 'This script must be run as root.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+set +eu
+# shellcheck source=/dev/null
+. /etc/rc
+set -eu
+
+
+if [ "$UPDATE" = true ] && [ "$DRY_RUN" = false ]; then
+	sudo -i guix pull -v3
+fi
+
+set -x
+sudo -u deployer git clone --depth=1 "file://$REPO" "$NOW_DIR"
+sudo -u deployer rm -f /opt/deploy/current
+sudo -u deployer ln -s "$NOW_DIR" /opt/deploy/current
+cd /opt/deploy/current
+sudo -u deployer git fetch --depth=1 "file://$REPO" "$SHA"
+sudo -u deployer --preserve-env=GIT_CONFIG_GLOBAL git checkout "$SHA"
+guix system describe
+
+if [ "$DRY_RUN" = true ]; then
+	sudo -i guix system -c$NPROC -v3 build "$PWD"/src/infrastructure/guix/system.scm
+else
+	# COMMENT: pre-receive is always running the previous version!
+	#          The same is true for the reconfigure script itself.
+	sudo cp description                              "$REPO"/description
+	sudo cp src/infrastructure/ci/git-pre-receive.sh "$REPO"/hooks/pre-receive
+	sudo cp src/infrastructure/guix/channels.scm /etc/guix/
+	sudo cp src/infrastructure/guix/system.scm   /etc/guix/
+
+	sudo -i guix system -c$NPROC -v3 reconfigure /etc/guix/system.scm
+
+	deploy
+fi
diff --git a/src/infrastructure/scripts/report.sh b/src/infrastructure/scripts/report.sh
new file mode 100755
index 0000000..8b3d3e3
--- /dev/null
+++ b/src/infrastructure/scripts/report.sh
@@ -0,0 +1,221 @@
+#!/bin/sh
+set -eu
+
+usage() {
+	cat <<-'EOF'
+		Usage:
+		  report [-C REPO] -o DIRECTORY
+		  report -h
+	EOF
+}
+
+help() {
+	cat <<-'EOF'
+
+		Options:
+		  -C REPO         change to REPO when doing Git operations (default: $PWD)
+		  -o DIRECTORY    the directory where to place the generated files
+		  -h, --help      show this message
+
+
+		Gather data from Git Notes, and generate an HTML report on CI runs.
+
+		Two refs with notes are expected:
+		1. refs/notes/ci-data: contains metadata abount the CI runs,
+		   with timestamps, filenames and exit status;
+		2. refs/notes/ci-logs: contains the content of the log.
+
+		When reconstructing the CI run, the $FILENAME present in
+		the refs/notes/ci-data ref names the file, and its content comes
+		from refs/notes/ci-logs.
+
+		On a CI run that generated the numbers from 1 to 10, for a file named
+		'my-ci-run-2020-01-01-deadbeef.log' that exited successfully, the
+		expected output on the target directory "public" is:
+
+		  $ tree public/
+		  public/
+		    index.html
+		    data/
+		      my-ci-run-2020-01-01-deadbeef.log
+		      ...
+		    logs/
+		      my-ci-run-2020-01-01-deadbeef.log
+		      ...
+
+		  $ cat public/data/my-ci-run-2020-01-01-deadbeef.log
+		  0 deadbeef my-ci-run-2020-01-01-deadbeef.log
+
+		  $ cat public/logs/my-ci-run-2020-01-01-deadbeef.log
+		  1
+		  2
+		  3
+		  4
+		  5
+		  6
+		  7
+		  8
+		  9
+		  10
+
+		The generated 'index.html' is a webpage with the list of all known
+		CI runs, their status, a link to the commit and a link to the
+		log file.
+
+		To enable fetching these refs by default, do so in the git config:
+
+		  $ git config --add remote.origin.fetch '+refs/notes/*:refs/notes/*'
+
+
+		Examples:
+
+		  Generate the report on the 'www' directory:
+
+		    $ report -o www
+	EOF
+}
+
+
+for flag in "$@"; do
+	case "$flag" in
+		--)
+			break
+			;;
+		--help)
+			usage
+			help
+			exit
+			;;
+		*)
+			;;
+	esac
+done
+
+REPO="$PWD"
+while getopts 'C:o:h' flag; do
+	case "$flag" in
+		C)
+			REPO="$OPTARG"
+			;;
+		o)
+			OUTDIR="$OPTARG"
+			;;
+		h)
+			usage
+			help
+			exit
+			;;
+		*)
+			exit 2
+			;;
+	esac
+done
+shift $((OPTIND - 1))
+
+if [ -z "${OUTDIR:-}" ]; then
+	printf 'Missing -o OUTDIR.\n\n' >&2
+	usage >&2
+	exit 2
+fi
+
+
+esc() {
+	sed \
+		-e 's|&|\&amp;|g'  \
+		-e 's|<|\&lt;|g'   \
+		-e 's|>|\&gt;|g'   \
+		-e 's|"|\&quot;|g' \
+		-e "s|'|\&#39;|g"
+}
+
+mkdir -p "$OUTDIR"
+cd "$OUTDIR"
+mkdir -p logs data
+
+for c in $(git -C "$REPO" notes list | cut -d' ' -f2); do
+	git -C "$REPO" notes --ref=refs/notes/ci-data show "$c" > data/FILENAME-tmp
+	FILENAME="$(grep '^filename ' data/FILENAME-tmp | cut -d' ' -f2-)"
+	mv data/FILENAME-tmp data/"$FILENAME"
+	git -C "$REPO" notes --ref=refs/notes/ci-logs show "$c" > logs/"$FILENAME"
+done
+
+{
+	cat <<-EOF
+		<!DOCTYPE html>
+		<html lang="en">
+		  <head>
+		    <meta charset="UTF-8" />
+		    <meta name="viewport"    content="width=device-width, initial-scale=1" />
+		    <meta name="description" content="CI logs for servers" />
+		    <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
+		    <title>servers - CI logs</title>
+		    <style>
+		      body {
+		        max-width: 800px;
+		        margin: 0 auto 0 auto;
+		      }
+
+		      code {
+		        display: block;
+		        margin: 1em 0em 3em 3em;
+		        overflow: auto;
+		      }
+
+		      pre {
+		        display: inline;
+		      }
+
+		      ol {
+		        list-style-type: disc;
+		      }
+		    </style>
+		  </head>
+		  <body>
+		    <main>
+		      <h1>
+		        CI logs for
+		        <a href="https://euandre.org/git/servers/">servers</a>
+		      </h1>
+		      <ol>
+	EOF
+
+
+	PASS='&#x2705;'  # ✅
+	WARN='&#x1F40C;' # 🐌
+	FAIL='&#x274C;'  # ❌
+	for f in $(find data/ -type f | LANG=C.UTF-8 sort -r); do
+		STATUS="$(  grep '^status '   "$f" | cut -d' ' -f2- | esc)"
+		SHA="$(     grep '^sha '      "$f" | cut -d' ' -f2- | esc)"
+		FILENAME="$(grep '^filename ' "$f" | cut -d' ' -f2- | esc)"
+		DURATION="$(grep '^duration ' "$f" | cut -d' ' -f2- | cut -d'"' -f1 | esc)"
+		MESSAGE="$(git -C "$REPO" log -1 --format=%B "$SHA" | esc)"
+
+		if [ "$STATUS" = 0 ]; then
+			if [ "$DURATION" -le 60 ]; then
+				STATUS_MARKER="$PASS"
+			else
+				STATUS_MARKER="$WARN"
+			fi
+		else
+			STATUS_MARKER="$FAIL"
+		fi
+
+		cat <<-EOF
+			        <li id="$FILENAME">
+			          <a href="#$FILENAME"><pre>#</pre></a>
+			          $STATUS_MARKER - <pre>${DURATION:-?}s</pre>
+			          <pre>(<a href="https://euandre.org/git/servers/commit/?id=$SHA">commit</a>)</pre>
+			          <a href="logs/$FILENAME"><pre>$FILENAME</pre></a>
+			          <br />
+			          <code><pre>$MESSAGE</pre></code>
+			        </li>
+		EOF
+	done
+
+	cat <<-EOF
+		      </ol>
+		    </main>
+		  </body>
+		</html>
+	EOF
+} > index.html