From da00227813b1fbeebae8c90e2122a8b73acb1af9 Mon Sep 17 00:00:00 2001 From: EuAndreh Date: Sun, 26 May 2019 11:51:51 -0300 Subject: Automate provisioning and deployment of VPS In order to perform that I had to remove Terraform's =.tfstate= files from the repository. Terraform does support "backends" for storing the state files, but I settled for storing it on a separate repo (vps-state). For now it solves the state management problem: - it has history of states; - all state files are GPG encrypted; - there's no coordination however, but only the CI should perform a deploy in order to avoid race conditions. I had to add GPG and SSH keys to sr.ht to achieve that: - SSH public key to my profile to authorize it to push to vps-state repo; - SSH private key to the secret builds.sr.ht environment to enable push to the repository from the pipeline; - GPG public key to git-crypt to make it possible for the pipeline to unlock the encrypted content; - GPG private key to the secret builds.sr.ht environment to enable decrypting git-crypt content from the pipeline. In order to avoid divergent environment from local and CI, the ./provision.sh script is ran through nix-shell. --- .build.yml | 7 +++++++ TODOs.org | 4 ++++ default.nix | 3 ++- deploy.sh | 34 +++++++++++++++++++++++++++------- provision.sh | 36 ++++++++++++++++++++++++++++++++++++ secrets/terraform.tfstate | Bin 2243 -> 0 bytes secrets/terraform.tfstate.backup | Bin 2244 -> 0 bytes terraform.tfstate | 2 +- terraform.tfstate.backup | 2 +- vps.tf | 6 +++++- 10 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 provision.sh delete mode 100644 secrets/terraform.tfstate delete mode 100644 secrets/terraform.tfstate.backup diff --git a/.build.yml b/.build.yml index c8e51f2..6346e2b 100644 --- a/.build.yml +++ b/.build.yml @@ -7,7 +7,14 @@ triggers: to: EuAndreh sources: - https://git.sr.ht/~euandreh/vps + - https://git.sr.ht/~euandreh/vps-state +secrets: + - b1f49116-7515-40ef-857c-42e4519b8472 + - 7084b7c7-12be-4509-8927-81ba6eeb1fc0 tasks: - tests: | cd vps/ nix-build -A test + - deploy: | + cd vps/ + nix-shell --run "bash -c ./provision.sh" diff --git a/TODOs.org b/TODOs.org index 25aa7df..1fa7ee7 100644 --- a/TODOs.org +++ b/TODOs.org @@ -18,9 +18,13 @@ We could try to share a shared volume, but that would be a consistency nightmare The other option is to always recreate everything, with downtime. The advantage is that we get actual immutable deployments with stateful storage, but there would be downtime for every deployment. This is due to the nature of most of the packaged applications being single node *only*. +There's also the IP reputation issue: recreating everything from scratch every time would lead to new droplets with new IP addresses, which is not a good thing to be changing in a server box. + A reasonable alternative would be to redeploy everything on a different node, with a different TLD, and manually check that. But that would be just like an staging environment, with all of it's downsides too. In this situation, I if go on with automating the deployment I'd rather pick the downtime option. + +I'll start with other services other than email and consider alternatives later. ** WAITING Configure DNS from Terraform * Must ** Fully deployable from code diff --git a/default.nix b/default.nix index c7c5bc4..67830e2 100644 --- a/default.nix +++ b/default.nix @@ -95,8 +95,9 @@ with pkgs.stdenv; rec { touch $out ''; }); + # Used in .build.yml to run ./provision.sh shell = mkShell rec { name = "vps-shell"; - buildInputs = [ nixfmt terraform-full ]; + buildInputs = [ terraform terraform-providers.digitalocean git-crypt ]; }; } diff --git a/deploy.sh b/deploy.sh index 3d12517..f96cfe5 100755 --- a/deploy.sh +++ b/deploy.sh @@ -2,22 +2,42 @@ set -Eeuo pipefail cd "${BASH_SOURCE%/*}/" -yellow "Ubuntu maintenence..." +alias ssh="ssh -i secrets/id_rsa root@$TLD" + +apt_wait() { + local i=0 + tput sc + while fuser /var/lib/apt/lists/lock >/dev/null 2>&1 ; do + case $((i % 4)) in + 0 ) j="-" ;; + 1 ) j="\\" ;; + 2 ) j="|" ;; + 3 ) j="/" ;; + esac + tput rc + echo -en "\r[$j] Waiting for other software managers to finish..." + sleep 0.5 + ((i=i+1)) + done +} + +apt_wait + +echo "Ubuntu update and install docker-compose..." ssh "$TLD" sudo apt-get update ssh "$TLD" sudo apt-get upgrade -y ssh "$TLD" sudo apt-get install -y docker-compose ssh "$TLD" sudo apt-get autoremove -y -green "Done.\n" +echo "Done.\n" -yellow "Copy over files..." +echo "Copy over files..." ssh "$TLD" mkdir -p /home/vps/ envsubst < docker-compose.yaml > docker-compose.yaml.fd scp docker-compose.yaml.fd "$TLD":/home/vps/docker-compose.yaml rm docker-compose.yaml.fd -green "Done.\n" +echo "Done.\n" -yellow "Restart docker-compose" -ssh "$TLD" "cd /home/vps/ && docker-compose down" +echo "Restart docker-compose" ssh "$TLD" "cd /home/vps/ && docker-compose pull" ssh "$TLD" "cd /home/vps/ && docker-compose up -d" -green "Done.\n" +echo "Done.\n" diff --git a/provision.sh b/provision.sh new file mode 100644 index 0000000..b13be57 --- /dev/null +++ b/provision.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +cd "${BASH_SOURCE%/*}/" + +echo "Unlocking git-crypt repositories and loading secrets..." +git crypt unlock +pushd ../vps-state/ +git crypt unlock +popd +source .envrc +echo "Done.\n" + +alias ssh="ssh -i secrets/id_rsa root@$TLD" + +echo "Shutting down running containers..." +ssh "cd /home/vps/ && docker-compose down" +echo "Done.\n" + +echo "Running `terraform apply`..." +terraform apply +echo "Done.\n" + +echo "Storing .tfstate file" +pushd ../vps-state/ +git add secrets/terraform.tfstate secrets/terraform.tfstate.backup +git commit -m "CI: update Terraform .tfstate files" +git push origin master +popd +echo "Done.\n" + +echo "Locking git-crypt repositories back..." +git crypt lock +pushd ../vps-state/ +git crypt lock +popd +echo "Done.\n" diff --git a/secrets/terraform.tfstate b/secrets/terraform.tfstate deleted file mode 100644 index 58d3d71..0000000 Binary files a/secrets/terraform.tfstate and /dev/null differ diff --git a/secrets/terraform.tfstate.backup b/secrets/terraform.tfstate.backup deleted file mode 100644 index f6e33dd..0000000 Binary files a/secrets/terraform.tfstate.backup and /dev/null differ diff --git a/terraform.tfstate b/terraform.tfstate index e3dc257..4fa5ae3 120000 --- a/terraform.tfstate +++ b/terraform.tfstate @@ -1 +1 @@ -secrets/terraform.tfstate \ No newline at end of file +../vps-state/secrets/terraform.tfstate \ No newline at end of file diff --git a/terraform.tfstate.backup b/terraform.tfstate.backup index 7916717..54845c8 120000 --- a/terraform.tfstate.backup +++ b/terraform.tfstate.backup @@ -1 +1 @@ -secrets/terraform.tfstate.backup \ No newline at end of file +../vps-state/secrets/terraform.tfstate.backup \ No newline at end of file diff --git a/vps.tf b/vps.tf index b89fd76..c599f36 100644 --- a/vps.tf +++ b/vps.tf @@ -25,4 +25,8 @@ resource "digitalocean_droplet" "vps" { private_key = "${file("${path.module}/secrets/id_rsa")}" timeout = "2m" } -} + + provisioner "remote-exec" { + script = "./deploy.sh" + } +} \ No newline at end of file -- cgit v1.2.3