From f6b508cc67c79b225d88f66c0fbfce787c80ace2 Mon Sep 17 00:00:00 2001 From: EuAndreh Date: Sat, 18 May 2019 23:10:32 -0300 Subject: Add ArchiveBox setup - add repo to mrconfig; - create daily log file; - add `archivebox` executable to $PATH; - create cronjob to run every 12 hours; - backup ~/archive/ folder. Side note: medium.com DNS resolves to an IPv6 address but rejects connections from clients that try to reach medium using it. I had to force an IPv4 connection instead. --- bash/env.sh | 1 + bash/symlinks.sh | 1 + .../manifest-timestamp | Bin 48 -> 48 bytes .../nonce | Bin 38 -> 38 bytes git/mrconfig.ini | Bin 9635 -> 9729 bytes nixos/configuration.nix | 7 +++--- scripts/archive.sh | 13 ++++++++++ scripts/backup.sh | 27 +++++++++++++++------ scripts/wget-4.sh | 2 ++ 9 files changed, 39 insertions(+), 12 deletions(-) create mode 100755 scripts/archive.sh create mode 100755 scripts/wget-4.sh diff --git a/bash/env.sh b/bash/env.sh index 7fe5f65..da62d73 100644 --- a/bash/env.sh +++ b/bash/env.sh @@ -36,6 +36,7 @@ export PATH="$HOME/dev/libre/website/:$PATH" export PATH="$DOTFILES/bin:$PATH" export PATH="$HOME/dev/misc/flutter/bin:$PATH" export PATH="$HOME/dev/libre/songbooks/cli:$PATH" +export PATH="$HOME/dev/misc/ArchiveBox/bin:$PATH" #export BROWSER="firefox" diff --git a/bash/symlinks.sh b/bash/symlinks.sh index 2250a86..fbc4a4d 100644 --- a/bash/symlinks.sh +++ b/bash/symlinks.sh @@ -75,6 +75,7 @@ ln -fs "$DOTFILES/mail/mbsyncrc.ini" ~/.mbsyncrc ln -fs "$DOTFILES/mail/notmuch.ini" ~/.notmuch-config ln -fs /tmp/mbsync."$(date +%Y-%m-%d)".log /tmp/mbsync.log ln -fs /tmp/notmuch."$(date +%Y-%m-%d)".log /tmp/notmuch.log +ln -fs /tmp/archivebox."$(date +%Y-%m-%d)".log /tmp/archivebox.log ## GNUS ln -fs "$DOTFILES/encrypted/IMAP/authinfo.gpg" ~/.authinfo.gpg diff --git a/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/manifest-timestamp b/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/manifest-timestamp index 5b1f054..1bcd04c 100644 Binary files a/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/manifest-timestamp and b/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/manifest-timestamp differ diff --git a/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/nonce b/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/nonce index 00796f3..2e27c8b 100644 Binary files a/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/nonce and b/encrypted/borg/security/425dbda99f4623e1f6f8a6f4ebbfca53ff19bee74c920e10c86f6e2f6c737160/nonce differ diff --git a/git/mrconfig.ini b/git/mrconfig.ini index 744b1bb..88c6884 100644 Binary files a/git/mrconfig.ini and b/git/mrconfig.ini differ diff --git a/nixos/configuration.nix b/nixos/configuration.nix index d865be4..c050ef5 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -338,11 +338,10 @@ in # We can see the actual generated crontab at /etc/crontab cron.systemCronJobs = [ # Sync emails with mbsync every hour - "0 * * * * andreh /home/andreh/dev/libre/dotfiles/scripts/mail.sh -s" - - # Refresh local RSS files every hour - "0 * * * * andreh /home/andreh/dev/libre/dotfiles/scripts/atom.sh" + "0 * * * * andreh /home/andreh/dev/libre/dotfiles/scripts/mail.sh -s" + # Archive Pocket data every 12 hours + "0 12 * * * andreh /home/andreh/dev/libre/dotfiles/scripts/archive.sh" ]; xserver = { diff --git a/scripts/archive.sh b/scripts/archive.sh new file mode 100755 index 0000000..416b833 --- /dev/null +++ b/scripts/archive.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" + +export POCKET_ARCHIVE='https://getpocket.com/users/euandreh/feed/read' +export OUTPUT_DIR="$HOME/archive/archivebox/" +export WGET_BINARY="$PWD/wget-4.sh" +export FETCH_MEDIA=False +if [ "${1-}" = "-s" ]; then + curl -s "$POCKET_ARCHIVE" | archivebox >> /tmp/archivebox.log +else + curl -s "$POCKET_ARCHIVE" | archivebox +fi diff --git a/scripts/backup.sh b/scripts/backup.sh index f31a503..b13108f 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -6,16 +6,21 @@ # if [ ! -d ~/UTCLOUD/ ]; then - red "$HOME/UTCLOUD not attached. Backup not started." - exit 1 + red "$HOME/UTCLOUD not attached. Backup not started." + exit 1 +fi + +if [ ! -d ~/archive/ ]; then + yellow "$HOME/archive/ folder doesn't exist. Downloading latest version from ~/UTCLOUD/archive/ into ~/archive/." + rsync --verbose --progress --stats --update --recursive ~/UTCLOUD/archive/ ~/archive fi if [ ! -d ~/borgbackup/ ]; then - yellow "$HOME/borgbackup/ repository doesn't exist. Downloading latest version from ~/UTCLOUD/borg/borgbackup/ into ~/borgbackup/" - # The initial borg repo was created with: - # $ borg init --append-only --encryption=keyfile ~/borgbackup - # See also: https://borgbackup.readthedocs.io/en/stable/usage/notes.html#append-only-mode - rsync --verbose --progress --stats --update --recursive ~/UTCLOUD/borg/borgbackup/ ~/borgbackup + yellow "$HOME/borgbackup/ repository doesn't exist. Downloading latest version from ~/UTCLOUD/borg/borgbackup/ into ~/borgbackup/." + # The initial borg repo was created with: + # $ borg init --append-only --encryption=keyfile ~/borgbackup + # See also: https://borgbackup.readthedocs.io/en/stable/usage/notes.html#append-only-mode + rsync --verbose --progress --stats --update --recursive ~/UTCLOUD/borg/borgbackup/ ~/borgbackup fi BACKUP_TAG="${1-default}" @@ -28,10 +33,16 @@ borg create \ --compression lzma,6 \ ~/borgbackup::"{hostname}-{now}-${BACKUP_TAG}" \ ~/ownCloud/ \ - ~/mbsync/ + ~/Nextcloud/ \ + ~/mbsync/ \ + ~/archive/ # add folders to be backed up here green "Done" +yellow "Syncing ~/archive to ~/UTCLOUD/archive" +rsync --verbose --progress --stats --update --recursive ~/archive/ ~/UTCLOUD/archive/ +green "Done" + yellow "Syncing ~/borgbackup to ~/UTCLOUD/borg/borgbackup/" rsync --verbose --progress --stats --update --recursive ~/borgbackup/ ~/UTCLOUD/borg/borgbackup/ green "Done" diff --git a/scripts/wget-4.sh b/scripts/wget-4.sh new file mode 100755 index 0000000..1066441 --- /dev/null +++ b/scripts/wget-4.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +wget -4 "$@" -- cgit v1.2.3