From 10a15d3eae483f6db54ee836c55fc5f020e8452a Mon Sep 17 00:00:00 2001 From: EuAndreh Date: Sat, 5 Apr 2025 06:45:33 -0300 Subject: Makefile: Add test for internal broken links --- .gitignore | 1 + Makefile | 22 +- deps.mk | 103 +++++++-- mkdeps.sh | 2 + .../blog/2020/08/31/database-i-wish-i-had.adoc | 257 +++++++++++++++++++++ .../blog/2020/08/31/database-i-with-i-had.adoc | 257 --------------------- src/content/blog/2020/10/19/feature-flags.adoc | 2 +- .../blog/2020/11/14/local-first-review.adoc | 2 +- .../tils/2020/09/04/cli-email-fun-profit.adoc | 75 ++++++ .../tils/2020/09/04/email-cli-fun-profit.adoc | 75 ------ 10 files changed, 444 insertions(+), 352 deletions(-) create mode 100644 src/content/blog/2020/08/31/database-i-wish-i-had.adoc delete mode 100644 src/content/blog/2020/08/31/database-i-with-i-had.adoc create mode 100644 src/content/tils/2020/09/04/cli-email-fun-profit.adoc delete mode 100644 src/content/tils/2020/09/04/email-cli-fun-profit.adoc diff --git a/.gitignore b/.gitignore index ff045ef..e9b6c3d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.embedded-config *.conf *.snippets +*.links *.html *.html.*.txt *.indexentry diff --git a/Makefile b/Makefile index d15e5aa..37eaa73 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ PUBURL = public.asc .SUFFIXES: .SUFFIXES: .adoc .conf .snippets .indexentry .feedentry .sortdata .xml -.SUFFIXES: .htmlbody .htmlheader .htmlfooter .htmllisting .html +.SUFFIXES: .htmlbody .htmlheader .htmlfooter .htmllisting .html .links .SUFFIXES: .gz .adoc.conf: @@ -58,6 +58,9 @@ PUBURL = public.asc .conf.sortdata: mkwb sortdata $< > $@ +.adoc.links: + mkwb links $< > $@ + all: @@ -70,6 +73,7 @@ sources.html = $(sources.adoc:.adoc=.html) sources.snippets = $(sources.adoc:.adoc=.snippets) sources.snippets.gz = $(sources.adoc:.adoc=.snippets.gz) sources.conf = $(sources.adoc:.adoc=.conf) +sources.links = $(sources.adoc:.adoc=.links) articles.indexentry = $(articles.adoc:.adoc=.indexentry) articles.feedentry = $(articles.adoc:.adoc=.feedentry) articles.sortdata = $(articles.adoc:.adoc=.sortdata) @@ -132,6 +136,7 @@ derived-assets = \ $(sources.htmlbody) \ $(sources.snippets) \ $(sources.snippets.gz) \ + $(sources.links) \ $(sources.conf) \ src/dyn.conf \ src/base.conf \ @@ -259,7 +264,20 @@ sources.txt install.txt: -check-unit: +.SUFFIXES: .links-check-internal +sources.links-check-internal = $(sources.adoc:.adoc=.links-check-internal) +$(sources.links-check-internal): $(sources.html) + grep '^link:' $*.links | cut -d: -f2- | xargs -tI% test -e $(*D)/% + +check-internal: $(sources.links-check-internal) + + +check-external: + + +check-links: check-internal check-external + +check-unit: check-links integration-tests = \ diff --git a/deps.mk b/deps.mk index 6567d22..da9a705 100644 --- a/deps.mk +++ b/deps.mk @@ -8,7 +8,7 @@ articles.adoc = \ src/content/blog/2018/12/21/ytdl-subs.adoc \ src/content/blog/2019/06/02/nixos-stateless-workstation.adoc \ src/content/blog/2020/08/10/guix-srht.adoc \ - src/content/blog/2020/08/31/database-i-with-i-had.adoc \ + src/content/blog/2020/08/31/database-i-wish-i-had.adoc \ src/content/blog/2020/10/05/cargo2nix.adoc \ src/content/blog/2020/10/05/swift2nix.adoc \ src/content/blog/2020/10/19/feature-flags.adoc \ @@ -49,7 +49,7 @@ articles.adoc = \ src/content/tils/2020/08/14/browse-git.adoc \ src/content/tils/2020/08/16/git-search.adoc \ src/content/tils/2020/08/28/grep-online.adoc \ - src/content/tils/2020/09/04/email-cli-fun-profit.adoc \ + src/content/tils/2020/09/04/cli-email-fun-profit.adoc \ src/content/tils/2020/09/05/oldschool-pr.adoc \ src/content/tils/2020/10/11/search-git-history.adoc \ src/content/tils/2020/11/08/find-broken-symlink.adoc \ @@ -112,7 +112,7 @@ src/content/blog/2018/08/01/npm-ci-reproducibility.htmlbody src/content/blog/201 src/content/blog/2018/12/21/ytdl-subs.htmlbody src/content/blog/2018/12/21/ytdl-subs.snippets src/content/blog/2018/12/21/ytdl-subs.conf: src/content/blog/2018/12/21/ytdl-subs.adoc src/content/blog/2019/06/02/nixos-stateless-workstation.htmlbody src/content/blog/2019/06/02/nixos-stateless-workstation.snippets src/content/blog/2019/06/02/nixos-stateless-workstation.conf: src/content/blog/2019/06/02/nixos-stateless-workstation.adoc src/content/blog/2020/08/10/guix-srht.htmlbody src/content/blog/2020/08/10/guix-srht.snippets src/content/blog/2020/08/10/guix-srht.conf: src/content/blog/2020/08/10/guix-srht.adoc -src/content/blog/2020/08/31/database-i-with-i-had.htmlbody src/content/blog/2020/08/31/database-i-with-i-had.snippets src/content/blog/2020/08/31/database-i-with-i-had.conf: src/content/blog/2020/08/31/database-i-with-i-had.adoc +src/content/blog/2020/08/31/database-i-wish-i-had.htmlbody src/content/blog/2020/08/31/database-i-wish-i-had.snippets src/content/blog/2020/08/31/database-i-wish-i-had.conf: src/content/blog/2020/08/31/database-i-wish-i-had.adoc src/content/blog/2020/10/05/cargo2nix.htmlbody src/content/blog/2020/10/05/cargo2nix.snippets src/content/blog/2020/10/05/cargo2nix.conf: src/content/blog/2020/10/05/cargo2nix.adoc src/content/blog/2020/10/05/swift2nix.htmlbody src/content/blog/2020/10/05/swift2nix.snippets src/content/blog/2020/10/05/swift2nix.conf: src/content/blog/2020/10/05/swift2nix.adoc src/content/blog/2020/10/19/feature-flags.htmlbody src/content/blog/2020/10/19/feature-flags.snippets src/content/blog/2020/10/19/feature-flags.conf: src/content/blog/2020/10/19/feature-flags.adoc @@ -153,7 +153,7 @@ src/content/tils/2020/08/13/code-jekyll.htmlbody src/content/tils/2020/08/13/cod src/content/tils/2020/08/14/browse-git.htmlbody src/content/tils/2020/08/14/browse-git.snippets src/content/tils/2020/08/14/browse-git.conf: src/content/tils/2020/08/14/browse-git.adoc src/content/tils/2020/08/16/git-search.htmlbody src/content/tils/2020/08/16/git-search.snippets src/content/tils/2020/08/16/git-search.conf: src/content/tils/2020/08/16/git-search.adoc src/content/tils/2020/08/28/grep-online.htmlbody src/content/tils/2020/08/28/grep-online.snippets src/content/tils/2020/08/28/grep-online.conf: src/content/tils/2020/08/28/grep-online.adoc -src/content/tils/2020/09/04/email-cli-fun-profit.htmlbody src/content/tils/2020/09/04/email-cli-fun-profit.snippets src/content/tils/2020/09/04/email-cli-fun-profit.conf: src/content/tils/2020/09/04/email-cli-fun-profit.adoc +src/content/tils/2020/09/04/cli-email-fun-profit.htmlbody src/content/tils/2020/09/04/cli-email-fun-profit.snippets src/content/tils/2020/09/04/cli-email-fun-profit.conf: src/content/tils/2020/09/04/cli-email-fun-profit.adoc src/content/tils/2020/09/05/oldschool-pr.htmlbody src/content/tils/2020/09/05/oldschool-pr.snippets src/content/tils/2020/09/05/oldschool-pr.conf: src/content/tils/2020/09/05/oldschool-pr.adoc src/content/tils/2020/10/11/search-git-history.htmlbody src/content/tils/2020/10/11/search-git-history.snippets src/content/tils/2020/10/11/search-git-history.conf: src/content/tils/2020/10/11/search-git-history.adoc src/content/tils/2020/11/08/find-broken-symlink.htmlbody src/content/tils/2020/11/08/find-broken-symlink.snippets src/content/tils/2020/11/08/find-broken-symlink.conf: src/content/tils/2020/11/08/find-broken-symlink.adoc @@ -183,7 +183,7 @@ src/content/blog/2018/08/01/npm-ci-reproducibility.html: src/content/blog/2018/0 src/content/blog/2018/12/21/ytdl-subs.html: src/content/blog/2018/12/21/ytdl-subs.conf src/content/blog/2018/12/21/ytdl-subs.htmlbody src/content/blog/2019/06/02/nixos-stateless-workstation.html: src/content/blog/2019/06/02/nixos-stateless-workstation.conf src/content/blog/2019/06/02/nixos-stateless-workstation.htmlbody src/content/blog/2020/08/10/guix-srht.html: src/content/blog/2020/08/10/guix-srht.conf src/content/blog/2020/08/10/guix-srht.htmlbody -src/content/blog/2020/08/31/database-i-with-i-had.html: src/content/blog/2020/08/31/database-i-with-i-had.conf src/content/blog/2020/08/31/database-i-with-i-had.htmlbody +src/content/blog/2020/08/31/database-i-wish-i-had.html: src/content/blog/2020/08/31/database-i-wish-i-had.conf src/content/blog/2020/08/31/database-i-wish-i-had.htmlbody src/content/blog/2020/10/05/cargo2nix.html: src/content/blog/2020/10/05/cargo2nix.conf src/content/blog/2020/10/05/cargo2nix.htmlbody src/content/blog/2020/10/05/swift2nix.html: src/content/blog/2020/10/05/swift2nix.conf src/content/blog/2020/10/05/swift2nix.htmlbody src/content/blog/2020/10/19/feature-flags.html: src/content/blog/2020/10/19/feature-flags.conf src/content/blog/2020/10/19/feature-flags.htmlbody @@ -224,7 +224,7 @@ src/content/tils/2020/08/13/code-jekyll.html: src/content/tils/2020/08/13/code-j src/content/tils/2020/08/14/browse-git.html: src/content/tils/2020/08/14/browse-git.conf src/content/tils/2020/08/14/browse-git.htmlbody src/content/tils/2020/08/16/git-search.html: src/content/tils/2020/08/16/git-search.conf src/content/tils/2020/08/16/git-search.htmlbody src/content/tils/2020/08/28/grep-online.html: src/content/tils/2020/08/28/grep-online.conf src/content/tils/2020/08/28/grep-online.htmlbody -src/content/tils/2020/09/04/email-cli-fun-profit.html: src/content/tils/2020/09/04/email-cli-fun-profit.conf src/content/tils/2020/09/04/email-cli-fun-profit.htmlbody +src/content/tils/2020/09/04/cli-email-fun-profit.html: src/content/tils/2020/09/04/cli-email-fun-profit.conf src/content/tils/2020/09/04/cli-email-fun-profit.htmlbody src/content/tils/2020/09/05/oldschool-pr.html: src/content/tils/2020/09/05/oldschool-pr.conf src/content/tils/2020/09/05/oldschool-pr.htmlbody src/content/tils/2020/10/11/search-git-history.html: src/content/tils/2020/10/11/search-git-history.conf src/content/tils/2020/10/11/search-git-history.htmlbody src/content/tils/2020/11/08/find-broken-symlink.html: src/content/tils/2020/11/08/find-broken-symlink.conf src/content/tils/2020/11/08/find-broken-symlink.htmlbody @@ -255,7 +255,7 @@ src/content/blog/2018/08/01/npm-ci-reproducibility.html.gz: src/content/blog/201 src/content/blog/2018/12/21/ytdl-subs.html.gz: src/content/blog/2018/12/21/ytdl-subs.html src/content/blog/2019/06/02/nixos-stateless-workstation.html.gz: src/content/blog/2019/06/02/nixos-stateless-workstation.html src/content/blog/2020/08/10/guix-srht.html.gz: src/content/blog/2020/08/10/guix-srht.html -src/content/blog/2020/08/31/database-i-with-i-had.html.gz: src/content/blog/2020/08/31/database-i-with-i-had.html +src/content/blog/2020/08/31/database-i-wish-i-had.html.gz: src/content/blog/2020/08/31/database-i-wish-i-had.html src/content/blog/2020/10/05/cargo2nix.html.gz: src/content/blog/2020/10/05/cargo2nix.html src/content/blog/2020/10/05/swift2nix.html.gz: src/content/blog/2020/10/05/swift2nix.html src/content/blog/2020/10/19/feature-flags.html.gz: src/content/blog/2020/10/19/feature-flags.html @@ -296,7 +296,7 @@ src/content/tils/2020/08/13/code-jekyll.html.gz: src/content/tils/2020/08/13/cod src/content/tils/2020/08/14/browse-git.html.gz: src/content/tils/2020/08/14/browse-git.html src/content/tils/2020/08/16/git-search.html.gz: src/content/tils/2020/08/16/git-search.html src/content/tils/2020/08/28/grep-online.html.gz: src/content/tils/2020/08/28/grep-online.html -src/content/tils/2020/09/04/email-cli-fun-profit.html.gz: src/content/tils/2020/09/04/email-cli-fun-profit.html +src/content/tils/2020/09/04/cli-email-fun-profit.html.gz: src/content/tils/2020/09/04/cli-email-fun-profit.html src/content/tils/2020/09/05/oldschool-pr.html.gz: src/content/tils/2020/09/05/oldschool-pr.html src/content/tils/2020/10/11/search-git-history.html.gz: src/content/tils/2020/10/11/search-git-history.html src/content/tils/2020/11/08/find-broken-symlink.html.gz: src/content/tils/2020/11/08/find-broken-symlink.html @@ -326,7 +326,7 @@ src/content/blog/2018/08/01/npm-ci-reproducibility.snippets.gz: src/content/blog src/content/blog/2018/12/21/ytdl-subs.snippets.gz: src/content/blog/2018/12/21/ytdl-subs.snippets src/content/blog/2019/06/02/nixos-stateless-workstation.snippets.gz: src/content/blog/2019/06/02/nixos-stateless-workstation.snippets src/content/blog/2020/08/10/guix-srht.snippets.gz: src/content/blog/2020/08/10/guix-srht.snippets -src/content/blog/2020/08/31/database-i-with-i-had.snippets.gz: src/content/blog/2020/08/31/database-i-with-i-had.snippets +src/content/blog/2020/08/31/database-i-wish-i-had.snippets.gz: src/content/blog/2020/08/31/database-i-wish-i-had.snippets src/content/blog/2020/10/05/cargo2nix.snippets.gz: src/content/blog/2020/10/05/cargo2nix.snippets src/content/blog/2020/10/05/swift2nix.snippets.gz: src/content/blog/2020/10/05/swift2nix.snippets src/content/blog/2020/10/19/feature-flags.snippets.gz: src/content/blog/2020/10/19/feature-flags.snippets @@ -367,7 +367,7 @@ src/content/tils/2020/08/13/code-jekyll.snippets.gz: src/content/tils/2020/08/13 src/content/tils/2020/08/14/browse-git.snippets.gz: src/content/tils/2020/08/14/browse-git.snippets src/content/tils/2020/08/16/git-search.snippets.gz: src/content/tils/2020/08/16/git-search.snippets src/content/tils/2020/08/28/grep-online.snippets.gz: src/content/tils/2020/08/28/grep-online.snippets -src/content/tils/2020/09/04/email-cli-fun-profit.snippets.gz: src/content/tils/2020/09/04/email-cli-fun-profit.snippets +src/content/tils/2020/09/04/cli-email-fun-profit.snippets.gz: src/content/tils/2020/09/04/cli-email-fun-profit.snippets src/content/tils/2020/09/05/oldschool-pr.snippets.gz: src/content/tils/2020/09/05/oldschool-pr.snippets src/content/tils/2020/10/11/search-git-history.snippets.gz: src/content/tils/2020/10/11/search-git-history.snippets src/content/tils/2020/11/08/find-broken-symlink.snippets.gz: src/content/tils/2020/11/08/find-broken-symlink.snippets @@ -396,12 +396,83 @@ src/content/podcasts/feed.xml.gz: src/content/podcasts/feed.xml src/content/screencasts/feed.xml.gz: src/content/screencasts/feed.xml src/content/tils/feed.xml.gz: src/content/tils/feed.xml +src/content/about.links-check-internal: src/content/about.links +src/content/index.links-check-internal: src/content/index.links +src/content/blog/2018/07/17/guix-nixos.links-check-internal: src/content/blog/2018/07/17/guix-nixos.links +src/content/blog/2018/08/01/npm-ci-reproducibility.links-check-internal: src/content/blog/2018/08/01/npm-ci-reproducibility.links +src/content/blog/2018/12/21/ytdl-subs.links-check-internal: src/content/blog/2018/12/21/ytdl-subs.links +src/content/blog/2019/06/02/nixos-stateless-workstation.links-check-internal: src/content/blog/2019/06/02/nixos-stateless-workstation.links +src/content/blog/2020/08/10/guix-srht.links-check-internal: src/content/blog/2020/08/10/guix-srht.links +src/content/blog/2020/08/31/database-i-wish-i-had.links-check-internal: src/content/blog/2020/08/31/database-i-wish-i-had.links +src/content/blog/2020/10/05/cargo2nix.links-check-internal: src/content/blog/2020/10/05/cargo2nix.links +src/content/blog/2020/10/05/swift2nix.links-check-internal: src/content/blog/2020/10/05/swift2nix.links +src/content/blog/2020/10/19/feature-flags.links-check-internal: src/content/blog/2020/10/19/feature-flags.links +src/content/blog/2020/10/20/wrong-interviewing.links-check-internal: src/content/blog/2020/10/20/wrong-interviewing.links +src/content/blog/2020/11/07/diy-bugs.links-check-internal: src/content/blog/2020/11/07/diy-bugs.links +src/content/blog/2020/11/08/paradigm-shift-review.links-check-internal: src/content/blog/2020/11/08/paradigm-shift-review.links +src/content/blog/2020/11/12/database-parsers-trees.links-check-internal: src/content/blog/2020/11/12/database-parsers-trees.links +src/content/blog/2020/11/14/local-first-review.links-check-internal: src/content/blog/2020/11/14/local-first-review.links +src/content/blog/2021/01/26/remembering-ann.links-check-internal: src/content/blog/2021/01/26/remembering-ann.links +src/content/blog/2021/02/17/fallible.links-check-internal: src/content/blog/2021/02/17/fallible.links +src/content/blog/2021/04/29/relational-review.links-check-internal: src/content/blog/2021/04/29/relational-review.links +src/content/pastebins/2016/04/05/rpn.links-check-internal: src/content/pastebins/2016/04/05/rpn.links +src/content/pastebins/2018/07/11/nix-pinning.links-check-internal: src/content/pastebins/2018/07/11/nix-pinning.links +src/content/pastebins/2018/07/13/guix-nixos-systemd.links-check-internal: src/content/pastebins/2018/07/13/guix-nixos-systemd.links +src/content/pastebins/2018/07/13/guixbuilder-nixos.links-check-internal: src/content/pastebins/2018/07/13/guixbuilder-nixos.links +src/content/pastebins/2018/07/13/guixbuilder.links-check-internal: src/content/pastebins/2018/07/13/guixbuilder.links +src/content/pastebins/2018/07/13/nix-strpad.links-check-internal: src/content/pastebins/2018/07/13/nix-strpad.links +src/content/pastebins/2018/07/25/nix-exps.links-check-internal: src/content/pastebins/2018/07/25/nix-exps.links +src/content/pastebins/2018/07/25/nix-showdrv.links-check-internal: src/content/pastebins/2018/07/25/nix-showdrv.links +src/content/pastebins/2019/06/08/inconsistent-hash.links-check-internal: src/content/pastebins/2019/06/08/inconsistent-hash.links +src/content/pastebins/2019/12/29/raku-tuple-type.links-check-internal: src/content/pastebins/2019/12/29/raku-tuple-type.links +src/content/pastebins/2020/01/04/guix-import-failure.links-check-internal: src/content/pastebins/2020/01/04/guix-import-failure.links +src/content/pastebins/2020/02/14/guix-shebang.links-check-internal: src/content/pastebins/2020/02/14/guix-shebang.links +src/content/pastebins/2020/11/27/guix-build-local.links-check-internal: src/content/pastebins/2020/11/27/guix-build-local.links +src/content/pastebins/2020/12/15/guix-pack-fail.links-check-internal: src/content/pastebins/2020/12/15/guix-pack-fail.links +src/content/pastebins/2021/04/03/naive-slugify-js.links-check-internal: src/content/pastebins/2021/04/03/naive-slugify-js.links +src/content/pastebins/2021/06/08/reading-session-pt1.links-check-internal: src/content/pastebins/2021/06/08/reading-session-pt1.links +src/content/pastebins/2021/06/22/curl-wget.links-check-internal: src/content/pastebins/2021/06/22/curl-wget.links +src/content/pastebins/2021/08/11/h1-spacing.links-check-internal: src/content/pastebins/2021/08/11/h1-spacing.links +src/content/pastebins/2021/09/02/sicp-3-19.links-check-internal: src/content/pastebins/2021/09/02/sicp-3-19.links +src/content/pastebins/2021/09/03/sicp-persistent-queue.links-check-internal: src/content/pastebins/2021/09/03/sicp-persistent-queue.links +src/content/pastebins/2022/07/14/git-cleanup.links-check-internal: src/content/pastebins/2022/07/14/git-cleanup.links +src/content/pastebins/2023/07/22/funcallable-amop.links-check-internal: src/content/pastebins/2023/07/22/funcallable-amop.links +src/content/podcasts/2020/12/19/test-entry.links-check-internal: src/content/podcasts/2020/12/19/test-entry.links +src/content/screencasts/2021/02/07/autoqemu.links-check-internal: src/content/screencasts/2021/02/07/autoqemu.links +src/content/tils/2020/08/12/filename-timestamp.links-check-internal: src/content/tils/2020/08/12/filename-timestamp.links +src/content/tils/2020/08/13/code-jekyll.links-check-internal: src/content/tils/2020/08/13/code-jekyll.links +src/content/tils/2020/08/14/browse-git.links-check-internal: src/content/tils/2020/08/14/browse-git.links +src/content/tils/2020/08/16/git-search.links-check-internal: src/content/tils/2020/08/16/git-search.links +src/content/tils/2020/08/28/grep-online.links-check-internal: src/content/tils/2020/08/28/grep-online.links +src/content/tils/2020/09/04/cli-email-fun-profit.links-check-internal: src/content/tils/2020/09/04/cli-email-fun-profit.links +src/content/tils/2020/09/05/oldschool-pr.links-check-internal: src/content/tils/2020/09/05/oldschool-pr.links +src/content/tils/2020/10/11/search-git-history.links-check-internal: src/content/tils/2020/10/11/search-git-history.links +src/content/tils/2020/11/08/find-broken-symlink.links-check-internal: src/content/tils/2020/11/08/find-broken-symlink.links +src/content/tils/2020/11/12/diy-nix-bash-ci.links-check-internal: src/content/tils/2020/11/12/diy-nix-bash-ci.links +src/content/tils/2020/11/12/git-bisect-automation.links-check-internal: src/content/tils/2020/11/12/git-bisect-automation.links +src/content/tils/2020/11/12/useful-bashvars.links-check-internal: src/content/tils/2020/11/12/useful-bashvars.links +src/content/tils/2020/11/14/gpodder-media.links-check-internal: src/content/tils/2020/11/14/gpodder-media.links +src/content/tils/2020/11/30/git-notes-ci.links-check-internal: src/content/tils/2020/11/30/git-notes-ci.links +src/content/tils/2020/12/15/shellcheck-repo.links-check-internal: src/content/tils/2020/12/15/shellcheck-repo.links +src/content/tils/2020/12/29/svg.links-check-internal: src/content/tils/2020/12/29/svg.links +src/content/tils/2021/01/12/curl-awk-emails.links-check-internal: src/content/tils/2021/01/12/curl-awk-emails.links +src/content/tils/2021/01/17/posix-shebang.links-check-internal: src/content/tils/2021/01/17/posix-shebang.links +src/content/tils/2021/04/24/cl-generic-precedence.links-check-internal: src/content/tils/2021/04/24/cl-generic-precedence.links +src/content/tils/2021/04/24/clojure-autocurry.links-check-internal: src/content/tils/2021/04/24/clojure-autocurry.links +src/content/tils/2021/04/24/scm-nif.links-check-internal: src/content/tils/2021/04/24/scm-nif.links +src/content/tils/2021/07/23/git-tls-gpg.links-check-internal: src/content/tils/2021/07/23/git-tls-gpg.links +src/content/tils/2021/08/11/js-bigint-reviver.links-check-internal: src/content/tils/2021/08/11/js-bigint-reviver.links +src/content/blog/index.links-check-internal: src/content/blog/index.links +src/content/pastebins/index.links-check-internal: src/content/pastebins/index.links +src/content/podcasts/index.links-check-internal: src/content/podcasts/index.links +src/content/screencasts/index.links-check-internal: src/content/screencasts/index.links +src/content/tils/index.links-check-internal: src/content/tils/index.links src/content/blog/2018/07/17/guix-nixos.feedentry: src/content/blog/2018/07/17/guix-nixos.conf src/content/blog/2018/07/17/guix-nixos.htmlbody src/content/blog/2018/08/01/npm-ci-reproducibility.feedentry: src/content/blog/2018/08/01/npm-ci-reproducibility.conf src/content/blog/2018/08/01/npm-ci-reproducibility.htmlbody src/content/blog/2018/12/21/ytdl-subs.feedentry: src/content/blog/2018/12/21/ytdl-subs.conf src/content/blog/2018/12/21/ytdl-subs.htmlbody src/content/blog/2019/06/02/nixos-stateless-workstation.feedentry: src/content/blog/2019/06/02/nixos-stateless-workstation.conf src/content/blog/2019/06/02/nixos-stateless-workstation.htmlbody src/content/blog/2020/08/10/guix-srht.feedentry: src/content/blog/2020/08/10/guix-srht.conf src/content/blog/2020/08/10/guix-srht.htmlbody -src/content/blog/2020/08/31/database-i-with-i-had.feedentry: src/content/blog/2020/08/31/database-i-with-i-had.conf src/content/blog/2020/08/31/database-i-with-i-had.htmlbody +src/content/blog/2020/08/31/database-i-wish-i-had.feedentry: src/content/blog/2020/08/31/database-i-wish-i-had.conf src/content/blog/2020/08/31/database-i-wish-i-had.htmlbody src/content/blog/2020/10/05/cargo2nix.feedentry: src/content/blog/2020/10/05/cargo2nix.conf src/content/blog/2020/10/05/cargo2nix.htmlbody src/content/blog/2020/10/05/swift2nix.feedentry: src/content/blog/2020/10/05/swift2nix.conf src/content/blog/2020/10/05/swift2nix.htmlbody src/content/blog/2020/10/19/feature-flags.feedentry: src/content/blog/2020/10/19/feature-flags.conf src/content/blog/2020/10/19/feature-flags.htmlbody @@ -442,7 +513,7 @@ src/content/tils/2020/08/13/code-jekyll.feedentry: src/content/tils/2020/08/13/c src/content/tils/2020/08/14/browse-git.feedentry: src/content/tils/2020/08/14/browse-git.conf src/content/tils/2020/08/14/browse-git.htmlbody src/content/tils/2020/08/16/git-search.feedentry: src/content/tils/2020/08/16/git-search.conf src/content/tils/2020/08/16/git-search.htmlbody src/content/tils/2020/08/28/grep-online.feedentry: src/content/tils/2020/08/28/grep-online.conf src/content/tils/2020/08/28/grep-online.htmlbody -src/content/tils/2020/09/04/email-cli-fun-profit.feedentry: src/content/tils/2020/09/04/email-cli-fun-profit.conf src/content/tils/2020/09/04/email-cli-fun-profit.htmlbody +src/content/tils/2020/09/04/cli-email-fun-profit.feedentry: src/content/tils/2020/09/04/cli-email-fun-profit.conf src/content/tils/2020/09/04/cli-email-fun-profit.htmlbody src/content/tils/2020/09/05/oldschool-pr.feedentry: src/content/tils/2020/09/05/oldschool-pr.conf src/content/tils/2020/09/05/oldschool-pr.htmlbody src/content/tils/2020/10/11/search-git-history.feedentry: src/content/tils/2020/10/11/search-git-history.conf src/content/tils/2020/10/11/search-git-history.htmlbody src/content/tils/2020/11/08/find-broken-symlink.feedentry: src/content/tils/2020/11/08/find-broken-symlink.conf src/content/tils/2020/11/08/find-broken-symlink.htmlbody @@ -465,7 +536,7 @@ src/content/blog/2018/08/01/npm-ci-reproducibility.sortdata: src/content/blog/20 src/content/blog/2018/12/21/ytdl-subs.sortdata: src/content/blog/2018/12/21/ytdl-subs.conf src/content/blog/2019/06/02/nixos-stateless-workstation.sortdata: src/content/blog/2019/06/02/nixos-stateless-workstation.conf src/content/blog/2020/08/10/guix-srht.sortdata: src/content/blog/2020/08/10/guix-srht.conf -src/content/blog/2020/08/31/database-i-with-i-had.sortdata: src/content/blog/2020/08/31/database-i-with-i-had.conf +src/content/blog/2020/08/31/database-i-wish-i-had.sortdata: src/content/blog/2020/08/31/database-i-wish-i-had.conf src/content/blog/2020/10/05/cargo2nix.sortdata: src/content/blog/2020/10/05/cargo2nix.conf src/content/blog/2020/10/05/swift2nix.sortdata: src/content/blog/2020/10/05/swift2nix.conf src/content/blog/2020/10/19/feature-flags.sortdata: src/content/blog/2020/10/19/feature-flags.conf @@ -506,7 +577,7 @@ src/content/tils/2020/08/13/code-jekyll.sortdata: src/content/tils/2020/08/13/co src/content/tils/2020/08/14/browse-git.sortdata: src/content/tils/2020/08/14/browse-git.conf src/content/tils/2020/08/16/git-search.sortdata: src/content/tils/2020/08/16/git-search.conf src/content/tils/2020/08/28/grep-online.sortdata: src/content/tils/2020/08/28/grep-online.conf -src/content/tils/2020/09/04/email-cli-fun-profit.sortdata: src/content/tils/2020/09/04/email-cli-fun-profit.conf +src/content/tils/2020/09/04/cli-email-fun-profit.sortdata: src/content/tils/2020/09/04/cli-email-fun-profit.conf src/content/tils/2020/09/05/oldschool-pr.sortdata: src/content/tils/2020/09/05/oldschool-pr.conf src/content/tils/2020/10/11/search-git-history.sortdata: src/content/tils/2020/10/11/search-git-history.conf src/content/tils/2020/11/08/find-broken-symlink.sortdata: src/content/tils/2020/11/08/find-broken-symlink.conf @@ -548,7 +619,7 @@ articles.blog.adoc = \ src/content/blog/2018/12/21/ytdl-subs.adoc \ src/content/blog/2019/06/02/nixos-stateless-workstation.adoc \ src/content/blog/2020/08/10/guix-srht.adoc \ - src/content/blog/2020/08/31/database-i-with-i-had.adoc \ + src/content/blog/2020/08/31/database-i-wish-i-had.adoc \ src/content/blog/2020/10/05/cargo2nix.adoc \ src/content/blog/2020/10/05/swift2nix.adoc \ src/content/blog/2020/10/19/feature-flags.adoc \ @@ -641,7 +712,7 @@ articles.tils.adoc = \ src/content/tils/2020/08/14/browse-git.adoc \ src/content/tils/2020/08/16/git-search.adoc \ src/content/tils/2020/08/28/grep-online.adoc \ - src/content/tils/2020/09/04/email-cli-fun-profit.adoc \ + src/content/tils/2020/09/04/cli-email-fun-profit.adoc \ src/content/tils/2020/09/05/oldschool-pr.adoc \ src/content/tils/2020/10/11/search-git-history.adoc \ src/content/tils/2020/11/08/find-broken-symlink.adoc \ diff --git a/mkdeps.sh b/mkdeps.sh index e9141f9..459d98f 100755 --- a/mkdeps.sh +++ b/mkdeps.sh @@ -53,6 +53,8 @@ files | sed 's/^\(.*\)\.adoc$/\1.snippets.gz:\t\1.snippets/' listings | sed 's|^\(.*\)/index\.adoc$|\1/feed.xml.gz:\t\1/feed.xml|' printf '\n' +files | sed 's/^\(.*\)\.adoc$/\1.links-check-internal:\t\1.links/' + articles | sed 's/^\(.*\)\.adoc$/\1.feedentry:\t\1.conf\t\1.htmlbody/' articles | sed 's/^\(.*\)\.adoc$/\1.sortdata:\t\1.conf/' printf '\n' diff --git a/src/content/blog/2020/08/31/database-i-wish-i-had.adoc b/src/content/blog/2020/08/31/database-i-wish-i-had.adoc new file mode 100644 index 0000000..443a54e --- /dev/null +++ b/src/content/blog/2020/08/31/database-i-wish-i-had.adoc @@ -0,0 +1,257 @@ += The database I wish I had + +:empty: +:values-talk: https://vimeo.com/230142234 +:haskell-startup: https://www.youtube.com/watch?v=ZR3Jirqk6W8 + +I watched the talk "{values-talk}[Platform as a Reflection of Values: Joyent, +Node.js and beyond]" by Bryan Cantrill, and I think he was able to put into +words something I already felt for some time: if there's no piece of software +out there that reflects your values, it's time for you to build that +software{empty}footnote:talk-time[ + At the very end, at time 29:49. When talking about the draft of this article + with a friend, he noted that Bryan O'Sullivan (a different Bryan) says a + similar thing on his talk "{haskell-startup}[Running a startup on Haskell]", + at time 4:15. +]. + +I kind of agree with what he said, because this is already happening to me. I +long for a database with a certain set of values, and for a few years I was just +waiting for someone to finally write it. After watching his talk, Bryan is +saying to me: "time to stop waiting, and start writing it yourself". + +So let me try to give an overview of such database, and go over its values. + +== Overview + +I want a database that allows me to create decentralized client-side +applications that can sync data. + +The best one-line description I can give right now is: + +____ +It's sort of like PouchDB, Git, Datomic, SQLite and Mentat. +____ + +A more descriptive version could be: + +____ +An embedded, immutable, syncable relational database. +____ + +Let's go over what I mean by each of those aspects one by one. + +=== Embedded + +:sqlite: https://sqlite.org/index.html +:sqlite-whentouse: https://sqlite.org/whentouse.html +:pouchdb: https://pouchdb.com/ +:couchdb: https://couchdb.apache.org/ +:mentat: https://github.com/mozilla/mentat +:pouchdb-adapters: https://pouchdb.com/adapters.html +:datomic-storage-services: https://docs.datomic.com/on-prem/storage.html +:sqlite-amalgamation: https://www.sqlite.org/amalgamation.html + +I think the server-side database landscape is diverse and mature enough for my +needs (even though I end up choosing SQLite most of the time), and what I'm +after is a database to be embedded on client-side applications itself, be it +desktop, browser, mobile, _etc._ + +The purpose of such database is not to keep some local cache of data in case of +lost connectivity: we have good solutions for that already. It should serve as +the source of truth, and allow the application to work on top of it. + +{sqlite}[*SQLite*] is a great example of that: it is a very powerful relational +database that runs {sqlite-whentouse}[almost anywhere]. What I miss from it +that SQLite doesn't provide is the ability to run it on the browser: even though +you could compile it to WebAssembly, [line-through]#it assumes a POSIX +filesystem that would have to be emulated#[multiblock footnote omitted FIXME]. + +{pouchdb}[*PouchDB*] is another great example: it's a full reimplementation of +{couchdb}[CouchDB] that targets JavaScript environments, mainly the browser and +Node.js. However I want a tool that can be deployed anywhere, and not limit its +applications to places that already have a JavaScript runtime environment, or +force the developer to bundle a JavaScript runtime environment with their +application. This is true for GTK+ applications, command line programs, Android +apps, _etc._ + +{mentat}[*Mentat*] was an interesting project, but its reliance on SQLite makes +it inherit most of the downsides (and benefits too) of SQLite itself. + +Having such a requirement imposes a different approach to storage: we have to +decouple the knowledge about the intricacies of storage from the usage of +storage itself, so that a module (say query processing) can access storage +through an API without needing to know about its implementation. This allows +the database to target a POSIX filesystems storage API and an IndexedDB storage +API, and make the rest of the code agnostic about storage. PouchDB has such +mechanism (called {pouchdb-adapters}[adapters]) and Datomic has them too (called +{datomic-storage-services}[storage services]). + +This would allow the database to adapt to where it is embedded: when targeting +the browser the IndexedDB storage API would provide the persistence layer that +the database requires, and similarly the POSIX filesystem storage API would +provide the persistence layer when targeting POSIX systems (like desktops, +mobile, _etc._). + +But there's also an extra restriction that comes from by being embedded: it +needs to provide and embeddable artifact, most likely a binary library object +that exposes a C compatible FFI, similar to {sqlite-amalgamation}[how SQLite +does]. Bundling a full runtime environment is possible, but doesn't make it a +compelling solution for embedding. This rules out most languages, and leaves +us with C, Rust, Zig, and similar options that can target POSIX systems and +WebAssembly. + +=== Immutable + +:datomic: https://www.datomic.com/ +:day-of-datomic: https://vimeo.com/116315075 +:git: https://git-scm.com/ +:sqlite-limits: https://sqlite.org/limits.html +:datomic-no-history: https://docs.datomic.com/cloud/best.html#nohistory-for-high-churn + +Being immutable means that only new information is added, no in-place update +ever happens, and nothing is ever deleted. + +Having an immutable database presents us with similar trade-offs found in +persistent data structures, like lack of coordination when doing reads, caches +being always coherent, and more usage of space. + +{datomic}[*Datomic*] is the go to database example of this: it will only add +information (datoms) and allows you to query them in a multitude of ways. +Stuart Halloway calls it "accumulate-only" over +"append-only"{empty}footnote:accumulate-only[ + Video "{day-of-datomic}[Day of Datomic Part 2]" on Datomic's information + model, at time 12:28. +]: + +____ +It's accumulate-only, it is not append-only. So append-only, most people when +they say that they're implying something physical about what happens. +____ + +Also a database can be append-only and overwrite existing information with new +information, by doing clean-ups of "stale" data. I prefer to adopt the +"accumulate-only" naming and approach. + +{git}[*Git*] is another example of this: new commits are always added on top of +the previous data, and it grows by adding commits instead of replacing existing +ones. + +Git repositories can only grow in size, and that is not only an acceptable +condition, but also one of the reasons to use it. + +All this means that no in-place updates happens on data, and the database will +be much more concerned about how compact and efficiently it stores data than how +fast it does writes to disk. Being embedded, the storage limitation is either +a) how much storage the device has or b) how much storage was designed for the +application to consume. So even though the database could theoretically operate +with hundreds of TBs, a browser page or mobile application wouldn't have access +to this amount of storage. SQLite even {sqlite-limits}[says] that it does +support approximately 280 TBs of data, but those limits are untested. + +The upside of keeping everything is that you can have historical views of your +data, which is very powerful. This also means that applications should turn +this off when not +relevant{empty}footnote:no-history[ + Similar to {datomic-no-history}[Datomic's `:db/noHistory`]. +]. + +=== Syncable + +:3-way-merge: https://en.wikipedia.org/wiki/Merge_(version_control) +:git-remote-gcrypt: https://spwhitton.name/tech/code/git-remote-gcrypt/ + +This is a frequent topic when talking about offline-first solutions. When +building applications that: + +* can fully work offline, +* stores data, +* propagates that data to other application instances, + +then you'll need a conflict resolution strategy to handle all the situations +where different application instances disagree. Those application instances +could be a desktop and a browser version of the same application, or the same +mobile app in different devices. + +A three-way merge seems to be the best approach, on top of which you could add +application specific conflict resolution functions, like: + +* pick the change with higher timestamp; +* if one change is a delete, pick it; +* present the diff on the screen and allow the user to merge them. + +Some databases try to make this "easy", by choosing a strategy for you, but I've +found that different applications require different conflict resolution +strategies. Instead, the database should leave this up to the user to decide, +and provide tools for them to do it. + +{3-way-merge}[*Three-way merges in version control*] are the best example, +performing automatic merges when possible and asking the user to resolve +conflicts when they appear. + +The unit of conflict for a version control system is a line of text. The +database equivalent would probably be a single attribute, not a full entity or a +full row. + +Making all the conflict resolution logic be local should allow the database to +have encrypted remotes similar to how {git-remote-gcrypt}[git-remote-gcrypt] +adds this functionality to Git. This would enable users to sync the application +data across devices using an untrusted intermediary. + +=== Relational + +:datomic-datalog: https://docs.datomic.com/on-prem/query.html +:datomic-model: https://docs.datomic.com/cloud/whatis/data-model.html#datoms + +I want the power of relational queries on the client applications. + +Most of the arguments against traditional table-oriented relational databases +are related to write performance, but those don't apply here. The bottlenecks +for client applications usually aren't write throughput. Nobody is interested +in differentiating between 1 MB/s or 10 MB/s when you're limited to 500 MB +total. + +The relational model of the database could either be based on SQL and tables +like in SQLite, or maybe {datomic-datalog}[datalog] and {datomic-model}[datoms] +like in Datomic. + +== From aspects to values + +Now let's try to translate the aspects above into values, as suggested by Bryan +Cantrill. + +=== Portability + +Being able to target so many different platforms is a bold goal, and the +embedded nature of the database demands portability to be a core value. + +=== Integrity + +When the local database becomes the source of truth of the application, it must +provide consistency guarantees that enables applications to rely on it. + +=== Expressiveness + +The database should empower applications to slice and dice the data in any way +it wants to. + +== Next steps + +Since I can't find any database that fits these requirements, I've finally come +to terms with doing it myself. + +It's probably going to take me a few years to do it, and making it portable +between POSIX and IndexedDB will probably be the biggest challenge. I got +myself a few books on databases to start. + +I wonder if I'll ever be able to get this done. + +== External links + +:reddit: https://old.reddit.com/r/programming/comments/ijwz5b/the_database_i_wish_i_had/ +:lobsters: https://lobste.rs/s/m9vkg4/database_i_wish_i_had +:hn: https://news.ycombinator.com/item?id=24337244 +:list: https://lists.sr.ht/~euandreh/public-inbox/%3C010101744a592b75-1dce9281-f0b8-4226-9d50-fd2c7901fa72-000000%40us-west-2.amazonses.com%3E + +See discussions on {reddit}[Reddit], {lobsters}[lobsters], {hn}[HN] and {list}[a +lengthy email exchange]. diff --git a/src/content/blog/2020/08/31/database-i-with-i-had.adoc b/src/content/blog/2020/08/31/database-i-with-i-had.adoc deleted file mode 100644 index 443a54e..0000000 --- a/src/content/blog/2020/08/31/database-i-with-i-had.adoc +++ /dev/null @@ -1,257 +0,0 @@ -= The database I wish I had - -:empty: -:values-talk: https://vimeo.com/230142234 -:haskell-startup: https://www.youtube.com/watch?v=ZR3Jirqk6W8 - -I watched the talk "{values-talk}[Platform as a Reflection of Values: Joyent, -Node.js and beyond]" by Bryan Cantrill, and I think he was able to put into -words something I already felt for some time: if there's no piece of software -out there that reflects your values, it's time for you to build that -software{empty}footnote:talk-time[ - At the very end, at time 29:49. When talking about the draft of this article - with a friend, he noted that Bryan O'Sullivan (a different Bryan) says a - similar thing on his talk "{haskell-startup}[Running a startup on Haskell]", - at time 4:15. -]. - -I kind of agree with what he said, because this is already happening to me. I -long for a database with a certain set of values, and for a few years I was just -waiting for someone to finally write it. After watching his talk, Bryan is -saying to me: "time to stop waiting, and start writing it yourself". - -So let me try to give an overview of such database, and go over its values. - -== Overview - -I want a database that allows me to create decentralized client-side -applications that can sync data. - -The best one-line description I can give right now is: - -____ -It's sort of like PouchDB, Git, Datomic, SQLite and Mentat. -____ - -A more descriptive version could be: - -____ -An embedded, immutable, syncable relational database. -____ - -Let's go over what I mean by each of those aspects one by one. - -=== Embedded - -:sqlite: https://sqlite.org/index.html -:sqlite-whentouse: https://sqlite.org/whentouse.html -:pouchdb: https://pouchdb.com/ -:couchdb: https://couchdb.apache.org/ -:mentat: https://github.com/mozilla/mentat -:pouchdb-adapters: https://pouchdb.com/adapters.html -:datomic-storage-services: https://docs.datomic.com/on-prem/storage.html -:sqlite-amalgamation: https://www.sqlite.org/amalgamation.html - -I think the server-side database landscape is diverse and mature enough for my -needs (even though I end up choosing SQLite most of the time), and what I'm -after is a database to be embedded on client-side applications itself, be it -desktop, browser, mobile, _etc._ - -The purpose of such database is not to keep some local cache of data in case of -lost connectivity: we have good solutions for that already. It should serve as -the source of truth, and allow the application to work on top of it. - -{sqlite}[*SQLite*] is a great example of that: it is a very powerful relational -database that runs {sqlite-whentouse}[almost anywhere]. What I miss from it -that SQLite doesn't provide is the ability to run it on the browser: even though -you could compile it to WebAssembly, [line-through]#it assumes a POSIX -filesystem that would have to be emulated#[multiblock footnote omitted FIXME]. - -{pouchdb}[*PouchDB*] is another great example: it's a full reimplementation of -{couchdb}[CouchDB] that targets JavaScript environments, mainly the browser and -Node.js. However I want a tool that can be deployed anywhere, and not limit its -applications to places that already have a JavaScript runtime environment, or -force the developer to bundle a JavaScript runtime environment with their -application. This is true for GTK+ applications, command line programs, Android -apps, _etc._ - -{mentat}[*Mentat*] was an interesting project, but its reliance on SQLite makes -it inherit most of the downsides (and benefits too) of SQLite itself. - -Having such a requirement imposes a different approach to storage: we have to -decouple the knowledge about the intricacies of storage from the usage of -storage itself, so that a module (say query processing) can access storage -through an API without needing to know about its implementation. This allows -the database to target a POSIX filesystems storage API and an IndexedDB storage -API, and make the rest of the code agnostic about storage. PouchDB has such -mechanism (called {pouchdb-adapters}[adapters]) and Datomic has them too (called -{datomic-storage-services}[storage services]). - -This would allow the database to adapt to where it is embedded: when targeting -the browser the IndexedDB storage API would provide the persistence layer that -the database requires, and similarly the POSIX filesystem storage API would -provide the persistence layer when targeting POSIX systems (like desktops, -mobile, _etc._). - -But there's also an extra restriction that comes from by being embedded: it -needs to provide and embeddable artifact, most likely a binary library object -that exposes a C compatible FFI, similar to {sqlite-amalgamation}[how SQLite -does]. Bundling a full runtime environment is possible, but doesn't make it a -compelling solution for embedding. This rules out most languages, and leaves -us with C, Rust, Zig, and similar options that can target POSIX systems and -WebAssembly. - -=== Immutable - -:datomic: https://www.datomic.com/ -:day-of-datomic: https://vimeo.com/116315075 -:git: https://git-scm.com/ -:sqlite-limits: https://sqlite.org/limits.html -:datomic-no-history: https://docs.datomic.com/cloud/best.html#nohistory-for-high-churn - -Being immutable means that only new information is added, no in-place update -ever happens, and nothing is ever deleted. - -Having an immutable database presents us with similar trade-offs found in -persistent data structures, like lack of coordination when doing reads, caches -being always coherent, and more usage of space. - -{datomic}[*Datomic*] is the go to database example of this: it will only add -information (datoms) and allows you to query them in a multitude of ways. -Stuart Halloway calls it "accumulate-only" over -"append-only"{empty}footnote:accumulate-only[ - Video "{day-of-datomic}[Day of Datomic Part 2]" on Datomic's information - model, at time 12:28. -]: - -____ -It's accumulate-only, it is not append-only. So append-only, most people when -they say that they're implying something physical about what happens. -____ - -Also a database can be append-only and overwrite existing information with new -information, by doing clean-ups of "stale" data. I prefer to adopt the -"accumulate-only" naming and approach. - -{git}[*Git*] is another example of this: new commits are always added on top of -the previous data, and it grows by adding commits instead of replacing existing -ones. - -Git repositories can only grow in size, and that is not only an acceptable -condition, but also one of the reasons to use it. - -All this means that no in-place updates happens on data, and the database will -be much more concerned about how compact and efficiently it stores data than how -fast it does writes to disk. Being embedded, the storage limitation is either -a) how much storage the device has or b) how much storage was designed for the -application to consume. So even though the database could theoretically operate -with hundreds of TBs, a browser page or mobile application wouldn't have access -to this amount of storage. SQLite even {sqlite-limits}[says] that it does -support approximately 280 TBs of data, but those limits are untested. - -The upside of keeping everything is that you can have historical views of your -data, which is very powerful. This also means that applications should turn -this off when not -relevant{empty}footnote:no-history[ - Similar to {datomic-no-history}[Datomic's `:db/noHistory`]. -]. - -=== Syncable - -:3-way-merge: https://en.wikipedia.org/wiki/Merge_(version_control) -:git-remote-gcrypt: https://spwhitton.name/tech/code/git-remote-gcrypt/ - -This is a frequent topic when talking about offline-first solutions. When -building applications that: - -* can fully work offline, -* stores data, -* propagates that data to other application instances, - -then you'll need a conflict resolution strategy to handle all the situations -where different application instances disagree. Those application instances -could be a desktop and a browser version of the same application, or the same -mobile app in different devices. - -A three-way merge seems to be the best approach, on top of which you could add -application specific conflict resolution functions, like: - -* pick the change with higher timestamp; -* if one change is a delete, pick it; -* present the diff on the screen and allow the user to merge them. - -Some databases try to make this "easy", by choosing a strategy for you, but I've -found that different applications require different conflict resolution -strategies. Instead, the database should leave this up to the user to decide, -and provide tools for them to do it. - -{3-way-merge}[*Three-way merges in version control*] are the best example, -performing automatic merges when possible and asking the user to resolve -conflicts when they appear. - -The unit of conflict for a version control system is a line of text. The -database equivalent would probably be a single attribute, not a full entity or a -full row. - -Making all the conflict resolution logic be local should allow the database to -have encrypted remotes similar to how {git-remote-gcrypt}[git-remote-gcrypt] -adds this functionality to Git. This would enable users to sync the application -data across devices using an untrusted intermediary. - -=== Relational - -:datomic-datalog: https://docs.datomic.com/on-prem/query.html -:datomic-model: https://docs.datomic.com/cloud/whatis/data-model.html#datoms - -I want the power of relational queries on the client applications. - -Most of the arguments against traditional table-oriented relational databases -are related to write performance, but those don't apply here. The bottlenecks -for client applications usually aren't write throughput. Nobody is interested -in differentiating between 1 MB/s or 10 MB/s when you're limited to 500 MB -total. - -The relational model of the database could either be based on SQL and tables -like in SQLite, or maybe {datomic-datalog}[datalog] and {datomic-model}[datoms] -like in Datomic. - -== From aspects to values - -Now let's try to translate the aspects above into values, as suggested by Bryan -Cantrill. - -=== Portability - -Being able to target so many different platforms is a bold goal, and the -embedded nature of the database demands portability to be a core value. - -=== Integrity - -When the local database becomes the source of truth of the application, it must -provide consistency guarantees that enables applications to rely on it. - -=== Expressiveness - -The database should empower applications to slice and dice the data in any way -it wants to. - -== Next steps - -Since I can't find any database that fits these requirements, I've finally come -to terms with doing it myself. - -It's probably going to take me a few years to do it, and making it portable -between POSIX and IndexedDB will probably be the biggest challenge. I got -myself a few books on databases to start. - -I wonder if I'll ever be able to get this done. - -== External links - -:reddit: https://old.reddit.com/r/programming/comments/ijwz5b/the_database_i_wish_i_had/ -:lobsters: https://lobste.rs/s/m9vkg4/database_i_wish_i_had -:hn: https://news.ycombinator.com/item?id=24337244 -:list: https://lists.sr.ht/~euandreh/public-inbox/%3C010101744a592b75-1dce9281-f0b8-4226-9d50-fd2c7901fa72-000000%40us-west-2.amazonses.com%3E - -See discussions on {reddit}[Reddit], {lobsters}[lobsters], {hn}[HN] and {list}[a -lengthy email exchange]. diff --git a/src/content/blog/2020/10/19/feature-flags.adoc b/src/content/blog/2020/10/19/feature-flags.adoc index c9adc8a..e0ce078 100644 --- a/src/content/blog/2020/10/19/feature-flags.adoc +++ b/src/content/blog/2020/10/19/feature-flags.adoc @@ -1,7 +1,7 @@ = Feature flags: differences between backend, frontend and mobile :empty: -:slides: link:../../../../slides/2020/10/19/feature-flags.html +:slides: link:../../../../slides/2020/10/19/feature-flags.html FIXME :fowler-article: https://martinfowler.com/articles/feature-toggles.html _This article is derived from a {slides}[presentation] on the same subject._ diff --git a/src/content/blog/2020/11/14/local-first-review.adoc b/src/content/blog/2020/11/14/local-first-review.adoc index 420c886..1c086d3 100644 --- a/src/content/blog/2020/11/14/local-first-review.adoc +++ b/src/content/blog/2020/11/14/local-first-review.adoc @@ -1,7 +1,7 @@ = Local-First Software: article review :empty: -:presentation: link:../../../../slides/2020/11/14/local-first.html +:presentation: link:../../../../slides/2020/11/14/local-first.html FIXME :reviewed-article: https://martin.kleppmann.com/papers/local-first.pdf _This article is derived from a {presentation}[presentation] given at a Papers diff --git a/src/content/tils/2020/09/04/cli-email-fun-profit.adoc b/src/content/tils/2020/09/04/cli-email-fun-profit.adoc new file mode 100644 index 0000000..5476fac --- /dev/null +++ b/src/content/tils/2020/09/04/cli-email-fun-profit.adoc @@ -0,0 +1,75 @@ += Send emails using the command line for fun and profit! + +:ssmtp: https://wiki.archlinux.org/index.php/SSMTP +:mailutils: https://mailutils.org/ + +Here are a few reasons why: + +. send yourself and other people notification of cronjobs, scripts runs, CI + jobs, _etc._ +. leverage the POSIX pipe `|`, and pipe emails away! +. because you can. + +Reason 3 is the fun part, reasons 1 and 2 are the profit part. + +First {ssmpt}[install and configure SSMTP] for using, say, Gmail as the email +server: + +[source,shell] +---- +# file /etc/ssmtp/ssmtp.conf +FromLineOverride=YES +MailHub=smtp.gmail.com:587 +UseSTARTTLS=YES +UseTLS=YES +rewriteDomain=gmail.com +root=username@gmail.com +AuthUser=username +AuthPass=password +---- + +Now install {mailutils}[GNU Mailutils] (`sudo apt-get install mailutils` or the +equivalent on your OS), and send yourself your first email: + +[source,shell] +---- +echo body | mail -aFrom:email@example.com email@example.com -s subject +---- + +And that's about it, you've got mail. Here are some more places where it might +be applicable: + +[source,shell] +---- +# report a backup cronjob, attaching logs +set -e + +finish() { + status=$? + if [[ $status = 0 ]]; then + STATUS="SUCCESS (status $status)" + else + STATUS="FAILURE (status $status)" + fi + + mail user@example.com \ + -s "Backup job report on $(hostname): ${STATUS}" \ + --content-type 'text/plain; charset=utf-8' \ + -A"$LOG_FILE" <<< 'The log report is in the attachment.' +} +trap finish EXIT + +do-long-backup-cmd-here +---- + +.... +# share the output of a cmd with someone +some-program | mail someone@example.com -s "The weird logs that I was talking about" +.... + +...and so on. + +You may consider adding a `alias mail='mail -aFrom:email@example.com'` so you +don't keep re-entering the ``From:'' part. + +Send yourself some emails to see it working! diff --git a/src/content/tils/2020/09/04/email-cli-fun-profit.adoc b/src/content/tils/2020/09/04/email-cli-fun-profit.adoc deleted file mode 100644 index 5476fac..0000000 --- a/src/content/tils/2020/09/04/email-cli-fun-profit.adoc +++ /dev/null @@ -1,75 +0,0 @@ -= Send emails using the command line for fun and profit! - -:ssmtp: https://wiki.archlinux.org/index.php/SSMTP -:mailutils: https://mailutils.org/ - -Here are a few reasons why: - -. send yourself and other people notification of cronjobs, scripts runs, CI - jobs, _etc._ -. leverage the POSIX pipe `|`, and pipe emails away! -. because you can. - -Reason 3 is the fun part, reasons 1 and 2 are the profit part. - -First {ssmpt}[install and configure SSMTP] for using, say, Gmail as the email -server: - -[source,shell] ----- -# file /etc/ssmtp/ssmtp.conf -FromLineOverride=YES -MailHub=smtp.gmail.com:587 -UseSTARTTLS=YES -UseTLS=YES -rewriteDomain=gmail.com -root=username@gmail.com -AuthUser=username -AuthPass=password ----- - -Now install {mailutils}[GNU Mailutils] (`sudo apt-get install mailutils` or the -equivalent on your OS), and send yourself your first email: - -[source,shell] ----- -echo body | mail -aFrom:email@example.com email@example.com -s subject ----- - -And that's about it, you've got mail. Here are some more places where it might -be applicable: - -[source,shell] ----- -# report a backup cronjob, attaching logs -set -e - -finish() { - status=$? - if [[ $status = 0 ]]; then - STATUS="SUCCESS (status $status)" - else - STATUS="FAILURE (status $status)" - fi - - mail user@example.com \ - -s "Backup job report on $(hostname): ${STATUS}" \ - --content-type 'text/plain; charset=utf-8' \ - -A"$LOG_FILE" <<< 'The log report is in the attachment.' -} -trap finish EXIT - -do-long-backup-cmd-here ----- - -.... -# share the output of a cmd with someone -some-program | mail someone@example.com -s "The weird logs that I was talking about" -.... - -...and so on. - -You may consider adding a `alias mail='mail -aFrom:email@example.com'` so you -don't keep re-entering the ``From:'' part. - -Send yourself some emails to see it working! -- cgit v1.2.3