Compute each selector once, not seven times

mkdeps.sh asked the same questions over and over: "pages" is built
on four of the other selectors, the tagged list mkwb reads asks for
all seven again, and "collections" alone was computed nine times, a
find and a grep and a sort apiece.  It spawned 372 processes to
print 2450 lines.

Every answer is now computed once, up front, into a file named
after it, and read back from there.  The three hand-rolled caches
go with it: they were the same idea, done three times, and the
loop that fills them says the dependency order out loud.  Two
smaller things fell out on the way: collections ran dirname once
per collection where sed does it in one, and articles ran one find
per collection where find takes them all at once.

The order the loop fills them in is not decoration.  A pipeline's
stages run side by side, and only_masters and with_translations
are two stages of one selector that both read what po4a writes ---
so an answer computed where it is first needed can be read while
it is still half-written, which is a race that shows up as an
empty list and a deps.mk missing half the site.  Filled in order,
up front, nothing is ever read while it is being written.

372 processes to 109, 0.47s to 0.24s, byte-identical deps.mk over
a dozen runs.  The floor now is exec(2): the tools this reaches
for cost 1.1 to 2.5ms each to start, and a hundred of them is the
whole of the remaining runtime.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KxLpaNQThKfMG3ecCJptP9

1 files changed, 76 insertions, 78 deletions

diff --git a/mkdeps.sh b/mkdeps.sh
index c280671..5cbe130 100755
--- a/mkdeps.sh
+++ b/mkdeps.sh
@@ -39,13 +39,28 @@ normalize() {
 	' | sort
 }
 
+## Every selector below is asked for more than once --- "pages"
+## alone is built on four of the others, and the tagged list at
+## the end asks for all seven again --- over a tree of a couple of
+## thousand files.  So each is computed once, by the loop at the
+## end of this section, into a file named after it; everything
+## else reads it back with memo().  The answer has to live in a
+## file rather than a variable: every stage of a pipeline runs in
+## a subshell, and what one sets there is gone by the next.
+CACHEDIR="$(mktemp -d)"
+trap 'rm -rf "$CACHEDIR"' EXIT
+
+memo() {
+	cat "$CACHEDIR/$1"
+}
+
 ## Every page there is: what the repository carries, plus what
 ## po4a writes from it.  A translation need not exist yet -- on a
 ## fresh clone none of them do -- so it is named, not found.
 universe() {
 	{
 		find src/content -type f -name '*.adoc'
-		page_translations
+		memo page_translations
 	} | normalize | uniq
 }
 
@@ -57,19 +72,22 @@ homes() {
 }
 
 collections() {
+	## dirname(1) did the last step, one process per collection
+	## and once for every time this was asked; every path here has
+	## a trailing component to drop, so sed says it in one.
 	grep -l '^:type: collection$' src/content/*/*/index.adoc |
-		normalize | only_masters | xargs -n1 dirname
+		normalize | only_masters | sed 's|/[^/]*$||'
 }
 
 indexes() {
-	all_collections | sed 's|$|/index.adoc|' | normalize
+	memo all_collections | sed 's|$|/index.adoc|' | normalize
 }
 
 ## Every collection there is, translations included: the rules
 ## below emit a listing, a feed and a sortdata list for each, and
 ## a translated collection needs all three as much as its master.
 all_collections() {
-	collections | sed 's|$|/index.adoc|' | with_translations |
+	memo collections | sed 's|$|/index.adoc|' | with_translations |
 		sed 's|/index.adoc$||' | normalize
 }
 
@@ -79,7 +97,12 @@ categories() {
 }
 
 articles() {
-	collections | xargs -I% find % -mindepth 3 -name '*.adoc' |
+	## One find over every collection at once: with "xargs -I%"
+	## it was one find per collection, for the same answer.
+	# shellcheck disable=SC2046 # a collection path holds no blanks
+	set -- $(memo collections)
+	[ $# -gt 0 ] || return 0
+	find "$@" -mindepth 3 -name '*.adoc' |
 		normalize | only_masters | with_translations | normalize
 }
 
@@ -89,10 +112,10 @@ slides() {
 
 ## Whatever no other rule claimed, at any depth.
 pages() {
-	claimed="$(mktemp)"
-	{ homes; indexes; categories; articles; } | sort -u > "$claimed"
-	universe | grep -vxF -f "$claimed"
-	rm -f "$claimed"
+	claimed="$CACHEDIR/claimed"
+	sort -u "$CACHEDIR/homes" "$CACHEDIR/indexes" \
+		"$CACHEDIR/categories" "$CACHEDIR/articles" > "$claimed"
+	memo universe | grep -vxF -f "$claimed"
 }
 
 media() {
@@ -127,9 +150,9 @@ shortener() {
 }
 
 extras() {
-	media
-	tarballs
-	shortener
+	memo media
+	memo tarballs
+	memo shortener
 }
 
 ## Which pages are one another's translations.  po4a already says
@@ -137,21 +160,15 @@ extras() {
 ## that can quietly disagree with the first.
 ## The languages po4a translates into: whichever .po files exist.
 langs() {
-	if [ -z "${LANGS_CACHE:-}" ]; then
-		LANGS_CACHE="$(
-			find po -maxdepth 1 -name '*.po' |
-				sed 's|.*/||; s|\.po$||' | sort
-		)"
-		export LANGS_CACHE
-	fi
-	echo "$LANGS_CACHE"
+	find po -maxdepth 1 -name '*.po' |
+		sed 's|.*/||; s|\.po$||' | sort
 }
 
 ## One stanza per line, master first, then the files po4a writes
 ## from it.  "$lang:" stands for every language, and "add_$lang:"
 ## names an addendum rather than an output.
 po4a_sets() {
-	langs | tr '\n' ' ' | {
+	memo langs | tr '\n' ' ' | {
 		## read hits EOF on the last field and reports it, which
 		## under set -e would end the block before awk ran.
 		read -r all || true
@@ -186,7 +203,7 @@ po4a_sets() {
 
 ## Everything po4a writes: every field of a stanza but the first.
 translations() {
-	cut -d' ' -f2- "$(po4a_cache)" |
+	cut -d' ' -f2- "$CACHEDIR/po4a_sets" |
 		tr ' ' '\n' | grep . | normalize
 }
 
@@ -194,39 +211,19 @@ translations() {
 ## same page in another language; the data files under src/i18n
 ## are not pages.
 page_translations() {
-	translations | grep '\.adoc$'
+	memo translations | grep '\.adoc$'
 }
 
 ## Drops the translations from a list of paths, so a rule reads
 ## the masters whether or not po4a has run.
-## po4a.cfg does not change while this runs, so parse it once:
-## every selector below asks for the same answer.
-po4a_cache() {
-	if [ -z "${PO4A_CACHE:-}" ]; then
-		PO4A_CACHE="$(mktemp)"
-		po4a_sets > "$PO4A_CACHE"
-		export PO4A_CACHE
-	fi
-	echo "$PO4A_CACHE"
-}
-
-translations_cache() {
-	if [ -z "${TRANSLATIONS_CACHE:-}" ]; then
-		TRANSLATIONS_CACHE="$(mktemp)"
-		translations > "$TRANSLATIONS_CACHE"
-		export TRANSLATIONS_CACHE
-	fi
-	echo "$TRANSLATIONS_CACHE"
-}
-
 only_masters() {
 	awk 'NR == FNR { t[$0]; next } !($0 in t)' \
-		"$(translations_cache)" -
+		"$CACHEDIR/translations" -
 }
 
 ## Reads masters on stdin, writes them and their translations.
 with_translations() {
-	awk -v sets="$(po4a_cache)" '
+	awk -v sets="$CACHEDIR/po4a_sets" '
 		BEGIN {
 			while ((getline line < sets) > 0) {
 				n = split(line, f, /[ \t]+/)
@@ -247,44 +244,45 @@ with_translations() {
 	'
 }
 
-## Prime the caches here, in this shell.  Every pipeline stage
-## below runs in a subshell, so a cache filled inside one is gone
-## by the next: filling them once, up here, is what makes them
-## caches rather than seven identical computations.
-PO4A_CACHE="$(mktemp)"
-po4a_sets > "$PO4A_CACHE"
-TRANSLATIONS_CACHE="$(mktemp)"
-translations > "$TRANSLATIONS_CACHE"
-LANGS_CACHE="$(find po -maxdepth 1 -name '*.po' |
-	sed 's|.*/||; s|\.po$||' | sort)"
-export PO4A_CACHE TRANSLATIONS_CACHE LANGS_CACHE
-trap 'rm -f "$PO4A_CACHE" "$TRANSLATIONS_CACHE"' EXIT
+## Computed here, in this shell, one selector after another.
+## Every stage of a pipeline runs at the same time as the stages
+## beside it, and two stages of one selector can ask for the same
+## answer --- only_masters and with_translations both read what
+## po4a writes --- so an answer computed where it is first needed
+## could be read while it was still half-written.  The order is
+## the order they are built on one another; put one too early and
+## it reads a file that is not there, and says so.
+for sel in langs po4a_sets translations page_translations universe \
+	collections all_collections indexes homes categories articles \
+	slides media tarballs shortener music pages; do
+	"$sel" > "$CACHEDIR/$sel"
+done
 
-pages                                      | varlist      'pages.adoc'
-homes                                      | varlist      'homes.adoc'
-articles                                   | varlist   'articles.adoc'
-slides                                     | varlist 'slides.eslaides'
-categories                                 | varlist 'categories.adoc'
-indexes                                    | varlist    'indexes.adoc'
-indexes | sed 's|/index\.adoc$|/feed.xml|' | varlist 'feeds.xml'
+memo pages                                      | varlist      'pages.adoc'
+memo homes                                 | varlist      'homes.adoc'
+memo articles                              | varlist   'articles.adoc'
+memo slides                                | varlist 'slides.eslaides'
+memo categories                            | varlist 'categories.adoc'
+memo indexes                               | varlist    'indexes.adoc'
+memo indexes | sed 's|/index\.adoc$|/feed.xml|' | varlist 'feeds.xml'
 find src/content/img/ -name '*.svg' | sort | varlist 'images.svg'
-media                                      | varlist 'sources.media'
-tarballs                                   | varlist 'sources.tarballs'
+memo media                                 | varlist 'sources.media'
+memo tarballs                              | varlist 'sources.tarballs'
 extras                                     | varlist 'sources.extras'
 find po/*.po po/*.pot                      | varlist 'sources.po'
-music                                      | varlist 'music.ly'
+memo music                                 | varlist 'music.ly'
 find src/content/music/*.ogg 2>/dev/null   | varlist 'music.ogg'
 find src/content/music/*.ly.include 2>/dev/null | \
 	varlist 'music.include'
 
 # lilypond emits the score alongside the MIDI, from one run.
-music | sed 's/^\(.*\)\.ly$/\1.pdf:\t\1.midi/'
+memo music | sed 's/^\(.*\)\.ly$/\1.pdf:\t\1.midi/'
 
 ## A short link points into the built tree, so on a fresh clone it
 ## dangles, and make --- which stats through a symlink --- reports no
 ## rule for a file that is right there.  Name the page each one points
 ## at as what the link is made from.
-shortener | while read -r link; do
+memo shortener | while read -r link; do
 	target="$(dirname "$link")/$(readlink "$link")"
 	printf '%s:\t%s\n' "$link" \
 		"$(printf '%s\n' "$target" | normalize)"
@@ -295,20 +293,20 @@ done
 ## built from.  Finding them is the site's half: only it knows
 ## where its pages live, or that it keeps decks and no music.
 {
-	pages      | sed 's/^/page\t/'
-	homes      | sed 's/^/home\t/'
-	articles   | sed 's/^/article\t/'
-	indexes    | sed 's/^/index\t/'
-	categories | sed 's/^/category\t/'
-	slides     | sed 's/^/slide\t/'
-	media      | sed 's/^/media\t/'
+	memo pages | sed 's/^/page\t/'
+	memo homes | sed 's/^/home\t/'
+	memo articles | sed 's/^/article\t/'
+	memo indexes | sed 's/^/index\t/'
+	memo categories | sed 's/^/category\t/'
+	memo slides | sed 's/^/slide\t/'
+	memo media | sed 's/^/media\t/'
 } | mkwbg deps src/config.json
 
 ## What the translator writes.  Naming them here is what lets them
 ## be derived: deps.mk is written before make runs, so a translation
 ## that is not named cannot be built, and would have to be
 ## committed instead.
-translations | varlist 'translations'
+memo translations | varlist 'translations'
 
 ## One run writes all of them, so they hang off a stamp rather
 ## than each invoking the translator for itself.  --no-update
@@ -325,7 +323,7 @@ printf '$(translations): src/i18n.sentinel\n\n'
 printf '\ntranslations.txt = src/translations.txt\n\n'
 printf 'src/translations.txt: deps.mk\n'
 printf '\tprintf %s \\\n' "'%s\\n'"
-po4a_sets | grep '\.adoc' | sed "s/^/\t\t'/; s/\$/' \\\\/"
+memo po4a_sets | grep '\.adoc' | sed "s/^/\t\t'/; s/\$/' \\\\/"
 printf '\t\t> $@\n\n'