#!/bin/sh set -eu export LANG=POSIX.UTF-8 ## -m so a path need not exist: po4a's outputs are named here ## before po4a has written them --- naming them is what the rules ## being generated are for --- and realpath without it fails on ## the first one, taking the whole script with it under set -e. ## For a path that does exist the answer is the same either way. normalize() { ## realpath(1) did this, but it is a page of string work and ## this runs it dozens of times: doing it here saves a ## process and most of the time. awk ' { n = split($0, c, "/") top = 0 for (i = 1; i <= n; i++) { if (c[i] == "" || c[i] == ".") { continue } if (c[i] == ".." && top > 0 && out[top] != "..") { top-- continue } out[++top] = c[i] } p = "" for (i = 1; i <= top; i++) { p = p (i > 1 ? "/" : "") out[i] } if (p != "") { print p } } ' | sort } ## Every selector below is asked for more than once --- "pages" ## alone is built on four of the others, and the tagged list at ## the end asks for all seven again --- over a tree of a couple of ## thousand files. So each is computed once, by the loop at the ## end of this section, into a file named after it; everything ## else reads it back with memo(). The answer has to live in a ## file rather than a variable: every stage of a pipeline runs in ## a subshell, and what one sets there is gone by the next. CACHEDIR="$(mktemp -d)" trap 'rm -rf "$CACHEDIR"' EXIT memo() { cat "$CACHEDIR/$1" } ## Every page there is: what the repository carries, plus what ## po4a writes from it. A translation need not exist yet -- on a ## fresh clone none of them do -- so it is named, not found. universe() { { find src/content -type f -name '*.adoc' memo page_translations } | normalize | uniq } ## The classifying rules read the masters, which the repository ## always carries; a translation is whatever its master was. homes() { find src/content/*/index.adoc -type f | normalize | only_masters | with_translations | normalize } collections() { ## dirname(1) did the last step, one process per collection ## and once for every time this was asked; every path here has ## a trailing component to drop, so sed says it in one. grep -l '^:type: collection$' src/content/*/*/index.adoc | normalize | only_masters | sed 's|/[^/]*$||' } indexes() { memo all_collections | sed 's|$|/index.adoc|' | normalize } ## Every collection there is, translations included: the rules ## below emit a listing, a feed and a sortdata list for each, and ## a translated collection needs all three as much as its master. all_collections() { memo collections | sed 's|$|/index.adoc|' | with_translations | sed 's|/index.adoc$||' | normalize } categories() { grep -l '^:type: categories$' src/content/*/*/*.adoc | normalize | only_masters | with_translations | normalize } articles() { ## One find over every collection at once: with "xargs -I%" ## it was one find per collection, for the same answer. # shellcheck disable=SC2046 # a collection path holds no blanks set -- $(memo collections) [ $# -gt 0 ] || return 0 find "$@" -mindepth 3 -name '*.adoc' | normalize | only_masters | with_translations | normalize } slides() { find src/content -type f -name '*.eslaides' | normalize } ## Whatever no other rule claimed, at any depth. pages() { claimed="$CACHEDIR/claimed" sort -u "$CACHEDIR/homes" "$CACHEDIR/indexes" \ "$CACHEDIR/categories" "$CACHEDIR/articles" > "$claimed" memo universe | grep -vxF -f "$claimed" } media() { # src/content/music is excluded: those .ogg files are score # renderings, not published media, and the old site never # offered them over BitTorrent. find src/content/ -path 'src/content/music' -prune -or \ \( \( -type f -and \! -type l \) -and \( \ -name '*.flac' -or \ -name '*.ogg' -or \ -name '*.webm' \ \) -print \) | normalize } ## A glob that matches nothing makes find exit non-zero, which ## under set -e would end the script. There is no music yet. music() { find src/content/music/*.ly 2>/dev/null | normalize || true } tarballs() { find src/content/ \( -type f -and \! -type l \) -and \( \ -name '*.tar.gz' \ \) | normalize } ## Short paths that stand in for a page's own, kept as symlinks ## beside it. Guarded like music() above: a site need not have any, ## and find exits non-zero on a directory that is not there. shortener() { find src/content/l -type l 2>/dev/null | normalize || true } extras() { memo media memo tarballs memo shortener } ## Which pages are one another's translations. po4a already says ## so, set by set, so read it there rather than keep a second list ## that can quietly disagree with the first. ## The languages po4a translates into: whichever .po files exist. langs() { find po -maxdepth 1 -name '*.po' | sed 's|.*/||; s|\.po$||' | sort } ## One stanza per line, master first, then the files po4a writes ## from it. "$lang:" stands for every language, and "add_$lang:" ## names an addendum rather than an output. po4a_sets() { memo langs | tr '\n' ' ' | { ## read hits EOF on the last field and reports it, which ## under set -e would end the block before awk ran. read -r all || true awk -v langs="$all" ' /^\[type:/ { set = "" } /^\[type:/ || cont { line = $0 sub(/^\[type:[^]]*\][ \t]*/, "", line) cont = (line ~ /\\$/) sub(/[ \t]*\\$/, "", line) n = split(line, f, /[ \t]+/) nl = split(langs, L, " ") for (i = 1; i <= n; i++) { if (f[i] == "" || f[i] ~ /^add_/) continue if (f[i] ~ /^\$lang:/) { sub(/^\$lang:/, "", f[i]) for (j = 1; j <= nl; j++) { t = f[i] gsub(/\$lang/, L[j], t) set = set " " t } continue } sub(/^[a-z]+:/, "", f[i]) set = set (set == "" ? "" : " ") f[i] } if (!cont && set != "") { print set; set = "" } } ' po/po4a.cfg } } ## Everything po4a writes: every field of a stanza but the first. translations() { cut -d' ' -f2- "$CACHEDIR/po4a_sets" | tr ' ' '\n' | grep . | normalize } ## Just the pages among them. The switcher offers a reader the ## same page in another language; the data files under src/i18n ## are not pages. page_translations() { memo translations | grep '\.adoc$' } ## Drops the translations from a list of paths, so a rule reads ## the masters whether or not po4a has run. only_masters() { awk 'NR == FNR { t[$0]; next } !($0 in t)' \ "$CACHEDIR/translations" - } ## Reads masters on stdin, writes them and their translations. with_translations() { awk -v sets="$CACHEDIR/po4a_sets" ' BEGIN { while ((getline line < sets) > 0) { n = split(line, f, /[ \t]+/) for (i = 2; i <= n; i++) { t[f[1]] = t[f[1]] " " f[i] } } } { print if ($0 in t) { n = split(t[$0], g, " ") for (i = 1; i <= n; i++) { print g[i] } } } ' } ## Computed here, in this shell, one selector after another. ## Every stage of a pipeline runs at the same time as the stages ## beside it, and two stages of one selector can ask for the same ## answer --- only_masters and with_translations both read what ## po4a writes --- so an answer computed where it is first needed ## could be read while it was still half-written. The order is ## the order they are built on one another; put one too early and ## it reads a file that is not there, and says so. for sel in langs po4a_sets translations page_translations universe \ collections all_collections indexes homes categories articles \ slides media tarballs shortener music pages; do "$sel" > "$CACHEDIR/$sel" done memo pages | varlist 'pages.adoc' memo homes | varlist 'homes.adoc' memo articles | varlist 'articles.adoc' memo slides | varlist 'slides.eslaides' memo categories | varlist 'categories.adoc' memo indexes | varlist 'indexes.adoc' memo indexes | sed 's|/index\.adoc$|/feed.xml|' | varlist 'feeds.xml' find src/content/img/ -name '*.svg' | sort | varlist 'images.svg' memo media | varlist 'sources.media' memo tarballs | varlist 'sources.tarballs' extras | varlist 'sources.extras' find po/*.po po/*.pot | varlist 'sources.po' memo music | varlist 'music.ly' find src/content/music/*.ogg 2>/dev/null | varlist 'music.ogg' find src/content/music/*.ly.include 2>/dev/null | \ varlist 'music.include' # lilypond emits the score alongside the MIDI, from one run. memo music | sed 's/^\(.*\)\.ly$/\1.pdf:\t\1.midi/' ## A short link points into the built tree, so on a fresh clone it ## dangles, and make --- which stats through a symlink --- reports no ## rule for a file that is right there. Name the page each one points ## at as what the link is made from. memo shortener | while read -r link; do target="$(dirname "$link")/$(readlink "$link")" printf '%s:\t%s\n' "$link" \ "$(printf '%s\n' "$target" | normalize)" done ## What this site has, for mkwb to say what each artifact is ## built from. Finding them is the site's half: only it knows ## where its pages live, or that it keeps decks and no music. { memo pages | sed 's/^/page\t/' memo homes | sed 's/^/home\t/' memo articles | sed 's/^/article\t/' memo indexes | sed 's/^/index\t/' memo categories | sed 's/^/category\t/' memo slides | sed 's/^/slide\t/' memo media | sed 's/^/media\t/' } | mkwbg deps src/config.json ## What the translator writes. Naming them here is what lets them ## be derived: deps.mk is written before make runs, so a translation ## that is not named cannot be built, and would have to be ## committed instead. memo translations | varlist 'translations' ## One run writes all of them, so they hang off a stamp rather ## than each invoking the translator for itself. --no-update ## because a build writes translations and never catalogues; "make ## i18n" is what brings those up to the masters. printf 'src/i18n.sentinel: po/po4a.cfg %s\n' "$(find po -maxdepth 1 -name '*.po' | sort | tr '\n' ' ')" printf '\tmaritacag --no-update po/po4a.cfg\n' printf '\ttouch $@\n\n' printf '$(translations): src/i18n.sentinel\n\n' ## The map mkwb conf reads to link a page to its translations, ## written out here so it cannot drift from po4a.cfg. printf '\ntranslations.txt = src/translations.txt\n\n' printf 'src/translations.txt: deps.mk\n' printf '\tprintf %s \\\n' "'%s\\n'" memo po4a_sets | grep '\.adoc' | sed "s/^/\t\t'/; s/\$/' \\\\/" printf '\t\t> $@\n\n' ## The canonical rules, inlined so no include is needed ## at make time. echo mkwbg rules ## Everything mkwb renders reads the site's wording table, which ## po4a writes. The rules above define the variables, so this ## comes after them. It has to name what renders rather than ## what a page is made of: an article never writes a .htmlheader ## of its own, so naming that missed every one of them. printf '\n$(sources.html) $(sources.json): src/i18n.sentinel\n' printf '$(listings.htmllisting) $(categories.xml): src/i18n.sentinel\n' printf '$(articles.feedentry): src/i18n.sentinel\n'