diff options
| -rw-r--r-- | .gitignore | 6 | ||||
| -rw-r--r-- | _plugins/lint-hook.rb | 128 | ||||
| -rwxr-xr-x | scripts/assert-content.sh | 269 | ||||
| -rw-r--r-- | site.json | 141 | ||||
| -rwxr-xr-x | tests.sh | 1 |
5 files changed, 132 insertions, 413 deletions
@@ -5,4 +5,8 @@ /.bundle/ /_site/ /.jekyll-cache/ -/images/graphviz/
\ No newline at end of file +/images/graphviz/ + +# Media +*.ogg +*.torrent
\ No newline at end of file diff --git a/_plugins/lint-hook.rb b/_plugins/lint-hook.rb index 6d883cd..57c2e27 100644 --- a/_plugins/lint-hook.rb +++ b/_plugins/lint-hook.rb @@ -1,6 +1,8 @@ require 'set' -IGNORED_PAGES = Set['site.json', 'sitemap.xml'] +IGNORED_PAGES = Set['sitemap.xml'] +LANGS = Set['en', 'pt', 'fr', 'eo'] # jp zh es de +TRACKERS = '-a udp://tracker.coppersurfer.tk:6969/announce -a udp://tracker.ccc.de:80/announce -a udp://tracker.publicbt.com:80 -a udp://tracker.istole.it:80 -a http://tracker.openbittorrent.com:80/announce -a http://tracker.ipv6tracker.org:80/announce' module Jekyll class Linter < Generator @@ -32,8 +34,132 @@ module Jekyll end end + def slugify(s) + s.ljust(100) + .gsub(/[\W]+/, ' ') + .strip + .gsub(/\s\s+/, '-') + .downcase + .gsub(' ', '-') + .gsub('_', '-') + end + + def assert(value, message) + unless value + raise message + end + value + end + + def assert_field(document, field) + f = document.data[field] + raise "Undefined '#{field}' for #{document.path}" unless f + f + end + + COLLECTION_LAYOUTS = { + 'page' => 'page', + 'articles' => 'post', + 'pastebins' => 'post', + 'tils' => 'post', + 'slides' => 'slides', + 'podcasts' => 'cast', + 'screencasts' => 'cast' + } + + def assert_frontmatter_fields(config, name, document) + title = assert_field document, 'title' + lang = assert_field document, 'lang' + ref = assert_field document, 'ref' + layout = assert_field document, 'layout' + date = document.date.strftime('%Y-%m-%d') unless layout == 'page' + slug = layout == 'page' ? ref : assert_field(document, 'slug') + extension = name == 'slides' ? 'slides' : 'md' + + unless LANGS.member? lang + raise "Invalid lang '#{lang}' in #{document.path}" + end + + if COLLECTION_LAYOUTS[name] != layout + raise "Layout mismatch: expected '#{COLLECTION_LAYOUTS[name]}', got '#{layout}' for #{document.path}" + end + + if lang == 'en' + unless ['index', 'root', 'tils'].include? ref + if slugify(title) != ref then + raise "#{ref} isn't a slug of the title.\nref: '#{ref}'\ntitle slug: '#{slugify(title)}'" + p slugify(title) + end + end + end + + unless layout == 'page' then + path = "_#{name}/#{date}-#{slug}.#{extension}" + unless path == document.relative_path then + raise "date/filename mismatch:\ndate+slug: #{path}\nfilename: #{document.relative_path}" + end + + if lang == 'en' then + unless ref == slug then + raise "ref/slug mismatch:\nref: #{ref}\nslug: #{slug}" + end + end + end + + if name == 'podcasts' then + flac = "resources/podcasts/#{date}-#{slug}.flac" + unless File.exist? flac then + raise "Missing FLAC file '#{flac}'" + end + + ogg = "resources/podcasts/#{date}-#{slug}.ogg" + unless File.exist? ogg then + puts "Missing '#{ogg}' file, generating..." + puts `ffmpeg -i #{flac} -ar 48000 -vn -c:a libvorbis -b:a 320k #{ogg}` + end + + torrent = "#{ogg}.torrent" + unless File.exist? torrent then + webseed = "#{config['url']}/#{ogg}" + file = "#{date}-#{slug}.ogg" + puts "Missing '#{torrent}' file, generating..." + puts `mktorrent #{TRACKERS} -f -v -d -c '#{document.content}' -n #{file} -w #{webseed} -o #{torrent} #{ogg}` + end + end + + if name == 'screencasts' then + mkv = "resources/screencasts/#{date}-#{slug}.mkv" + unless File.exist? mkv then + raise "Missing MKV file '#{mkv}'" + end + + torrent = "#{mkv}.torrent" + unless File.exist? torrent then + webseed = "#{config['url']}/#{mkv}" + file = "#{date}-#{slug}.mkv" + puts "Missing '#{torrent}' file, generating..." + puts `mktorrent #{TRACKERS} -f -v -d -c '#{document.content}' -n #{file} -w #{webseed} -o #{torrent} #{mkv}` + end + end + end + + def assert_frontmatter(site) + site.collections.each do |name, collection| + collection.docs.each do |document| + assert_frontmatter_fields site.config, name, document + end + end + + site.pages.each do |page| + unless IGNORED_PAGES.include? page.path + assert_frontmatter_fields site.config, 'page', page + end + end + end + def generate(site) assert_unique_ids(site) + assert_frontmatter(site) end end end diff --git a/scripts/assert-content.sh b/scripts/assert-content.sh deleted file mode 100755 index d42bdfb..0000000 --- a/scripts/assert-content.sh +++ /dev/null @@ -1,269 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -end="\033[0m" -red="\033[0;31m" -yellow="\033[0;33m" -red() { echo -e "${red}${1}${end}"; } -yellow() { echo -e "${yellow}${1}${end}"; } - -TRACKERS='-a udp://tracker.coppersurfer.tk:6969/announce -a udp://tracker.ccc.de:80/announce -a udp://tracker.publicbt.com:80 -a udp://tracker.istole.it:80 -a http://tracker.openbittorrent.com:80/announce -a http://tracker.ipv6tracker.org:80/announce' -# shellcheck disable=2016 -AWK_S=' -BEGIN { - FRONTMATTER=0 -} - -/^---$/ { - FRONTMATTER=!FRONTMATTER; - getline; # strip empty line below end of frontmatter - next; -} - -FRONTMATTER==0 { - print $0; -} -' - -## Constant definitions - -jekyll build --future -JSON='_site/site.json' - -LANGS=(en pt fr eo) # jp zh es de -IGNORED_PAGES=(site.json sitemap.xml) - -## Helper function definitions - -slugify() { - echo "${1}" | \ - tr '[:upper:]' '[:lower:]' | \ - perl -ne 'tr/\000-\177//cd; - s|/|-|g; - s/[^\w\s-.]//g; - s/^\s+|\s+$//g; - s/[-\s.]+/-/g; - print;' -} - -contains-element() { - local e match="$1" - shift - for e; do [[ "$e" == "$match" ]] && return 0; done - return 1 -} - -fail-attr() { - ATTRIBUTE="${1}" - URL="${2}" - red "Undefined '${ATTRIBUTE}' for ${URL}." >&2 - exit 1 -} - -get-lang() { - echo "${1}" | base64 --decode | jq -r .lang -} - -get-url() { - # Remove leading / to match more closely the filesystem hierarchy - echo "${1}" | base64 --decode | jq -r .url | sed 's_^/__' -} - -get-date() { - echo "${1}" | base64 --decode | jq -r .date | awk '{print $1}' -} - -get-x() { - echo "${2}" | base64 --decode | jq -r ".$1" -} - -is-ignored() { - URL="$1" - EXTENSION="${URL##*.}" - if contains-element "${URL}" "${IGNORED_PAGES[@]}" || [[ "$EXTENSION" == 'atom' ]]; then - return 0 - else - return 1 - fi -} - -## Assertions - -assert-frontmatter() { - F="$1" - DESIRED_LAYOUT="$2" - PREFIX="${3:-}" - EXTENSION="${4:-md}" - LLANG="$(get-lang "$F")" - REF="$(get-x ref "$F")" - URL="$(get-url "$F")" - LAYOUT="$(get-x layout "$F")" - TITLE="$(get-x title "$F")" - - [[ -z "${LLANG}" ]] && fail-attr 'lang' "${URL}" - [[ -z "${REF}" ]] && fail-attr 'ref' "${URL}" - [[ -z "${TITLE}" ]] && fail-attr 'title' "${URL}" - - if ! contains-element "${LLANG}" "${LANGS[@]}"; then - red "Invalid lang '${LLANG}' in ${URL}." >&2 - exit 1 - fi - - if [[ "${DESIRED_LAYOUT}" != "${LAYOUT}" ]]; then - red "Layout mismatch: expected '${DESIRED_LAYOUT}', got '${LAYOUT}'." - red "Page: ${URL}." - exit 1 - fi - - if [[ "$PREFIX" = '_podcasts/' ]]; then - AUDIO="$(get-x audio "$F")" - SLUG="$(get-x slug "$F")" - - [[ -z "$AUDIO" ]] && fail-attr 'audio' "${URL}" - [[ -z "$SLUG" ]] && fail-attr 'slug' "${URL}" - - TITLE_SLUG="$(slugify "$TITLE")" - if [[ "$SLUG" != "$TITLE_SLUG" ]]; then - red "slug and title don't match." - red "slug: '$SLUG'" - red "title slug: '$TITLE_SLUG'" - exit 1 - fi - - for audiofmt in flac ogg; do - DATE="$(get-date "$F")" - URL_BASENAME="$(basename "$(get-url "$F")")" - AUDIO="resources/podcasts/${DATE}-${SLUG}.$audiofmt" - if [[ ! -f "$AUDIO" ]]; then - red "Missing audio file '$AUDIO'." - exit 1 - fi - done - - OGG="resources/podcasts/$DATE-$SLUG.ogg" - TORRENT="$OGG.torrent" - WEBSEED="https://euandre.org/$OGG" - if [ ! -f "$TORRENT" ]; then - yellow "Missing torrent $TORRENT, generating..." - NOTES="$(awk "$AWK_S" "_podcasts/$DATE-$SLUG.md")" - # shellcheck disable=2086 - mktorrent $TRACKERS \ - -f \ - -v \ - -d \ - -c "$NOTES" \ - -n "$TITLE.ogg" \ - -w "$WEBSEED" \ - -o "$TORRENT" \ - "$OGG" - fi - fi - - if [[ "$PREFIX" = '_screencasts/' ]]; then - VIDEO="$(get-x video "$F")" - SLUG="$(get-x slug "$F")" - - [[ -z "$VIDEO" ]] && fail-attr 'video' "${URL}" - [[ -z "$SLUG" ]] && fail-attr 'slug' "${URL}" - - TITLE_SLUG="$(slugify "$TITLE")" - if [[ "$SLUG" != "$TITLE_SLUG" ]]; then - red "slug and title don't match." - red "slug: '$SLUG'" - red "title slug: '$TITLE_SLUG'" - exit 1 - fi - - DATE="$(get-date "$F")" - URL_BASENAME="$(basename "$(get-url "$F")")" - VIDEO="resources/screencasts/${DATE}-${SLUG}.webm" - if [[ ! -f "$VIDEO" ]]; then - red "Missing video file '$VIDEO'." - exit 1 - fi - - WEBM="resources/screencasts/$DATE-$SLUG.webm" - TORRENT="$WEBM.torrent" - WEBSEED="https://euandre.org/$WEBM" - if [ ! -f "$TORRENT" ]; then - yellow "Missing torrent $TORRENT, generating..." - NOTES="$(awk "$AWK_S" "_screencasts/$DATE-$SLUG.md")" - # shellcheck disable=2086 - mktorrent $TRACKERS \ - -f \ - -v \ - -d \ - -c "$NOTES" \ - -n "$TITLE.webm" \ - -w "$WEBSEED" \ - -o "$TORRENT" \ - "$WEBM" - fi - fi - - if [[ "$DESIRED_LAYOUT" != 'page' ]]; then - DATE="$(get-date "$F")" - URL_BASENAME="$(basename "$(get-url "$F")")" - FILE="${PREFIX}${DATE}-${URL_BASENAME%.html}.${EXTENSION}" - - [[ -f "${FILE}" ]] || { - red "date/filename mismatch: '${FILE}' does not exist." - exit 1 - } - - if [[ "$LLANG" = 'en' ]]; then - TITLE_SLUG="$(slugify "$TITLE")" - if [[ "$TITLE_SLUG" != "$REF" ]]; then - red "ref isn't the slug of the title." - red "ref: '$REF'" - red "title slug: '$TITLE_SLUG'" - exit 1 - fi - DESIRED_FILE="${PREFIX}${DATE}-${TITLE_SLUG}.${EXTENSION}" - if [[ ! -f "$DESIRED_FILE" ]]; then - red "File can't be guessed from date+slug: '$DESIRED_FILE' does not exist" - exit 1 - fi - fi - fi -} - -echo Linting pages... >&2 -for page in $(jq -r '.pages[] | @base64' "${JSON}"); do - URL="$(get-url "$page")" - if ! is-ignored "${URL}"; then - assert-frontmatter "${page}" 'page' - fi -done - -echo Linting articles... >&2 -for article in $(jq -r '.articles[] | @base64' "${JSON}"); do - assert-frontmatter "$article" 'post' '_articles/' -done - -echo Linting pastebins... >&2 -for pastebin in $(jq -r '.pastebins[] | @base64' "${JSON}"); do - assert-frontmatter "$pastebin" 'post' '_pastebins/' -done - -echo Linting tils... >&2 -for til in $(jq -r '.tils[] | @base64' "${JSON}"); do - assert-frontmatter "$til" 'post' '_tils/' -done - -echo Linting slides... >&2 -for slide in $(jq -r '.slides[] | @base64' "${JSON}"); do - assert-frontmatter "$slide" 'slides' '_slides/' 'slides' -done - -echo Linting podcasts... >&2 -for podcast in $(jq -r '.podcasts[] | @base64' "${JSON}"); do - assert-frontmatter "$podcast" 'cast' '_podcasts/' -done - -echo Linting screencasts... >&2 -for screencast in $(jq -r '.screencasts[] | @base64' "${JSON}"); do - assert-frontmatter "$screencast" 'cast' '_screencasts/' -done - -echo Done. >&2 diff --git a/site.json b/site.json deleted file mode 100644 index f27c7b7..0000000 --- a/site.json +++ /dev/null @@ -1,141 +0,0 @@ ---- ---- -{ - "pages": [ - {% assign filtered_pages = "" | split:"" %} - {% for page in site.pages %} - {% unless page.plaintext %} - {% assign filtered_pages = filtered_pages | push:page %} - {% endunless %} - {% endfor %} - {% for page in filtered_pages %} - { - "title": "{{ page.title | smartify }}", - "url": "{{ page.url }}", - "lang": "{{ page.lang }}", - "ref": "{{ page.ref }}", - "plaintext": "{{ page.plaintext }}", - "layout": "{{ page.layout }}", - "content": {{ page.content | strip_html | jsonify }} - }{% unless forloop.last %},{% endunless %} - {% endfor %} - ], - "articles": [ - {% assign filtered_articles = "" | split:"" %} - {% for post in site.articles %} - {% unless post.plaintext %} - {% assign filtered_articles = filtered_articles | push:post %} - {% endunless %} - {% endfor %} - {% for post in filtered_articles %} - { - "title": "{{ post.title | smartify }}", - "date": "{{ post.date }}", - "url": "{{ post.url }}", - "lang": "{{ post.lang }}", - "ref": "{{ post.ref }}", - "layout": "{{ post.layout }}", - "content": {{ post.content | strip_html | jsonify }} - }{% unless forloop.last %},{% endunless %} - {% endfor %} - ], - "pastebins": [ - {% assign filtered_pastebins = "" | split:"" %} - {% for pastebin in site.pastebins %} - {% unless pastebin.plaintext %} - {% assign filtered_pastebins = filtered_pastebins | push:pastebin %} - {% endunless %} - {% endfor %} - {% for pastebin in filtered_pastebins %} - { - "title": "{{ pastebin.title | smartify }}", - "date": "{{ pastebin.date }}", - "url": "{{ pastebin.url }}", - "lang": "{{ pastebin.lang }}", - "ref": "{{ pastebin.ref }}", - "layout": "{{ pastebin.layout }}", - "content": {{ pastebin.content | strip_html | jsonify }} - }{% unless forloop.last %},{% endunless %} - {% endfor %} - ], - "tils": [ - {% assign filtered_tils = "" | split:"" %} - {% for til in site.tils %} - {% unless til.plaintext %} - {% assign filtered_tils = filtered_tils | push:til %} - {% endunless %} - {% endfor %} - {% for til in filtered_tils %} - { - "title": "{{ til.title | smartify }}", - "date": "{{ til.date }}", - "url": "{{ til.url }}", - "lang": "{{ til.lang }}", - "ref": "{{ til.ref }}", - "layout": "{{ til.layout }}", - "content": {{ til.content | strip_html | jsonify }} - }{% unless forloop.last %},{% endunless %} - {% endfor %} - ], - "slides": [ - {% assign filtered_slides = "" | split:"" %} - {% for slide in site.slides %} - {% unless slide.plaintext %} - {% assign filtered_slides = filtered_slides | push:slide %} - {% endunless %} - {% endfor %} - {% for slide in filtered_slides %} - { - "title": "{{ slide.title | smartify }}", - "date": "{{ slide.date }}", - "url": "{{ slide.url }}", - "lang": "{{ slide.lang }}", - "ref": "{{ slide.ref }}", - "layout": "{{ slide.layout }}", - "content": {{ slide.content | strip_html | jsonify }} - }{% unless forloop.last %},{% endunless %} - {% endfor %} - ], - "podcasts": [ - {% assign filtered_podcasts = "" | split:"" %} - {% for podcast in site.podcasts %} - {% unless podcast.plaintext %} - {% assign filtered_podcasts = filtered_podcasts | push:podcast %} - {% endunless %} - {% endfor %} - {% for podcast in filtered_podcasts %} - { - "title": "{{ podcast.title | smartify }}", - "date": "{{ podcast.date }}", - "url": "{{ podcast.url }}", - "lang": "{{ podcast.lang }}", - "ref": "{{ podcast.ref }}", - "layout": "{{ podcast.layout }}", - "content": {{ podcast.content | strip_html | jsonify }}, - "audio": "{{ podcast.audio }}", - "slug": "{{ podcast.slug }}" - }{% unless forloop.last %},{% endunless %} - {% endfor %} - ], - "screencasts": [ - {% assign filtered_screencasts = "" | split:"" %} - {% for screencast in site.screencasts %} - {% unless screencast.plaintext %} - {% assign filtered_screencasts = filtered_screencasts | push:screencast %} - {% endunless %} - {% endfor %} - {% for screencast in filtered_screencasts %} - { - "title": "{{ screencast.title | smartify }}", - "date": "{{ screencast.date }}", - "url": "{{ screencast.url }}", - "lang": "{{ screencast.lang }}", - "ref": "{{ screencast.ref }}", - "layout": "{{ screencast.layout }}", - "content": {{ screencast.content | strip_html | jsonify }}, - "video": "{{ screencast.video }}", - "slug": "{{ screencast.slug }}" - }{% unless forloop.last %},{% endunless %} - {% endfor %} - ] -} @@ -9,5 +9,4 @@ set -x ./scripts/assert-shellcheck.sh ./scripts/assert-todos.sh -./scripts/assert-content.sh ./scripts/sync-translations.sh |