From 41c9e08d63176cf7c239574d1d07f2b302a2d3ec Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Sun, 27 Mar 2022 21:33:00 +0200 Subject: Fix use current millis in the database, not nano offset --- src/com/github/ivarref/yoltq/migrate.clj | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/com/github/ivarref/yoltq/migrate.clj (limited to 'src/com/github/ivarref/yoltq/migrate.clj') diff --git a/src/com/github/ivarref/yoltq/migrate.clj b/src/com/github/ivarref/yoltq/migrate.clj new file mode 100644 index 0000000..89fc286 --- /dev/null +++ b/src/com/github/ivarref/yoltq/migrate.clj @@ -0,0 +1,58 @@ +(ns com.github.ivarref.yoltq.migrate + (:require [datomic.api :as d] + [clojure.tools.logging :as log])) + +(defn to->v2-ent [{:keys [conn]} now-ms id] + (log/info "Migrating id" id) + (let [attr-val (fn [attr] + (when-let [old (d/q '[:find ?time . + :in $ ?e ?a + :where + [?e ?a ?time]] + (d/db conn) + [:com.github.ivarref.yoltq/id id] + attr)] + (let [now-ms (or now-ms + (.getTime (d/q '[:find (max ?txinst) . + :in $ ?e ?a + :where + [?e ?a _ ?tx true] + [?tx :db/txInstant ?txinst]] + (d/history (d/db conn)) + [:com.github.ivarref.yoltq/id id] + attr)))] + (log/info "Updating" id attr "to" now-ms) + [[:db/cas [:com.github.ivarref.yoltq/id id] + attr old now-ms]])))] + (vec (concat [[:db/cas [:com.github.ivarref.yoltq/id id] + :com.github.ivarref.yoltq/version nil "2"]] + (mapcat attr-val [:com.github.ivarref.yoltq/init-time + :com.github.ivarref.yoltq/processing-time + :com.github.ivarref.yoltq/done-time + :com.github.ivarref.yoltq/error-time]))))) + +(defn to->v2 [{:keys [conn loop? now-ms] + :or {loop? true} + :as cfg}] + (loop [tx-vec []] + (if-let [id (some->> (d/q '[:find [?id ...] + :in $ + :where + [?e :com.github.ivarref.yoltq/id ?id] + [(missing? $ ?e :com.github.ivarref.yoltq/version)]] + (d/db conn)) + (sort) + (not-empty) + (first))] + (let [tx (to->v2-ent cfg now-ms id)] + @(d/transact conn tx) + (if loop? + (recur (conj tx-vec tx)) + tx)) + (do + (log/info "No items left to migrate") + tx-vec)))) + + +(defn migrate! [cfg] + (to->v2 cfg)) -- cgit v1.2.3 From fc6dbf142005c57623cd1386a576b1fb487ac44a Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Mon, 28 Mar 2022 19:56:37 +0200 Subject: Don't OOM on migrating large datasets. Always prefer user specified value for config, also for `false`/`nil` values. --- src/com/github/ivarref/yoltq.clj | 2 +- src/com/github/ivarref/yoltq/migrate.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/com/github/ivarref/yoltq/migrate.clj') diff --git a/src/com/github/ivarref/yoltq.clj b/src/com/github/ivarref/yoltq.clj index 4b324c4..e1fd1b0 100644 --- a/src/com/github/ivarref/yoltq.clj +++ b/src/com/github/ivarref/yoltq.clj @@ -80,7 +80,7 @@ @(d/transact conn i/schema) (let [new-cfg (swap! *config* (fn [old-conf] - (-> (merge-with (fn [a b] (or b a)) + (-> (merge-with (fn [_ b] b) {:running-queues (atom #{}) :start-execute-time (atom {}) :system-error (atom {}) diff --git a/src/com/github/ivarref/yoltq/migrate.clj b/src/com/github/ivarref/yoltq/migrate.clj index 89fc286..6313b73 100644 --- a/src/com/github/ivarref/yoltq/migrate.clj +++ b/src/com/github/ivarref/yoltq/migrate.clj @@ -47,7 +47,7 @@ (let [tx (to->v2-ent cfg now-ms id)] @(d/transact conn tx) (if loop? - (recur (conj tx-vec tx)) + (recur (vec (take 10 (conj tx-vec tx)))) tx)) (do (log/info "No items left to migrate") -- cgit v1.2.3 From 5df374a5abd25eff5ebabe3c4605d3fb704a5af7 Mon Sep 17 00:00:00 2001 From: Ivar Refsdal Date: Tue, 29 Mar 2022 11:28:44 +0200 Subject: Release 0.2.55\nSet version numbers for change log ... Write new release script Write new release script Write new release script Release 0.2.55 Try squashing release ... Try squashing release ... --- README.md | 6 +++++- pom.xml | 4 ++-- release.sh | 24 +++++++++++++++++++----- src/com/github/ivarref/yoltq.clj | 5 +++++ src/com/github/ivarref/yoltq/migrate.clj | 3 +++ 5 files changed, 34 insertions(+), 8 deletions(-) (limited to 'src/com/github/ivarref/yoltq/migrate.clj') diff --git a/README.md b/README.md index 465a0cf..e5b2059 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,11 @@ easier. ## Change log -### 2022-03-28 v0.2.?? [diff](https://github.com/ivarref/yoltq/compare/v0.2.51...v0.2.??) +### 2022-03-29 v0.2.55 [diff](https://github.com/ivarref/yoltq/compare/v0.2.54...v0.2.55) +Added: `unhealthy?` function which returns `true` if there are queues in error, +or `false` otherwise. + +### 2022-03-28 v0.2.54 [diff](https://github.com/ivarref/yoltq/compare/v0.2.51...v0.2.54) Fixed: Schedules should now be using milliseconds and not nanoseconds. ### 2022-03-28 v0.2.51 [diff](https://github.com/ivarref/yoltq/compare/v0.2.48...v0.2.51) diff --git a/pom.xml b/pom.xml index 28e0ece..9f591b9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ jar com.github.ivarref yoltq - 0.2.54 + 0.2.55 yoltq @@ -30,7 +30,7 @@ scm:git:git://github.com/ivarref/yoltq.git scm:git:ssh://git@github.com/ivarref/yoltq.git - v0.2.54 + v0.2.55 https://github.com/ivarref/yoltq \ No newline at end of file diff --git a/release.sh b/release.sh index dec59a2..cf0f09f 100755 --- a/release.sh +++ b/release.sh @@ -2,17 +2,31 @@ set -ex +git update-index --refresh +git diff-index --quiet HEAD -- + clojure -Spom clojure -M:test clojure -M:jar clojure -X:release ivarref.pom-patch/clojars-repo-only! -VERSION=$(clojure -X:release ivarref.pom-patch/set-patch-version! :patch :commit-count+1) -git add pom.xml +LAST_TAG="$(git rev-list --tags --no-walk --max-count=1)" +COMMITS_SINCE_LAST_TAG="$(git rev-list "$LAST_TAG"..HEAD --count)" +echo "Squashing $COMMITS_SINCE_LAST_TAG commits ..." +git reset --soft HEAD~"$COMMITS_SINCE_LAST_TAG" +MSG="$(git log --format=%B --reverse HEAD..HEAD@{1})" +git commit -m"$MSG" + +VERSION="$(clojure -X:release ivarref.pom-patch/set-patch-version! :patch :commit-count)" +echo "Releasing $VERSION" +sed -i "s/HEAD/v$VERSION/g" ./README.md +git add pom.xml README.md git commit -m "Release $VERSION" -git tag -a v$VERSION -m "Release v$VERSION" -git push --follow-tags +git reset --soft HEAD~2 +git commit -m"Release $VERSION\n$MSG" -clojure -X:deploy +git tag -a v"$VERSION" -m "Release v$VERSION\n$MSG" +git push --follow-tags --force +clojure -X:deploy echo "Released $VERSION" diff --git a/src/com/github/ivarref/yoltq.clj b/src/com/github/ivarref/yoltq.clj index 7d5434e..bb7a43e 100644 --- a/src/com/github/ivarref/yoltq.clj +++ b/src/com/github/ivarref/yoltq.clj @@ -160,6 +160,11 @@ :healthy? (deref))) +(defn unhealthy? + "Returns `true` if there are queues in error, otherwise `false`." + [] + (false? (healthy?))) + (defn queue-stats [] (let [{:keys [conn]} @*config* db (d/db conn)] diff --git a/src/com/github/ivarref/yoltq/migrate.clj b/src/com/github/ivarref/yoltq/migrate.clj index 6313b73..c97f679 100644 --- a/src/com/github/ivarref/yoltq/migrate.clj +++ b/src/com/github/ivarref/yoltq/migrate.clj @@ -56,3 +56,6 @@ (defn migrate! [cfg] (to->v2 cfg)) + +(comment + (migrate! @com.github.ivarref.yoltq/*config*)) -- cgit v1.2.3