aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIvar Refsdal <ivar.refsdal@nsd.no>2021-09-17 14:51:01 +0200
committerIvar Refsdal <ivar.refsdal@nsd.no>2021-09-17 14:51:01 +0200
commitd13b0cb0b72a9cef9f8e9bd82616899796a4853f (patch)
tree1772613c72c1c71e6086af953424cae73e2c69cd
parentCapture-bindings works (diff)
downloadfiinha-d13b0cb0b72a9cef9f8e9bd82616899796a4853f.tar.gz
fiinha-d13b0cb0b72a9cef9f8e9bd82616899796a4853f.tar.xz
Use [#'taoensso.timbre/*context*] as default :capture-bindings if present
-rw-r--r--README.md10
-rw-r--r--deps.edn3
-rw-r--r--src/com/github/ivarref/yoltq.clj7
-rw-r--r--test/com/github/ivarref/yoltq/log_init.clj3
-rw-r--r--test/com/github/ivarref/yoltq/virtual_test.clj24
5 files changed, 34 insertions, 13 deletions
diff --git a/README.md b/README.md
index 514c4a3..902be2f 100644
--- a/README.md
+++ b/README.md
@@ -115,7 +115,7 @@ Inspecting `(yq/put :q {:work 123})]` you will see something like this:
:queue-name :q, ; Destination queue
:status :init, ; Status
:payload "{:work 123}", ; Payload persisted to the database with pr-str
- :bindings "{}",
+ :bindings "{}", ; Bindings that will be applied before executing consumer function
:lock #uuid"037d7da1-5158-4243-8f72-feb1e47e15ca", ; Lock to protect from multiple consumers
:tries 0, ; How many times the job has been executed
:init-time 4305758012289 ; Time of initialization (System/nanoTime)
@@ -153,7 +153,7 @@ The `payload` will be deserialized from the database using `clojure.edn/read-str
you will get back what you put into `yq/put`.
The yoltq system treats a queue consumer function invocation as successful if it does not throw an exception.
-Thus any regular return value, be it `nil`, `false`, `true`, etc. is considered a success.
+Any return value, be it `nil`, `false`, `true`, etc. is considered a success.
### Listening for queue jobs
@@ -166,7 +166,7 @@ and process newly created queue jobs fairly quickly.
This also means that queue jobs in status `:init` will almost always be processed without
any type of backoff*.
-This pool also schedules polling jobs that will regularly check for various statuses:
+The threadpool also schedules polling jobs that will check for various statuses regularly:
* Jobs in status `:error` that have waited for at least `:error-backoff-time` (default: 5 seconds) will be retried.
* Jobs that have been in `:processing` for at least `:hung-backoff-time` (default: 30 minutes) will be considered hung and retried.
@@ -212,6 +212,10 @@ A queue job will remain in status `:error` once `:max-retries` (default: 100) ha
Ideally this will not happen.
+### Logging
+
+
+
### Total health and system sanity
diff --git a/deps.edn b/deps.edn
index cf8297c..a457628 100644
--- a/deps.edn
+++ b/deps.edn
@@ -3,7 +3,8 @@
:paths ["src"]
- :aliases {:test {:extra-paths ["test"]
+ :aliases {:datomic {:extra-deps {com.datomic/datomic-pro {:mvn/version "1.0.6316" :exclusions [org.slf4j/slf4j-nop]}}}
+ :test {:extra-paths ["test"]
:extra-deps {ivarref/datomic-schema {:mvn/version "0.2.0"}
com.taoensso/timbre {:mvn/version "5.1.2"}
com.fzakaria/slf4j-timbre {:mvn/version "0.3.21"}
diff --git a/src/com/github/ivarref/yoltq.clj b/src/com/github/ivarref/yoltq.clj
index 6341e41..58efca1 100644
--- a/src/com/github/ivarref/yoltq.clj
+++ b/src/com/github/ivarref/yoltq.clj
@@ -1,6 +1,5 @@
(ns com.github.ivarref.yoltq
- (:require [datomic-schema.core]
- [datomic.api :as d]
+ (:require [datomic.api :as d]
[clojure.tools.logging :as log]
[com.github.ivarref.yoltq.impl :as i]
[com.github.ivarref.yoltq.report-queue :as rq]
@@ -58,7 +57,9 @@
; contain the stacktrace of the stuck threads.
:pool-size 4
- :capture-bindings []
+ :capture-bindings (if-let [s (resolve (symbol "taoensso.timbre/*context*"))]
+ [s]
+ [])
; How often should the system be polled for failed queue jobs
:system-error-poll-delay (Duration/ofMinutes 1)
diff --git a/test/com/github/ivarref/yoltq/log_init.clj b/test/com/github/ivarref/yoltq/log_init.clj
index cf69e55..1aa6c02 100644
--- a/test/com/github/ivarref/yoltq/log_init.clj
+++ b/test/com/github/ivarref/yoltq/log_init.clj
@@ -39,6 +39,9 @@
" "
(color-f (min-length 5 (str/upper-case (name level))))
" "
+
+ (when-let [x-req-id (:x-request-id context)]
+ (str "[" x-req-id "] "))
#_(.getName ^Thread (Thread/currentThread))
(color-f (force msg_))
diff --git a/test/com/github/ivarref/yoltq/virtual_test.clj b/test/com/github/ivarref/yoltq/virtual_test.clj
index e2ea19b..3c7c5b4 100644
--- a/test/com/github/ivarref/yoltq/virtual_test.clj
+++ b/test/com/github/ivarref/yoltq/virtual_test.clj
@@ -9,8 +9,7 @@
[clojure.tools.logging :as log]
[com.github.ivarref.yoltq.impl :as i]
[com.github.ivarref.yoltq :as yq]
- [clojure.pprint :as pprint]
- [clojure.edn :as edn]))
+ [taoensso.timbre :as timbre]))
(use-fixtures :each vq/call-with-virtual-queue!)
@@ -254,11 +253,24 @@
(deftest binding-test
(let [conn (u/empty-conn)]
(dq/init! {:conn conn
- :capture-bindings [#'*some-binding*]})
+ :capture-bindings [#'*some-binding* #'timbre/*context*]})
(dq/add-consumer! :q (fn [_] *some-binding*))
- (binding [*some-binding* 1] @(d/transact conn [(dq/put :q nil)]))
- (binding [*some-binding* 2] @(d/transact conn [(dq/put :q nil)]))
- @(d/transact conn [(dq/put :q nil)])
+ (binding [timbre/*context* {:x-request-id "wooho"}]
+ (binding [*some-binding* 1]
+ @(d/transact conn [(dq/put :q nil)]))
+ (binding [*some-binding* 2]
+ @(d/transact conn [(dq/put :q nil)]))
+ @(d/transact conn [(dq/put :q nil)]))
+
(is (= 1 (vq/consume-expect! :q :done)))
(is (= 2 (vq/consume-expect! :q :done)))
(is (nil? (vq/consume-expect! :q :done)))))
+
+
+(deftest default-binding-test
+ (let [conn (u/empty-conn)]
+ (dq/init! {:conn conn})
+ (dq/add-consumer! :q (fn [_] (:x-request-id timbre/*context*)))
+ (binding [timbre/*context* {:x-request-id "123"}]
+ @(d/transact conn [(dq/put :q nil)]))
+ (is (= "123" (vq/consume-expect! :q :done)))))