aboutsummaryrefslogtreecommitdiff
path: root/cmd
diff options
context:
space:
mode:
authorBen Johnson <benbjohnson@yahoo.com>2015-04-14 16:32:20 -0600
committerBen Johnson <benbjohnson@yahoo.com>2015-04-14 16:32:20 -0600
commitd0e8a99e30eebd30669908bbe57c969e8921ca6c (patch)
treed3c9a20e201869fd5fd6ad495d4dd76a5c0f518d /cmd
parentAdd --path to bolt bench. (diff)
downloaddedo-d0e8a99e30eebd30669908bbe57c969e8921ca6c.tar.gz
dedo-d0e8a99e30eebd30669908bbe57c969e8921ca6c.tar.xz
Refactor bolt CLI.
Diffstat (limited to 'cmd')
-rw-r--r--cmd/bolt/bench.go425
-rw-r--r--cmd/bolt/buckets.go33
-rw-r--r--cmd/bolt/buckets_test.go31
-rw-r--r--cmd/bolt/check.go47
-rw-r--r--cmd/bolt/get.go45
-rw-r--r--cmd/bolt/get_test.go54
-rw-r--r--cmd/bolt/info.go26
-rw-r--r--cmd/bolt/info_test.go31
-rw-r--r--cmd/bolt/keys.go41
-rw-r--r--cmd/bolt/keys_test.go42
-rw-r--r--cmd/bolt/main.go955
-rw-r--r--cmd/bolt/main_test.go162
-rw-r--r--cmd/bolt/pages.go57
-rw-r--r--cmd/bolt/stats.go77
-rw-r--r--cmd/bolt/stats_test.go61
15 files changed, 914 insertions, 1173 deletions
diff --git a/cmd/bolt/bench.go b/cmd/bolt/bench.go
deleted file mode 100644
index 3ade8b8..0000000
--- a/cmd/bolt/bench.go
+++ /dev/null
@@ -1,425 +0,0 @@
-package main
-
-import (
- "encoding/binary"
- "encoding/json"
- "errors"
- "fmt"
- "io/ioutil"
- "math/rand"
- "os"
- "runtime"
- "runtime/pprof"
- "time"
-
- "github.com/boltdb/bolt"
-)
-
-// File handlers for the various profiles.
-var cpuprofile, memprofile, blockprofile *os.File
-
-var benchBucketName = []byte("bench")
-
-// Bench executes a customizable, synthetic benchmark against Bolt.
-func Bench(options *BenchOptions) {
- var results BenchResults
-
- // Validate options.
- if options.BatchSize == 0 {
- options.BatchSize = options.Iterations
- } else if options.Iterations%options.BatchSize != 0 {
- fatal("number of iterations must be divisible by the batch size")
- }
-
- // Generate temp path if one is not passed in.
- path := options.Path
- if path == "" {
- path = tempfile()
- }
-
- if options.Clean {
- defer os.Remove(path)
- } else {
- println("work:", path)
- }
-
- // Create database.
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- db.NoSync = options.NoSync
- defer db.Close()
-
- // Enable streaming stats.
- if options.StatsInterval > 0 {
- go printStats(db, options.StatsInterval)
- }
-
- // Start profiling for writes.
- if options.ProfileMode == "rw" || options.ProfileMode == "w" {
- benchStartProfiling(options)
- }
-
- // Write to the database.
- if err := benchWrite(db, options, &results); err != nil {
- fatal("bench: write: ", err)
- }
-
- // Stop profiling for writes only.
- if options.ProfileMode == "w" {
- benchStopProfiling()
- }
-
- // Start profiling for reads.
- if options.ProfileMode == "r" {
- benchStartProfiling(options)
- }
-
- // Read from the database.
- if err := benchRead(db, options, &results); err != nil {
- fatal("bench: read: ", err)
- }
-
- // Stop profiling for writes only.
- if options.ProfileMode == "rw" || options.ProfileMode == "r" {
- benchStopProfiling()
- }
-
- // Print results.
- fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond())
- fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond())
- fmt.Fprintln(os.Stderr, "")
-}
-
-// Writes to the database.
-func benchWrite(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- var err error
- var t = time.Now()
-
- switch options.WriteMode {
- case "seq":
- err = benchWriteSequential(db, options, results)
- case "rnd":
- err = benchWriteRandom(db, options, results)
- case "seq-nest":
- err = benchWriteSequentialNested(db, options, results)
- case "rnd-nest":
- err = benchWriteRandomNested(db, options, results)
- default:
- return fmt.Errorf("invalid write mode: %s", options.WriteMode)
- }
-
- results.WriteDuration = time.Since(t)
-
- return err
-}
-
-func benchWriteSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- var i = uint32(0)
- return benchWriteWithSource(db, options, results, func() uint32 { i++; return i })
-}
-
-func benchWriteRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- r := rand.New(rand.NewSource(time.Now().UnixNano()))
- return benchWriteWithSource(db, options, results, func() uint32 { return r.Uint32() })
-}
-
-func benchWriteSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- var i = uint32(0)
- return benchWriteNestedWithSource(db, options, results, func() uint32 { i++; return i })
-}
-
-func benchWriteRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- r := rand.New(rand.NewSource(time.Now().UnixNano()))
- return benchWriteNestedWithSource(db, options, results, func() uint32 { return r.Uint32() })
-}
-
-func benchWriteWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
- results.WriteOps = options.Iterations
-
- for i := 0; i < options.Iterations; i += options.BatchSize {
- err := db.Update(func(tx *bolt.Tx) error {
- b, _ := tx.CreateBucketIfNotExists(benchBucketName)
- b.FillPercent = options.FillPercent
-
- for j := 0; j < options.BatchSize; j++ {
- var key = make([]byte, options.KeySize)
- var value = make([]byte, options.ValueSize)
- binary.BigEndian.PutUint32(key, keySource())
- if err := b.Put(key, value); err != nil {
- return err
- }
- }
-
- return nil
- })
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-func benchWriteNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
- results.WriteOps = options.Iterations
-
- for i := 0; i < options.Iterations; i += options.BatchSize {
- err := db.Update(func(tx *bolt.Tx) error {
- top, _ := tx.CreateBucketIfNotExists(benchBucketName)
- top.FillPercent = options.FillPercent
-
- var name = make([]byte, options.KeySize)
- binary.BigEndian.PutUint32(name, keySource())
- b, _ := top.CreateBucketIfNotExists(name)
- b.FillPercent = options.FillPercent
-
- for j := 0; j < options.BatchSize; j++ {
- var key = make([]byte, options.KeySize)
- var value = make([]byte, options.ValueSize)
- binary.BigEndian.PutUint32(key, keySource())
- if err := b.Put(key, value); err != nil {
- return err
- }
- }
-
- return nil
- })
- if err != nil {
- return err
- }
- }
- return nil
-}
-
-// Reads from the database.
-func benchRead(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- var err error
- var t = time.Now()
-
- switch options.ReadMode {
- case "seq":
- if options.WriteMode == "seq-nest" || options.WriteMode == "rnd-nest" {
- err = benchReadSequentialNested(db, options, results)
- } else {
- err = benchReadSequential(db, options, results)
- }
- default:
- return fmt.Errorf("invalid read mode: %s", options.ReadMode)
- }
-
- results.ReadDuration = time.Since(t)
-
- return err
-}
-
-func benchReadSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- return db.View(func(tx *bolt.Tx) error {
- var t = time.Now()
-
- for {
- c := tx.Bucket(benchBucketName).Cursor()
- var count int
- for k, v := c.First(); k != nil; k, v = c.Next() {
- if v == nil {
- return errors.New("invalid value")
- }
- count++
- }
-
- if options.WriteMode == "seq" && count != options.Iterations {
- return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
- }
-
- results.ReadOps += count
-
- // Make sure we do this for at least a second.
- if time.Since(t) >= time.Second {
- break
- }
- }
-
- return nil
- })
-}
-
-func benchReadSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
- return db.View(func(tx *bolt.Tx) error {
- var t = time.Now()
-
- for {
- var count int
- var top = tx.Bucket(benchBucketName)
- top.ForEach(func(name, _ []byte) error {
- c := top.Bucket(name).Cursor()
- for k, v := c.First(); k != nil; k, v = c.Next() {
- if v == nil {
- return errors.New("invalid value")
- }
- count++
- }
- return nil
- })
-
- if options.WriteMode == "seq-nest" && count != options.Iterations {
- return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
- }
-
- results.ReadOps += count
-
- // Make sure we do this for at least a second.
- if time.Since(t) >= time.Second {
- break
- }
- }
-
- return nil
- })
-}
-
-// Starts all profiles set on the options.
-func benchStartProfiling(options *BenchOptions) {
- var err error
-
- // Start CPU profiling.
- if options.CPUProfile != "" {
- cpuprofile, err = os.Create(options.CPUProfile)
- if err != nil {
- fatalf("bench: could not create cpu profile %q: %v", options.CPUProfile, err)
- }
- pprof.StartCPUProfile(cpuprofile)
- }
-
- // Start memory profiling.
- if options.MemProfile != "" {
- memprofile, err = os.Create(options.MemProfile)
- if err != nil {
- fatalf("bench: could not create memory profile %q: %v", options.MemProfile, err)
- }
- runtime.MemProfileRate = 4096
- }
-
- // Start fatal profiling.
- if options.BlockProfile != "" {
- blockprofile, err = os.Create(options.BlockProfile)
- if err != nil {
- fatalf("bench: could not create block profile %q: %v", options.BlockProfile, err)
- }
- runtime.SetBlockProfileRate(1)
- }
-}
-
-// Stops all profiles.
-func benchStopProfiling() {
- if cpuprofile != nil {
- pprof.StopCPUProfile()
- cpuprofile.Close()
- cpuprofile = nil
- }
-
- if memprofile != nil {
- pprof.Lookup("heap").WriteTo(memprofile, 0)
- memprofile.Close()
- memprofile = nil
- }
-
- if blockprofile != nil {
- pprof.Lookup("block").WriteTo(blockprofile, 0)
- blockprofile.Close()
- blockprofile = nil
- runtime.SetBlockProfileRate(0)
- }
-}
-
-// Continuously prints stats on the database at given intervals.
-func printStats(db *bolt.DB, interval time.Duration) {
- var prevStats = db.Stats()
- var encoder = json.NewEncoder(os.Stdout)
-
- for {
- // Wait for the stats interval.
- time.Sleep(interval)
-
- // Retrieve new stats and find difference from previous iteration.
- var stats = db.Stats()
- var diff = stats.Sub(&prevStats)
-
- // Print as JSON to STDOUT.
- if err := encoder.Encode(diff); err != nil {
- fatal(err)
- }
-
- // Save stats for next iteration.
- prevStats = stats
- }
-}
-
-// BenchOptions represents the set of options that can be passed to Bench().
-type BenchOptions struct {
- ProfileMode string
- WriteMode string
- ReadMode string
- Iterations int
- BatchSize int
- KeySize int
- ValueSize int
- CPUProfile string
- MemProfile string
- BlockProfile string
- StatsInterval time.Duration
- FillPercent float64
- NoSync bool
- Clean bool
- Path string
-}
-
-// BenchResults represents the performance results of the benchmark.
-type BenchResults struct {
- WriteOps int
- WriteDuration time.Duration
- ReadOps int
- ReadDuration time.Duration
-}
-
-// Returns the duration for a single write operation.
-func (r *BenchResults) WriteOpDuration() time.Duration {
- if r.WriteOps == 0 {
- return 0
- }
- return r.WriteDuration / time.Duration(r.WriteOps)
-}
-
-// Returns average number of write operations that can be performed per second.
-func (r *BenchResults) WriteOpsPerSecond() int {
- var op = r.WriteOpDuration()
- if op == 0 {
- return 0
- }
- return int(time.Second) / int(op)
-}
-
-// Returns the duration for a single read operation.
-func (r *BenchResults) ReadOpDuration() time.Duration {
- if r.ReadOps == 0 {
- return 0
- }
- return r.ReadDuration / time.Duration(r.ReadOps)
-}
-
-// Returns average number of read operations that can be performed per second.
-func (r *BenchResults) ReadOpsPerSecond() int {
- var op = r.ReadOpDuration()
- if op == 0 {
- return 0
- }
- return int(time.Second) / int(op)
-}
-
-// tempfile returns a temporary file path.
-func tempfile() string {
- f, _ := ioutil.TempFile("", "bolt-bench-")
- f.Close()
- os.Remove(f.Name())
- return f.Name()
-}
diff --git a/cmd/bolt/buckets.go b/cmd/bolt/buckets.go
deleted file mode 100644
index 68e7dde..0000000
--- a/cmd/bolt/buckets.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package main
-
-import (
- "os"
-
- "github.com/boltdb/bolt"
-)
-
-// Buckets prints a list of all buckets.
-func Buckets(path string) {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- fatal(err)
- return
- }
-
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- defer db.Close()
-
- err = db.View(func(tx *bolt.Tx) error {
- return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
- println(string(name))
- return nil
- })
- })
- if err != nil {
- fatal(err)
- return
- }
-}
diff --git a/cmd/bolt/buckets_test.go b/cmd/bolt/buckets_test.go
deleted file mode 100644
index d5050fd..0000000
--- a/cmd/bolt/buckets_test.go
+++ /dev/null
@@ -1,31 +0,0 @@
-package main_test
-
-import (
- "testing"
-
- "github.com/boltdb/bolt"
- . "github.com/boltdb/bolt/cmd/bolt"
-)
-
-// Ensure that a list of buckets can be retrieved.
-func TestBuckets(t *testing.T) {
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Update(func(tx *bolt.Tx) error {
- tx.CreateBucket([]byte("woojits"))
- tx.CreateBucket([]byte("widgets"))
- tx.CreateBucket([]byte("whatchits"))
- return nil
- })
- db.Close()
- output := run("buckets", path)
- equals(t, "whatchits\nwidgets\nwoojits", output)
- })
-}
-
-// Ensure that an error is reported if the database is not found.
-func TestBucketsDBNotFound(t *testing.T) {
- SetTestMode(true)
- output := run("buckets", "no/such/db")
- equals(t, "stat no/such/db: no such file or directory", output)
-}
diff --git a/cmd/bolt/check.go b/cmd/bolt/check.go
deleted file mode 100644
index 125f2b8..0000000
--- a/cmd/bolt/check.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package main
-
-import (
- "os"
-
- "github.com/boltdb/bolt"
-)
-
-// Check performs a consistency check on the database and prints any errors found.
-func Check(path string) {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- fatal(err)
- return
- }
-
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- defer db.Close()
-
- // Perform consistency check.
- _ = db.View(func(tx *bolt.Tx) error {
- var count int
- ch := tx.Check()
- loop:
- for {
- select {
- case err, ok := <-ch:
- if !ok {
- break loop
- }
- println(err)
- count++
- }
- }
-
- // Print summary of errors.
- if count > 0 {
- fatalf("%d errors found", count)
- } else {
- println("OK")
- }
- return nil
- })
-}
diff --git a/cmd/bolt/get.go b/cmd/bolt/get.go
deleted file mode 100644
index 90e0c1d..0000000
--- a/cmd/bolt/get.go
+++ /dev/null
@@ -1,45 +0,0 @@
-package main
-
-import (
- "os"
-
- "github.com/boltdb/bolt"
-)
-
-// Get retrieves the value for a given bucket/key.
-func Get(path, name, key string) {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- fatal(err)
- return
- }
-
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- defer db.Close()
-
- err = db.View(func(tx *bolt.Tx) error {
- // Find bucket.
- b := tx.Bucket([]byte(name))
- if b == nil {
- fatalf("bucket not found: %s", name)
- return nil
- }
-
- // Find value for a given key.
- value := b.Get([]byte(key))
- if value == nil {
- fatalf("key not found: %s", key)
- return nil
- }
-
- println(string(value))
- return nil
- })
- if err != nil {
- fatal(err)
- return
- }
-}
diff --git a/cmd/bolt/get_test.go b/cmd/bolt/get_test.go
deleted file mode 100644
index 8acd0f4..0000000
--- a/cmd/bolt/get_test.go
+++ /dev/null
@@ -1,54 +0,0 @@
-package main_test
-
-import (
- "testing"
-
- "github.com/boltdb/bolt"
- . "github.com/boltdb/bolt/cmd/bolt"
-)
-
-// Ensure that a value can be retrieved from the CLI.
-func TestGet(t *testing.T) {
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Update(func(tx *bolt.Tx) error {
- tx.CreateBucket([]byte("widgets"))
- tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar"))
- return nil
- })
- db.Close()
- output := run("get", path, "widgets", "foo")
- equals(t, "bar", output)
- })
-}
-
-// Ensure that an error is reported if the database is not found.
-func TestGetDBNotFound(t *testing.T) {
- SetTestMode(true)
- output := run("get", "no/such/db", "widgets", "foo")
- equals(t, "stat no/such/db: no such file or directory", output)
-}
-
-// Ensure that an error is reported if the bucket is not found.
-func TestGetBucketNotFound(t *testing.T) {
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Close()
- output := run("get", path, "widgets", "foo")
- equals(t, "bucket not found: widgets", output)
- })
-}
-
-// Ensure that an error is reported if the key is not found.
-func TestGetKeyNotFound(t *testing.T) {
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Update(func(tx *bolt.Tx) error {
- _, err := tx.CreateBucket([]byte("widgets"))
- return err
- })
- db.Close()
- output := run("get", path, "widgets", "foo")
- equals(t, "key not found: foo", output)
- })
-}
diff --git a/cmd/bolt/info.go b/cmd/bolt/info.go
deleted file mode 100644
index cb01e38..0000000
--- a/cmd/bolt/info.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package main
-
-import (
- "os"
-
- "github.com/boltdb/bolt"
-)
-
-// Info prints basic information about a database.
-func Info(path string) {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- fatal(err)
- return
- }
-
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- defer db.Close()
-
- // Print basic database info.
- var info = db.Info()
- printf("Page Size: %d\n", info.PageSize)
-}
diff --git a/cmd/bolt/info_test.go b/cmd/bolt/info_test.go
deleted file mode 100644
index dab74f6..0000000
--- a/cmd/bolt/info_test.go
+++ /dev/null
@@ -1,31 +0,0 @@
-package main_test
-
-import (
- "testing"
-
- "github.com/boltdb/bolt"
- . "github.com/boltdb/bolt/cmd/bolt"
-)
-
-// Ensure that a database info can be printed.
-func TestInfo(t *testing.T) {
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Update(func(tx *bolt.Tx) error {
- tx.CreateBucket([]byte("widgets"))
- b := tx.Bucket([]byte("widgets"))
- b.Put([]byte("foo"), []byte("0000"))
- return nil
- })
- db.Close()
- output := run("info", path)
- equals(t, `Page Size: 4096`, output)
- })
-}
-
-// Ensure that an error is reported if the database is not found.
-func TestInfo_NotFound(t *testing.T) {
- SetTestMode(true)
- output := run("info", "no/such/db")
- equals(t, "stat no/such/db: no such file or directory", output)
-}
diff --git a/cmd/bolt/keys.go b/cmd/bolt/keys.go
deleted file mode 100644
index d4bb3c3..0000000
--- a/cmd/bolt/keys.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package main
-
-import (
- "os"
-
- "github.com/boltdb/bolt"
-)
-
-// Keys retrieves a list of keys for a given bucket.
-func Keys(path, name string) {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- fatal(err)
- return
- }
-
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- defer db.Close()
-
- err = db.View(func(tx *bolt.Tx) error {
- // Find bucket.
- b := tx.Bucket([]byte(name))
- if b == nil {
- fatalf("bucket not found: %s", name)
- return nil
- }
-
- // Iterate over each key.
- return b.ForEach(func(key, _ []byte) error {
- println(string(key))
- return nil
- })
- })
- if err != nil {
- fatal(err)
- return
- }
-}
diff --git a/cmd/bolt/keys_test.go b/cmd/bolt/keys_test.go
deleted file mode 100644
index 0cc4e0c..0000000
--- a/cmd/bolt/keys_test.go
+++ /dev/null
@@ -1,42 +0,0 @@
-package main_test
-
-import (
- "testing"
-
- "github.com/boltdb/bolt"
- . "github.com/boltdb/bolt/cmd/bolt"
-)
-
-// Ensure that a list of keys can be retrieved for a given bucket.
-func TestKeys(t *testing.T) {
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Update(func(tx *bolt.Tx) error {
- tx.CreateBucket([]byte("widgets"))
- tx.Bucket([]byte("widgets")).Put([]byte("0002"), []byte(""))
- tx.Bucket([]byte("widgets")).Put([]byte("0001"), []byte(""))
- tx.Bucket([]byte("widgets")).Put([]byte("0003"), []byte(""))
- return nil
- })
- db.Close()
- output := run("keys", path, "widgets")
- equals(t, "0001\n0002\n0003", output)
- })
-}
-
-// Ensure that an error is reported if the database is not found.
-func TestKeysDBNotFound(t *testing.T) {
- SetTestMode(true)
- output := run("keys", "no/such/db", "widgets")
- equals(t, "stat no/such/db: no such file or directory", output)
-}
-
-// Ensure that an error is reported if the bucket is not found.
-func TestKeysBucketNotFound(t *testing.T) {
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Close()
- output := run("keys", path, "widgets")
- equals(t, "bucket not found: widgets", output)
- })
-}
diff --git a/cmd/bolt/main.go b/cmd/bolt/main.go
index 0372e19..7fcf530 100644
--- a/cmd/bolt/main.go
+++ b/cmd/bolt/main.go
@@ -2,199 +2,834 @@ package main
import (
"bytes"
+ "encoding/binary"
+ "errors"
+ "flag"
"fmt"
- "log"
+ "io"
+ "io/ioutil"
+ "math/rand"
"os"
+ "runtime"
+ "runtime/pprof"
+ "strconv"
+ "strings"
"time"
"github.com/boltdb/bolt"
- "github.com/codegangsta/cli"
)
-var branch, commit string
+var (
+ // ErrCommandRequired is returned when a CLI command is not specified.
+ ErrCommandRequired = errors.New("command required")
+
+ // ErrUnknownCommand is returned when a CLI command is not specified.
+ ErrUnknownCommand = errors.New("unknown command")
+
+ // ErrPathRequired is returned when the path to a Bolt database is not specified.
+ ErrPathRequired = errors.New("path required")
+
+ // ErrFileNotFound is returned when a Bolt database does not exist.
+ ErrFileNotFound = errors.New("file not found")
+
+ // ErrInvalidValue is returned when a benchmark reads an unexpected value.
+ ErrInvalidValue = errors.New("invalid value")
+
+ // ErrCorrupt is returned when a checking a data file finds errors.
+ ErrCorrupt = errors.New("invalid value")
+
+ // ErrNonDivisibleBatchSize is returned when the batch size can't be evenly
+ // divided by the iteration count.
+ ErrNonDivisibleBatchSize = errors.New("number of iterations must be divisible by the batch size")
+)
func main() {
- log.SetFlags(0)
- NewApp().Run(os.Args)
-}
-
-// NewApp creates an Application instance.
-func NewApp() *cli.App {
- app := cli.NewApp()
- app.Name = "bolt"
- app.Usage = "BoltDB toolkit"
- app.Version = fmt.Sprintf("0.1.0 (%s %s)", branch, commit)
- app.Commands = []cli.Command{
- {
- Name: "info",
- Usage: "Print basic information about a database",
- Action: func(c *cli.Context) {
- path := c.Args().Get(0)
- Info(path)
- },
- },
- {
- Name: "get",
- Usage: "Retrieve a value for given key in a bucket",
- Action: func(c *cli.Context) {
- path, name, key := c.Args().Get(0), c.Args().Get(1), c.Args().Get(2)
- Get(path, name, key)
- },
- },
- {
- Name: "keys",
- Usage: "Retrieve a list of all keys in a bucket",
- Action: func(c *cli.Context) {
- path, name := c.Args().Get(0), c.Args().Get(1)
- Keys(path, name)
- },
- },
- {
- Name: "buckets",
- Usage: "Retrieves a list of all buckets",
- Action: func(c *cli.Context) {
- path := c.Args().Get(0)
- Buckets(path)
- },
- },
- {
- Name: "pages",
- Usage: "Dumps page information for a database",
- Action: func(c *cli.Context) {
- path := c.Args().Get(0)
- Pages(path)
- },
- },
- {
- Name: "check",
- Usage: "Performs a consistency check on the database",
- Action: func(c *cli.Context) {
- path := c.Args().Get(0)
- Check(path)
- },
- },
- {
- Name: "stats",
- Usage: "Aggregate statistics for all buckets matching specified prefix",
- Action: func(c *cli.Context) {
- path, name := c.Args().Get(0), c.Args().Get(1)
- Stats(path, name)
- },
- },
- {
- Name: "bench",
- Usage: "Performs a synthetic benchmark",
- Flags: []cli.Flag{
- &cli.StringFlag{Name: "profile-mode", Value: "rw", Usage: "Profile mode"},
- &cli.StringFlag{Name: "write-mode", Value: "seq", Usage: "Write mode"},
- &cli.StringFlag{Name: "read-mode", Value: "seq", Usage: "Read mode"},
- &cli.IntFlag{Name: "count", Value: 1000, Usage: "Item count"},
- &cli.IntFlag{Name: "batch-size", Usage: "Write batch size"},
- &cli.IntFlag{Name: "key-size", Value: 8, Usage: "Key size"},
- &cli.IntFlag{Name: "value-size", Value: 32, Usage: "Value size"},
- &cli.StringFlag{Name: "cpuprofile", Usage: "CPU profile output path"},
- &cli.StringFlag{Name: "memprofile", Usage: "Memory profile output path"},
- &cli.StringFlag{Name: "blockprofile", Usage: "Block profile output path"},
- &cli.StringFlag{Name: "stats-interval", Value: "0s", Usage: "Continuous stats interval"},
- &cli.Float64Flag{Name: "fill-percent", Value: bolt.DefaultFillPercent, Usage: "Fill percentage"},
- &cli.BoolFlag{Name: "no-sync", Usage: "Skip fsync on every commit"},
- &cli.BoolFlag{Name: "work", Usage: "Print the temp db and do not delete on exit"},
- &cli.StringFlag{Name: "path", Usage: "Path to database to use"},
- },
- Action: func(c *cli.Context) {
- statsInterval, err := time.ParseDuration(c.String("stats-interval"))
- if err != nil {
- fatal(err)
+ m := NewMain()
+ if err := m.Run(os.Args[1:]...); err != nil {
+ fmt.Println(err.Error())
+ os.Exit(1)
+ }
+}
+
+// Main represents the main program execution.
+type Main struct {
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
+}
+
+// NewMain returns a new instance of Main connect to the standard input/output.
+func NewMain() *Main {
+ return &Main{
+ Stdin: os.Stdin,
+ Stdout: os.Stdout,
+ Stderr: os.Stderr,
+ }
+}
+
+// Run executes the program.
+func (m *Main) Run(args ...string) error {
+ // Require a command at the beginning.
+ if len(args) == 0 || strings.HasPrefix(args[0], "-") {
+ return ErrCommandRequired
+ }
+
+ // Execute command.
+ switch args[0] {
+ case "bench":
+ return newBenchCommand(m).Run(args[1:]...)
+ case "check":
+ return newCheckCommand(m).Run(args[1:]...)
+ case "info":
+ return newInfoCommand(m).Run(args[1:]...)
+ case "pages":
+ return newPagesCommand(m).Run(args[1:]...)
+ case "stats":
+ return newStatsCommand(m).Run(args[1:]...)
+ default:
+ return ErrUnknownCommand
+ }
+}
+
+// CheckCommand represents the "check" command execution.
+type CheckCommand struct {
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
+}
+
+// NewCheckCommand returns a CheckCommand.
+func newCheckCommand(m *Main) *CheckCommand {
+ return &CheckCommand{
+ Stdin: m.Stdin,
+ Stdout: m.Stdout,
+ Stderr: m.Stderr,
+ }
+}
+
+// Run executes the command.
+func (cmd *CheckCommand) Run(args ...string) error {
+ // Parse flags.
+ fs := flag.NewFlagSet("", flag.ContinueOnError)
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+
+ // Require database path.
+ path := fs.Arg(0)
+ if path == "" {
+ return ErrPathRequired
+ } else if _, err := os.Stat(path); os.IsNotExist(err) {
+ return ErrFileNotFound
+ }
+
+ // Open database.
+ db, err := bolt.Open(path, 0666, nil)
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+
+ // Perform consistency check.
+ return db.View(func(tx *bolt.Tx) error {
+ var count int
+ ch := tx.Check()
+ loop:
+ for {
+ select {
+ case err, ok := <-ch:
+ if !ok {
+ break loop
}
+ fmt.Fprintln(cmd.Stdout, err)
+ count++
+ }
+ }
- Bench(&BenchOptions{
- ProfileMode: c.String("profile-mode"),
- WriteMode: c.String("write-mode"),
- ReadMode: c.String("read-mode"),
- Iterations: c.Int("count"),
- BatchSize: c.Int("batch-size"),
- KeySize: c.Int("key-size"),
- ValueSize: c.Int("value-size"),
- CPUProfile: c.String("cpuprofile"),
- MemProfile: c.String("memprofile"),
- BlockProfile: c.String("blockprofile"),
- StatsInterval: statsInterval,
- FillPercent: c.Float64("fill-percent"),
- NoSync: c.Bool("no-sync"),
- Clean: !c.Bool("work"),
- Path: c.String("path"),
- })
- },
- }}
- return app
-}
-
-var logger = log.New(os.Stderr, "", 0)
-var logBuffer *bytes.Buffer
-
-func print(v ...interface{}) {
- if testMode {
- logger.Print(v...)
- } else {
- fmt.Print(v...)
+ // Print summary of errors.
+ if count > 0 {
+ fmt.Fprintf(cmd.Stdout, "%d errors found\n", count)
+ return ErrCorrupt
+ }
+
+ // Notify user that database is valid.
+ fmt.Fprintln(cmd.Stdout, "OK")
+ return nil
+ })
+}
+
+// InfoCommand represents the "info" command execution.
+type InfoCommand struct {
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
+}
+
+// NewInfoCommand returns a InfoCommand.
+func newInfoCommand(m *Main) *InfoCommand {
+ return &InfoCommand{
+ Stdin: m.Stdin,
+ Stdout: m.Stdout,
+ Stderr: m.Stderr,
}
}
-func printf(format string, v ...interface{}) {
- if testMode {
- logger.Printf(format, v...)
- } else {
- fmt.Printf(format, v...)
+// Run executes the command.
+func (cmd *InfoCommand) Run(args ...string) error {
+ // Parse flags.
+ fs := flag.NewFlagSet("", flag.ContinueOnError)
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+
+ // Require database path.
+ path := fs.Arg(0)
+ if path == "" {
+ return ErrPathRequired
+ } else if _, err := os.Stat(path); os.IsNotExist(err) {
+ return ErrFileNotFound
+ }
+
+ // Open the database.
+ db, err := bolt.Open(path, 0666, nil)
+ if err != nil {
+ return err
}
+ defer db.Close()
+
+ // Print basic database info.
+ info := db.Info()
+ fmt.Fprintf(cmd.Stdout, "Page Size: %d\n", info.PageSize)
+
+ return nil
}
-func println(v ...interface{}) {
- if testMode {
- logger.Println(v...)
- } else {
- fmt.Println(v...)
+// PagesCommand represents the "pages" command execution.
+type PagesCommand struct {
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
+}
+
+// NewPagesCommand returns a PagesCommand.
+func newPagesCommand(m *Main) *PagesCommand {
+ return &PagesCommand{
+ Stdin: m.Stdin,
+ Stdout: m.Stdout,
+ Stderr: m.Stderr,
}
}
-func fatal(v ...interface{}) {
- logger.Print(v...)
- if !testMode {
- os.Exit(1)
+// Run executes the command.
+func (cmd *PagesCommand) Run(args ...string) error {
+ // Parse flags.
+ fs := flag.NewFlagSet("", flag.ContinueOnError)
+ if err := fs.Parse(args); err != nil {
+ return err
}
+
+ // Require database path.
+ path := fs.Arg(0)
+ if path == "" {
+ return ErrPathRequired
+ } else if _, err := os.Stat(path); os.IsNotExist(err) {
+ return ErrFileNotFound
+ }
+
+ // Open database.
+ db, err := bolt.Open(path, 0666, nil)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = db.Close() }()
+
+ // Write header.
+ fmt.Fprintln(cmd.Stdout, "ID TYPE ITEMS OVRFLW")
+ fmt.Fprintln(cmd.Stdout, "======== ========== ====== ======")
+
+ return db.Update(func(tx *bolt.Tx) error {
+ var id int
+ for {
+ p, err := tx.Page(id)
+ if err != nil {
+ return &PageError{ID: id, Err: err}
+ } else if p == nil {
+ break
+ }
+
+ // Only display count and overflow if this is a non-free page.
+ var count, overflow string
+ if p.Type != "free" {
+ count = strconv.Itoa(p.Count)
+ if p.OverflowCount > 0 {
+ overflow = strconv.Itoa(p.OverflowCount)
+ }
+ }
+
+ // Print table row.
+ fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow)
+
+ // Move to the next non-overflow page.
+ id += 1
+ if p.Type != "free" {
+ id += p.OverflowCount
+ }
+ }
+ return nil
+ })
}
-func fatalf(format string, v ...interface{}) {
- logger.Printf(format, v...)
- if !testMode {
- os.Exit(1)
+// StatsCommand represents the "stats" command execution.
+type StatsCommand struct {
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
+}
+
+// NewStatsCommand returns a StatsCommand.
+func newStatsCommand(m *Main) *StatsCommand {
+ return &StatsCommand{
+ Stdin: m.Stdin,
+ Stdout: m.Stdout,
+ Stderr: m.Stderr,
}
}
-func fatalln(v ...interface{}) {
- logger.Println(v...)
- if !testMode {
- os.Exit(1)
+// Run executes the command.
+func (cmd *StatsCommand) Run(args ...string) error {
+ // Parse flags.
+ fs := flag.NewFlagSet("", flag.ContinueOnError)
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+
+ // Require database path.
+ path, prefix := fs.Arg(0), fs.Arg(1)
+ if path == "" {
+ return ErrPathRequired
+ } else if _, err := os.Stat(path); os.IsNotExist(err) {
+ return ErrFileNotFound
}
+
+ // Open database.
+ db, err := bolt.Open(path, 0666, nil)
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+
+ return db.View(func(tx *bolt.Tx) error {
+ var s bolt.BucketStats
+ var count int
+ if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
+ if bytes.HasPrefix(name, []byte(prefix)) {
+ s.Add(b.Stats())
+ count += 1
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ fmt.Fprintf(cmd.Stdout, "Aggregate statistics for %d buckets\n\n", count)
+
+ fmt.Fprintln(cmd.Stdout, "Page count statistics")
+ fmt.Fprintf(cmd.Stdout, "\tNumber of logical branch pages: %d\n", s.BranchPageN)
+ fmt.Fprintf(cmd.Stdout, "\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN)
+ fmt.Fprintf(cmd.Stdout, "\tNumber of logical leaf pages: %d\n", s.LeafPageN)
+ fmt.Fprintf(cmd.Stdout, "\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN)
+
+ fmt.Fprintln(cmd.Stdout, "Tree statistics")
+ fmt.Fprintf(cmd.Stdout, "\tNumber of keys/value pairs: %d\n", s.KeyN)
+ fmt.Fprintf(cmd.Stdout, "\tNumber of levels in B+tree: %d\n", s.Depth)
+
+ fmt.Fprintln(cmd.Stdout, "Page size utilization")
+ fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc)
+ var percentage int
+ if s.BranchAlloc != 0 {
+ percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc))
+ }
+ fmt.Fprintf(cmd.Stdout, "\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage)
+ fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc)
+ percentage = 0
+ if s.LeafAlloc != 0 {
+ percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc))
+ }
+ fmt.Fprintf(cmd.Stdout, "\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage)
+
+ fmt.Fprintln(cmd.Stdout, "Bucket statistics")
+ fmt.Fprintf(cmd.Stdout, "\tTotal number of buckets: %d\n", s.BucketN)
+ percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN))
+ fmt.Fprintf(cmd.Stdout, "\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage)
+ percentage = 0
+ if s.LeafInuse != 0 {
+ percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse))
+ }
+ fmt.Fprintf(cmd.Stdout, "\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage)
+
+ return nil
+ })
}
-// LogBuffer returns the contents of the log.
-// This only works while the CLI is in test mode.
-func LogBuffer() string {
- if logBuffer != nil {
- return logBuffer.String()
+var benchBucketName = []byte("bench")
+
+// BenchCommand represents the "bench" command execution.
+type BenchCommand struct {
+ Stdin io.Reader
+ Stdout io.Writer
+ Stderr io.Writer
+}
+
+// NewBenchCommand returns a BenchCommand using the
+func newBenchCommand(m *Main) *BenchCommand {
+ return &BenchCommand{
+ Stdin: m.Stdin,
+ Stdout: m.Stdout,
+ Stderr: m.Stderr,
}
- return ""
}
-var testMode bool
+// Run executes the "bench" command.
+func (cmd *BenchCommand) Run(args ...string) error {
+ // Parse CLI arguments.
+ options, err := cmd.ParseFlags(args)
+ if err != nil {
+ return err
+ }
-// SetTestMode sets whether the CLI is running in test mode and resets the logger.
-func SetTestMode(value bool) {
- testMode = value
- if testMode {
- logBuffer = bytes.NewBuffer(nil)
- logger = log.New(logBuffer, "", 0)
+ // Remove path if "-work" is not set. Otherwise keep path.
+ if options.Work {
+ fmt.Fprintf(cmd.Stdout, "work: %s\n", options.Path)
} else {
- logger = log.New(os.Stderr, "", 0)
+ defer os.Remove(options.Path)
+ }
+
+ // Create database.
+ db, err := bolt.Open(options.Path, 0666, nil)
+ if err != nil {
+ return err
+ }
+ db.NoSync = options.NoSync
+ defer db.Close()
+
+ // Write to the database.
+ var results BenchResults
+ if err := cmd.runWrites(db, options, &results); err != nil {
+ return fmt.Errorf("write: ", err)
+ }
+
+ // Read from the database.
+ if err := cmd.runReads(db, options, &results); err != nil {
+ return fmt.Errorf("bench: read: %s", err)
+ }
+
+ // Print results.
+ fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond())
+ fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond())
+ fmt.Fprintln(os.Stderr, "")
+ return nil
+}
+
+// ParseFlags parses the command line flags.
+func (cmd *BenchCommand) ParseFlags(args []string) (*BenchOptions, error) {
+ var options BenchOptions
+
+ // Parse flagset.
+ fs := flag.NewFlagSet("", flag.ContinueOnError)
+ fs.StringVar(&options.ProfileMode, "profile-mode", "rw", "")
+ fs.StringVar(&options.WriteMode, "write-mode", "seq", "")
+ fs.StringVar(&options.ReadMode, "read-mode", "seq", "")
+ fs.IntVar(&options.Iterations, "count", 1000, "")
+ fs.IntVar(&options.BatchSize, "batch-size", 0, "")
+ fs.IntVar(&options.KeySize, "key-size", 8, "")
+ fs.IntVar(&options.ValueSize, "value-size", 32, "")
+ fs.StringVar(&options.CPUProfile, "cpuprofile", "", "")
+ fs.StringVar(&options.MemProfile, "memprofile", "", "")
+ fs.StringVar(&options.BlockProfile, "blockprofile", "", "")
+ fs.StringVar(&options.BlockProfile, "blockprofile", "", "")
+ fs.Float64Var(&options.FillPercent, "fill-percent", bolt.DefaultFillPercent, "")
+ fs.BoolVar(&options.NoSync, "no-sync", false, "")
+ fs.BoolVar(&options.Work, "work", false, "")
+ fs.StringVar(&options.Path, "path", "", "")
+ fs.SetOutput(cmd.Stderr)
+ if err := fs.Parse(args); err != nil {
+ return nil, err
+ }
+
+ // Set batch size to iteration size if not set.
+ // Require that batch size can be evenly divided by the iteration count.
+ if options.BatchSize == 0 {
+ options.BatchSize = options.Iterations
+ } else if options.Iterations%options.BatchSize != 0 {
+ return nil, ErrNonDivisibleBatchSize
+ }
+
+ // Generate temp path if one is not passed in.
+ if options.Path == "" {
+ f, err := ioutil.TempFile("", "bolt-bench-")
+ if err != nil {
+ return nil, fmt.Errorf("temp file: %s", err)
+ }
+ f.Close()
+ os.Remove(f.Name())
+ options.Path = f.Name()
+ }
+
+ return &options, nil
+}
+
+// Writes to the database.
+func (cmd *BenchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ // Start profiling for writes.
+ if options.ProfileMode == "rw" || options.ProfileMode == "w" {
+ cmd.startProfiling(options)
}
+
+ t := time.Now()
+
+ var err error
+ switch options.WriteMode {
+ case "seq":
+ err = cmd.runWritesSequential(db, options, results)
+ case "rnd":
+ err = cmd.runWritesRandom(db, options, results)
+ case "seq-nest":
+ err = cmd.runWritesSequentialNested(db, options, results)
+ case "rnd-nest":
+ err = cmd.runWritesRandomNested(db, options, results)
+ default:
+ return fmt.Errorf("invalid write mode: %s", options.WriteMode)
+ }
+
+ // Save time to write.
+ results.WriteDuration = time.Since(t)
+
+ // Stop profiling for writes only.
+ if options.ProfileMode == "w" {
+ cmd.stopProfiling()
+ }
+
+ return err
+}
+
+func (cmd *BenchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ var i = uint32(0)
+ return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
+}
+
+func (cmd *BenchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ r := rand.New(rand.NewSource(time.Now().UnixNano()))
+ return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
+}
+
+func (cmd *BenchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ var i = uint32(0)
+ return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i })
+}
+
+func (cmd *BenchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ r := rand.New(rand.NewSource(time.Now().UnixNano()))
+ return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() })
+}
+
+func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
+ results.WriteOps = options.Iterations
+
+ for i := 0; i < options.Iterations; i += options.BatchSize {
+ if err := db.Update(func(tx *bolt.Tx) error {
+ b, _ := tx.CreateBucketIfNotExists(benchBucketName)
+ b.FillPercent = options.FillPercent
+
+ for j := 0; j < options.BatchSize; j++ {
+ key := make([]byte, options.KeySize)
+ value := make([]byte, options.ValueSize)
+
+ // Write key as uint32.
+ binary.BigEndian.PutUint32(key, keySource())
+
+ // Insert key/value.
+ if err := b.Put(key, value); err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (cmd *BenchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error {
+ results.WriteOps = options.Iterations
+
+ for i := 0; i < options.Iterations; i += options.BatchSize {
+ if err := db.Update(func(tx *bolt.Tx) error {
+ top, err := tx.CreateBucketIfNotExists(benchBucketName)
+ if err != nil {
+ return err
+ }
+ top.FillPercent = options.FillPercent
+
+ // Create bucket key.
+ name := make([]byte, options.KeySize)
+ binary.BigEndian.PutUint32(name, keySource())
+
+ // Create bucket.
+ b, err := top.CreateBucketIfNotExists(name)
+ if err != nil {
+ return err
+ }
+ b.FillPercent = options.FillPercent
+
+ for j := 0; j < options.BatchSize; j++ {
+ var key = make([]byte, options.KeySize)
+ var value = make([]byte, options.ValueSize)
+
+ // Generate key as uint32.
+ binary.BigEndian.PutUint32(key, keySource())
+
+ // Insert value into subbucket.
+ if err := b.Put(key, value); err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Reads from the database.
+func (cmd *BenchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ // Start profiling for reads.
+ if options.ProfileMode == "r" {
+ cmd.startProfiling(options)
+ }
+
+ t := time.Now()
+
+ var err error
+ switch options.ReadMode {
+ case "seq":
+ switch options.WriteMode {
+ case "seq-nest", "rnd-nest":
+ err = cmd.runReadsSequentialNested(db, options, results)
+ default:
+ err = cmd.runReadsSequential(db, options, results)
+ }
+ default:
+ return fmt.Errorf("invalid read mode: %s", options.ReadMode)
+ }
+
+ // Save read time.
+ results.ReadDuration = time.Since(t)
+
+ // Stop profiling for reads.
+ if options.ProfileMode == "rw" || options.ProfileMode == "r" {
+ cmd.stopProfiling()
+ }
+
+ return err
+}
+
+func (cmd *BenchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ return db.View(func(tx *bolt.Tx) error {
+ t := time.Now()
+
+ for {
+ var count int
+
+ c := tx.Bucket(benchBucketName).Cursor()
+ for k, v := c.First(); k != nil; k, v = c.Next() {
+ if v == nil {
+ return errors.New("invalid value")
+ }
+ count++
+ }
+
+ if options.WriteMode == "seq" && count != options.Iterations {
+ return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count)
+ }
+
+ results.ReadOps += count
+
+ // Make sure we do this for at least a second.
+ if time.Since(t) >= time.Second {
+ break
+ }
+ }
+
+ return nil
+ })
+}
+
+func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error {
+ return db.View(func(tx *bolt.Tx) error {
+ t := time.Now()
+
+ for {
+ var count int
+ var top = tx.Bucket(benchBucketName)
+ if err := top.ForEach(func(name, _ []byte) error {
+ c := top.Bucket(name).Cursor()
+ for k, v := c.First(); k != nil; k, v = c.Next() {
+ if v == nil {
+ return ErrInvalidValue
+ }
+ count++
+ }
+ return nil
+ }); err != nil {
+ return err
+ }
+
+ if options.WriteMode == "seq-nest" && count != options.Iterations {
+ return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count)
+ }
+
+ results.ReadOps += count
+
+ // Make sure we do this for at least a second.
+ if time.Since(t) >= time.Second {
+ break
+ }
+ }
+
+ return nil
+ })
+}
+
+// File handlers for the various profiles.
+var cpuprofile, memprofile, blockprofile *os.File
+
+// Starts all profiles set on the options.
+func (cmd *BenchCommand) startProfiling(options *BenchOptions) {
+ var err error
+
+ // Start CPU profiling.
+ if options.CPUProfile != "" {
+ cpuprofile, err = os.Create(options.CPUProfile)
+ if err != nil {
+ fmt.Fprintf(cmd.Stderr, "bench: could not create cpu profile %q: %v\n", options.CPUProfile, err)
+ os.Exit(1)
+ }
+ pprof.StartCPUProfile(cpuprofile)
+ }
+
+ // Start memory profiling.
+ if options.MemProfile != "" {
+ memprofile, err = os.Create(options.MemProfile)
+ if err != nil {
+ fmt.Fprintf(cmd.Stderr, "bench: could not create memory profile %q: %v\n", options.MemProfile, err)
+ os.Exit(1)
+ }
+ runtime.MemProfileRate = 4096
+ }
+
+ // Start fatal profiling.
+ if options.BlockProfile != "" {
+ blockprofile, err = os.Create(options.BlockProfile)
+ if err != nil {
+ fmt.Fprintf(cmd.Stderr, "bench: could not create block profile %q: %v\n", options.BlockProfile, err)
+ os.Exit(1)
+ }
+ runtime.SetBlockProfileRate(1)
+ }
+}
+
+// Stops all profiles.
+func (cmd *BenchCommand) stopProfiling() {
+ if cpuprofile != nil {
+ pprof.StopCPUProfile()
+ cpuprofile.Close()
+ cpuprofile = nil
+ }
+
+ if memprofile != nil {
+ pprof.Lookup("heap").WriteTo(memprofile, 0)
+ memprofile.Close()
+ memprofile = nil
+ }
+
+ if blockprofile != nil {
+ pprof.Lookup("block").WriteTo(blockprofile, 0)
+ blockprofile.Close()
+ blockprofile = nil
+ runtime.SetBlockProfileRate(0)
+ }
+}
+
+// BenchOptions represents the set of options that can be passed to "bolt bench".
+type BenchOptions struct {
+ ProfileMode string
+ WriteMode string
+ ReadMode string
+ Iterations int
+ BatchSize int
+ KeySize int
+ ValueSize int
+ CPUProfile string
+ MemProfile string
+ BlockProfile string
+ StatsInterval time.Duration
+ FillPercent float64
+ NoSync bool
+ Work bool
+ Path string
+}
+
+// BenchResults represents the performance results of the benchmark.
+type BenchResults struct {
+ WriteOps int
+ WriteDuration time.Duration
+ ReadOps int
+ ReadDuration time.Duration
+}
+
+// Returns the duration for a single write operation.
+func (r *BenchResults) WriteOpDuration() time.Duration {
+ if r.WriteOps == 0 {
+ return 0
+ }
+ return r.WriteDuration / time.Duration(r.WriteOps)
+}
+
+// Returns average number of write operations that can be performed per second.
+func (r *BenchResults) WriteOpsPerSecond() int {
+ var op = r.WriteOpDuration()
+ if op == 0 {
+ return 0
+ }
+ return int(time.Second) / int(op)
+}
+
+// Returns the duration for a single read operation.
+func (r *BenchResults) ReadOpDuration() time.Duration {
+ if r.ReadOps == 0 {
+ return 0
+ }
+ return r.ReadDuration / time.Duration(r.ReadOps)
+}
+
+// Returns average number of read operations that can be performed per second.
+func (r *BenchResults) ReadOpsPerSecond() int {
+ var op = r.ReadOpDuration()
+ if op == 0 {
+ return 0
+ }
+ return int(time.Second) / int(op)
+}
+
+type PageError struct {
+ ID int
+ Err error
+}
+
+func (e *PageError) Error() string {
+ return fmt.Sprintf("page error: id=%d, err=%s", e.ID, e.Err)
}
diff --git a/cmd/bolt/main_test.go b/cmd/bolt/main_test.go
index 4448d6e..b9e8c67 100644
--- a/cmd/bolt/main_test.go
+++ b/cmd/bolt/main_test.go
@@ -1,69 +1,145 @@
package main_test
import (
- "fmt"
+ "bytes"
"io/ioutil"
"os"
- "path/filepath"
- "reflect"
- "runtime"
- "strings"
+ "strconv"
"testing"
"github.com/boltdb/bolt"
- . "github.com/boltdb/bolt/cmd/bolt"
+ "github.com/boltdb/bolt/cmd/bolt"
)
-// open creates and opens a Bolt database in the temp directory.
-func open(fn func(*bolt.DB, string)) {
- path := tempfile()
- defer os.RemoveAll(path)
+// Ensure the "info" command can print information about a database.
+func TestInfoCommand_Run(t *testing.T) {
+ db := MustOpen(0666, nil)
+ db.DB.Close()
+ defer db.Close()
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- panic("db open error: " + err.Error())
+ // Run the info command.
+ m := NewMain()
+ if err := m.Run("info", db.Path); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Ensure the "stats" command can execute correctly.
+func TestStatsCommand_Run(t *testing.T) {
+ // Ignore
+ if os.Getpagesize() != 4096 {
+ t.Skip("system does not use 4KB page size")
+ }
+
+ db := MustOpen(0666, nil)
+ defer db.Close()
+
+ if err := db.Update(func(tx *bolt.Tx) error {
+ // Create "foo" bucket.
+ b, err := tx.CreateBucket([]byte("foo"))
+ if err != nil {
+ return err
+ }
+ for i := 0; i < 10; i++ {
+ if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil {
+ return err
+ }
+ }
+
+ // Create "bar" bucket.
+ b, err = tx.CreateBucket([]byte("bar"))
+ if err != nil {
+ return err
+ }
+ for i := 0; i < 100; i++ {
+ if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil {
+ return err
+ }
+ }
+
+ // Create "baz" bucket.
+ b, err = tx.CreateBucket([]byte("baz"))
+ if err != nil {
+ return err
+ }
+ if err := b.Put([]byte("key"), []byte("value")); err != nil {
+ return err
+ }
+
+ return nil
+ }); err != nil {
+ t.Fatal(err)
}
- fn(db, path)
+ db.DB.Close()
+
+ // Generate expected result.
+ exp := "Aggregate statistics for 3 buckets\n\n" +
+ "Page count statistics\n" +
+ "\tNumber of logical branch pages: 0\n" +
+ "\tNumber of physical branch overflow pages: 0\n" +
+ "\tNumber of logical leaf pages: 1\n" +
+ "\tNumber of physical leaf overflow pages: 0\n" +
+ "Tree statistics\n" +
+ "\tNumber of keys/value pairs: 111\n" +
+ "\tNumber of levels in B+tree: 1\n" +
+ "Page size utilization\n" +
+ "\tBytes allocated for physical branch pages: 0\n" +
+ "\tBytes actually used for branch data: 0 (0%)\n" +
+ "\tBytes allocated for physical leaf pages: 4096\n" +
+ "\tBytes actually used for leaf data: 1996 (48%)\n" +
+ "Bucket statistics\n" +
+ "\tTotal number of buckets: 3\n" +
+ "\tTotal number on inlined buckets: 2 (66%)\n" +
+ "\tBytes used for inlined buckets: 236 (11%)\n"
+
+ // Run the command.
+ m := NewMain()
+ if err := m.Run("stats", db.Path); err != nil {
+ t.Fatal(err)
+ } else if m.Stdout.String() != exp {
+ t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String())
+ }
+}
+
+// Main represents a test wrapper for main.Main that records output.
+type Main struct {
+ *main.Main
+ Stdin bytes.Buffer
+ Stdout bytes.Buffer
+ Stderr bytes.Buffer
}
-// run executes a command against the CLI and returns the output.
-func run(args ...string) string {
- args = append([]string{"bolt"}, args...)
- NewApp().Run(args)
- return strings.TrimSpace(LogBuffer())
+// NewMain returns a new instance of Main.
+func NewMain() *Main {
+ m := &Main{Main: main.NewMain()}
+ m.Main.Stdin = &m.Stdin
+ m.Main.Stdout = &m.Stdout
+ m.Main.Stderr = &m.Stderr
+ return m
}
-// tempfile returns a temporary file path.
-func tempfile() string {
+// MustOpen creates a Bolt database in a temporary location.
+func MustOpen(mode os.FileMode, options *bolt.Options) *DB {
+ // Create temporary path.
f, _ := ioutil.TempFile("", "bolt-")
f.Close()
os.Remove(f.Name())
- return f.Name()
-}
-// assert fails the test if the condition is false.
-func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
- if !condition {
- _, file, line, _ := runtime.Caller(1)
- fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
- tb.FailNow()
+ db, err := bolt.Open(f.Name(), mode, options)
+ if err != nil {
+ panic(err.Error())
}
+ return &DB{DB: db, Path: f.Name()}
}
-// ok fails the test if an err is not nil.
-func ok(tb testing.TB, err error) {
- if err != nil {
- _, file, line, _ := runtime.Caller(1)
- fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
- tb.FailNow()
- }
+// DB is a test wrapper for bolt.DB.
+type DB struct {
+ *bolt.DB
+ Path string
}
-// equals fails the test if exp is not equal to act.
-func equals(tb testing.TB, exp, act interface{}) {
- if !reflect.DeepEqual(exp, act) {
- _, file, line, _ := runtime.Caller(1)
- fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
- tb.FailNow()
- }
+// Close closes and removes the database.
+func (db *DB) Close() error {
+ defer os.Remove(db.Path)
+ return db.DB.Close()
}
diff --git a/cmd/bolt/pages.go b/cmd/bolt/pages.go
deleted file mode 100644
index ec1c4b4..0000000
--- a/cmd/bolt/pages.go
+++ /dev/null
@@ -1,57 +0,0 @@
-package main
-
-import (
- "os"
- "strconv"
-
- "github.com/boltdb/bolt"
-)
-
-// Pages prints a list of all pages in a database.
-func Pages(path string) {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- fatal(err)
- return
- }
-
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- defer db.Close()
-
- println("ID TYPE ITEMS OVRFLW")
- println("======== ========== ====== ======")
-
- db.Update(func(tx *bolt.Tx) error {
- var id int
- for {
- p, err := tx.Page(id)
- if err != nil {
- fatalf("page error: %d: %s", id, err)
- } else if p == nil {
- break
- }
-
- // Only display count and overflow if this is a non-free page.
- var count, overflow string
- if p.Type != "free" {
- count = strconv.Itoa(p.Count)
- if p.OverflowCount > 0 {
- overflow = strconv.Itoa(p.OverflowCount)
- }
- }
-
- // Print table row.
- printf("%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow)
-
- // Move to the next non-overflow page.
- id += 1
- if p.Type != "free" {
- id += p.OverflowCount
- }
- }
- return nil
- })
-}
diff --git a/cmd/bolt/stats.go b/cmd/bolt/stats.go
deleted file mode 100644
index b5d0083..0000000
--- a/cmd/bolt/stats.go
+++ /dev/null
@@ -1,77 +0,0 @@
-package main
-
-import (
- "bytes"
- "os"
-
- "github.com/boltdb/bolt"
-)
-
-// Collect stats for all top level buckets matching the prefix.
-func Stats(path, prefix string) {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- fatal(err)
- return
- }
-
- db, err := bolt.Open(path, 0600, nil)
- if err != nil {
- fatal(err)
- return
- }
- defer db.Close()
-
- err = db.View(func(tx *bolt.Tx) error {
- var s bolt.BucketStats
- var count int
- var prefix = []byte(prefix)
- tx.ForEach(func(name []byte, b *bolt.Bucket) error {
- if bytes.HasPrefix(name, prefix) {
- s.Add(b.Stats())
- count += 1
- }
- return nil
- })
- printf("Aggregate statistics for %d buckets\n\n", count)
-
- println("Page count statistics")
- printf("\tNumber of logical branch pages: %d\n", s.BranchPageN)
- printf("\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN)
- printf("\tNumber of logical leaf pages: %d\n", s.LeafPageN)
- printf("\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN)
-
- println("Tree statistics")
- printf("\tNumber of keys/value pairs: %d\n", s.KeyN)
- printf("\tNumber of levels in B+tree: %d\n", s.Depth)
-
- println("Page size utilization")
- printf("\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc)
- var percentage int
- if s.BranchAlloc != 0 {
- percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc))
- }
- printf("\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage)
- printf("\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc)
- percentage = 0
- if s.LeafAlloc != 0 {
- percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc))
- }
- printf("\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage)
-
- println("Bucket statistics")
- printf("\tTotal number of buckets: %d\n", s.BucketN)
- percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN))
- printf("\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage)
- percentage = 0
- if s.LeafInuse != 0 {
- percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse))
- }
- printf("\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage)
-
- return nil
- })
- if err != nil {
- fatal(err)
- return
- }
-}
diff --git a/cmd/bolt/stats_test.go b/cmd/bolt/stats_test.go
deleted file mode 100644
index 44ed434..0000000
--- a/cmd/bolt/stats_test.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package main_test
-
-import (
- "os"
- "strconv"
- "testing"
-
- "github.com/boltdb/bolt"
- . "github.com/boltdb/bolt/cmd/bolt"
-)
-
-func TestStats(t *testing.T) {
- if os.Getpagesize() != 4096 {
- t.Skip()
- }
- SetTestMode(true)
- open(func(db *bolt.DB, path string) {
- db.Update(func(tx *bolt.Tx) error {
- b, err := tx.CreateBucket([]byte("foo"))
- if err != nil {
- return err
- }
- for i := 0; i < 10; i++ {
- b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i)))
- }
- b, err = tx.CreateBucket([]byte("bar"))
- if err != nil {
- return err
- }
- for i := 0; i < 100; i++ {
- b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i)))
- }
- b, err = tx.CreateBucket([]byte("baz"))
- if err != nil {
- return err
- }
- b.Put([]byte("key"), []byte("value"))
- return nil
- })
- db.Close()
- output := run("stats", path, "b")
- equals(t, "Aggregate statistics for 2 buckets\n\n"+
- "Page count statistics\n"+
- "\tNumber of logical branch pages: 0\n"+
- "\tNumber of physical branch overflow pages: 0\n"+
- "\tNumber of logical leaf pages: 1\n"+
- "\tNumber of physical leaf overflow pages: 0\n"+
- "Tree statistics\n"+
- "\tNumber of keys/value pairs: 101\n"+
- "\tNumber of levels in B+tree: 1\n"+
- "Page size utilization\n"+
- "\tBytes allocated for physical branch pages: 0\n"+
- "\tBytes actually used for branch data: 0 (0%)\n"+
- "\tBytes allocated for physical leaf pages: 4096\n"+
- "\tBytes actually used for leaf data: 1996 (48%)\n"+
- "Bucket statistics\n"+
- "\tTotal number of buckets: 2\n"+
- "\tTotal number on inlined buckets: 1 (50%)\n"+
- "\tBytes used for inlined buckets: 40 (2%)", output)
- })
-}