From c4ad027df774d84c29c9cc53436071c87138c60d Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Fri, 9 May 2014 18:16:38 +0000 Subject: aggregate bucket stats recursively and add stats to cmd --- bucket_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'bucket_test.go') diff --git a/bucket_test.go b/bucket_test.go index b87031b..626927a 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -628,6 +628,50 @@ func TestBucket_Stats_Small(t *testing.T) { }) } +// Ensure a bucket can calculate stats. +func TestBucket_Stats_Nested(t *testing.T) { + + withOpenDB(func(db *DB, path string) { + db.Update(func(tx *Tx) error { + b, err := tx.CreateBucket([]byte("foo")) + assert.NoError(t, err) + for i := 0; i < 100; i++ { + b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + bar, err := b.CreateBucket([]byte("bar")) + assert.NoError(t, err) + for i := 0; i < 10; i++ { + bar.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + baz, err := b.CreateBucket([]byte("baz")) + assert.NoError(t, err) + for i := 0; i < 10; i++ { + baz.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + } + return nil + }) + mustCheck(db) + db.View(func(tx *Tx) error { + b := tx.Bucket([]byte("foo")) + stats := b.Stats() + assert.Equal(t, stats.BranchPageN, 0) + assert.Equal(t, stats.BranchOverflowN, 0) + assert.Equal(t, stats.LeafPageN, 3) + assert.Equal(t, stats.LeafOverflowN, 0) + assert.Equal(t, stats.KeyN, 122) + assert.Equal(t, stats.Depth, 2) + if os.Getpagesize() != 4096 { + // Incompatible page size + assert.Equal(t, stats.BranchInuse, 0) + assert.Equal(t, stats.BranchAlloc, 0) + assert.Equal(t, stats.LeafInuse, 38) + assert.Equal(t, stats.LeafAlloc, 4096) + } + return nil + }) + }) +} + // Ensure a large bucket can calculate stats. func TestBucket_Stats_Large(t *testing.T) { if testing.Short() { -- cgit v1.2.3 From deffc06a05af5b39f8b06c29768d2300fedbcc76 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 17:29:16 +0000 Subject: fix inline bucket stats --- bucket.go | 32 ++++++++++++++---- bucket_test.go | 102 ++++++++++++++++++++++++++++++++------------------------- 2 files changed, 83 insertions(+), 51 deletions(-) (limited to 'bucket_test.go') diff --git a/bucket.go b/bucket.go index 6f7bbd2..a2463a3 100644 --- a/bucket.go +++ b/bucket.go @@ -132,11 +132,6 @@ func (b *Bucket) Bucket(name []byte) *Bucket { var child = b.openBucket(v) b.buckets[string(name)] = child - // Save a reference to the inline page if the bucket is inline. - if child.root == 0 { - child.page = (*page)(unsafe.Pointer(&v[bucketHeaderSize])) - } - return child } @@ -146,6 +141,12 @@ func (b *Bucket) openBucket(value []byte) *Bucket { var child = newBucket(b.tx) child.bucket = &bucket{} *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) + + // Save a reference to the inline page if the bucket is inline. + if child.root == 0 { + child.page = (*page)(unsafe.Pointer(&value[bucketHeaderSize])) + } + return &child } @@ -363,8 +364,18 @@ func (b *Bucket) ForEach(fn func(k, v []byte) error) error { func (b *Bucket) Stats() BucketStats { var s, subStats BucketStats pageSize := b.tx.db.pageSize + s.BucketN += 1 + if b.root == 0 { + s.InlineBucketN += 1 + } b.forEachPage(func(p *page, depth int) { - if (p.flags & leafPageFlag) != 0 { + if b.root == 0 { // inline bucket + s.KeyN += int(p.count) + lastElement := p.leafPageElement(p.count - 1) + used := bucketHeaderSize + pageHeaderSize + (leafPageElementSize * int(p.count-1)) + used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) + s.InlineBucketInuse += used + } else if (p.flags & leafPageFlag) != 0 { s.LeafPageN++ if p.count == 0 { return @@ -663,6 +674,11 @@ type BucketStats struct { BranchInuse int // bytes actually used for branch data LeafAlloc int // bytes allocated for physical leaf pages LeafInuse int // bytes actually used for leaf data + + // Bucket statistics + BucketN int // total number of buckets including the top bucket + InlineBucketN int // total number on inlined buckets + InlineBucketInuse int // bytes used for inlined buckets (also accounted for in LeafInuse) } func (s *BucketStats) Add(other BucketStats) { @@ -678,6 +694,10 @@ func (s *BucketStats) Add(other BucketStats) { s.BranchInuse += other.BranchInuse s.LeafAlloc += other.LeafAlloc s.LeafInuse += other.LeafInuse + + s.BucketN += other.BucketN + s.InlineBucketN += other.InlineBucketN + s.InlineBucketInuse += other.InlineBucketInuse } // cloneBytes returns a copy of a given slice. diff --git a/bucket_test.go b/bucket_test.go index 626927a..b53a74b 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -576,19 +576,22 @@ func TestBucket_Stats(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("woojits")) stats := b.Stats() - assert.Equal(t, stats.BranchPageN, 1) - assert.Equal(t, stats.BranchOverflowN, 0) - assert.Equal(t, stats.LeafPageN, 6) - assert.Equal(t, stats.LeafOverflowN, 2) - assert.Equal(t, stats.KeyN, 501) - assert.Equal(t, stats.Depth, 2) - if os.Getpagesize() != 4096 { + assert.Equal(t, 1, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 6, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 2, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 501, stats.KeyN, "KeyN") + assert.Equal(t, 2, stats.Depth, "Depth") + assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 20908, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, stats.BranchInuse, 125) - assert.Equal(t, stats.BranchAlloc, 4096) - assert.Equal(t, stats.LeafInuse, 20908) - assert.Equal(t, stats.LeafAlloc, 32768) + assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 32768, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 0, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, 0, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -610,19 +613,22 @@ func TestBucket_Stats_Small(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("whozawhats")) stats := b.Stats() - assert.Equal(t, 0, stats.BranchPageN) - assert.Equal(t, 0, stats.BranchOverflowN) - assert.Equal(t, 1, stats.LeafPageN) - assert.Equal(t, 0, stats.LeafOverflowN) - assert.Equal(t, 1, stats.KeyN) - assert.Equal(t, 1, stats.Depth) - if os.Getpagesize() != 4096 { + assert.Equal(t, 0, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 0, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 1, stats.KeyN, "KeyN") + assert.Equal(t, 1, stats.Depth, "Depth") + assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 0, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, 0, stats.BranchInuse) - assert.Equal(t, 0, stats.BranchAlloc) - assert.Equal(t, 38, stats.LeafInuse) - assert.Equal(t, 4096, stats.LeafAlloc) + assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 0, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, bucketHeaderSize+pageHeaderSize+leafPageElementSize+6, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -643,7 +649,7 @@ func TestBucket_Stats_Nested(t *testing.T) { for i := 0; i < 10; i++ { bar.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - baz, err := b.CreateBucket([]byte("baz")) + baz, err := bar.CreateBucket([]byte("baz")) assert.NoError(t, err) for i := 0; i < 10; i++ { baz.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) @@ -654,19 +660,22 @@ func TestBucket_Stats_Nested(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("foo")) stats := b.Stats() - assert.Equal(t, stats.BranchPageN, 0) - assert.Equal(t, stats.BranchOverflowN, 0) - assert.Equal(t, stats.LeafPageN, 3) - assert.Equal(t, stats.LeafOverflowN, 0) - assert.Equal(t, stats.KeyN, 122) - assert.Equal(t, stats.Depth, 2) - if os.Getpagesize() != 4096 { + assert.Equal(t, 0, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 2, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 122, stats.KeyN, "KeyN") + assert.Equal(t, 3, stats.Depth, "Depth") + assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 2474, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, stats.BranchInuse, 0) - assert.Equal(t, stats.BranchAlloc, 0) - assert.Equal(t, stats.LeafInuse, 38) - assert.Equal(t, stats.LeafAlloc, 4096) + assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 8192, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 3, stats.BucketN, "BucketN") + assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, 212, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -696,19 +705,22 @@ func TestBucket_Stats_Large(t *testing.T) { db.View(func(tx *Tx) error { b := tx.Bucket([]byte("widgets")) stats := b.Stats() - assert.Equal(t, 19, stats.BranchPageN) - assert.Equal(t, 0, stats.BranchOverflowN) - assert.Equal(t, 1291, stats.LeafPageN) - assert.Equal(t, 0, stats.LeafOverflowN) - assert.Equal(t, 100000, stats.KeyN) - assert.Equal(t, 3, stats.Depth) - if os.Getpagesize() != 4096 { + assert.Equal(t, 19, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 1291, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 100000, stats.KeyN, "KeyN") + assert.Equal(t, 3, stats.Depth, "Depth") + assert.Equal(t, 27007, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 2598436, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size - assert.Equal(t, 27289, stats.BranchInuse) - assert.Equal(t, 61440, stats.BranchAlloc) - assert.Equal(t, 2598276, stats.LeafInuse) - assert.Equal(t, 5246976, stats.LeafAlloc) + assert.Equal(t, 77824, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 5287936, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 0, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, 0, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) -- cgit v1.2.3 From 0dbfa3f08a0d2657cc7fc9b7b46af1f0aa271d45 Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 18:22:45 +0000 Subject: merge inline branch into leaf branch --- bucket.go | 39 ++++++++++++++++++--------------------- bucket_test.go | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 24 deletions(-) (limited to 'bucket_test.go') diff --git a/bucket.go b/bucket.go index a2463a3..fdc9852 100644 --- a/bucket.go +++ b/bucket.go @@ -369,29 +369,26 @@ func (b *Bucket) Stats() BucketStats { s.InlineBucketN += 1 } b.forEachPage(func(p *page, depth int) { - if b.root == 0 { // inline bucket + if (p.flags & leafPageFlag) != 0 { s.KeyN += int(p.count) - lastElement := p.leafPageElement(p.count - 1) - used := bucketHeaderSize + pageHeaderSize + (leafPageElementSize * int(p.count-1)) - used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) - s.InlineBucketInuse += used - } else if (p.flags & leafPageFlag) != 0 { - s.LeafPageN++ - if p.count == 0 { - return + used := pageHeaderSize + (leafPageElementSize * int(p.count)) + if p.count != 0 { + lastElement := p.leafPageElement(p.count - 1) + used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) } - s.KeyN += int(p.count) - lastElement := p.leafPageElement(p.count - 1) - used := pageHeaderSize + (leafPageElementSize * int(p.count-1)) - used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) - s.LeafInuse += used - s.LeafOverflowN += int(p.overflow) - - // Collect stats from sub-buckets - for i := uint16(0); i < p.count; i++ { - e := p.leafPageElement(i) - if (e.flags & bucketLeafFlag) != 0 { - subStats.Add(b.openBucket(e.value()).Stats()) + if b.root == 0 { + s.InlineBucketInuse += used + } else { + s.LeafPageN++ + s.LeafInuse += used + s.LeafOverflowN += int(p.overflow) + + // Collect stats from sub-buckets + for i := uint16(0); i < p.count; i++ { + e := p.leafPageElement(i) + if (e.flags & bucketLeafFlag) != 0 { + subStats.Add(b.openBucket(e.value()).Stats()) + } } } } else if (p.flags & branchPageFlag) != 0 { diff --git a/bucket_test.go b/bucket_test.go index b53a74b..11d1d77 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -583,7 +583,7 @@ func TestBucket_Stats(t *testing.T) { assert.Equal(t, 501, stats.KeyN, "KeyN") assert.Equal(t, 2, stats.Depth, "Depth") assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 20908, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 21004, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") @@ -634,6 +634,40 @@ func TestBucket_Stats_Small(t *testing.T) { }) } +func TestBucket_Stats_EmptyBucket(t *testing.T) { + + withOpenDB(func(db *DB, path string) { + db.Update(func(tx *Tx) error { + // Add a bucket that fits on a single root leaf. + _, err := tx.CreateBucket([]byte("whozawhats")) + assert.NoError(t, err) + return nil + }) + mustCheck(db) + db.View(func(tx *Tx) error { + b := tx.Bucket([]byte("whozawhats")) + stats := b.Stats() + assert.Equal(t, 0, stats.BranchPageN, "BranchPageN") + assert.Equal(t, 0, stats.BranchOverflowN, "BranchOverflowN") + assert.Equal(t, 0, stats.LeafPageN, "LeafPageN") + assert.Equal(t, 0, stats.LeafOverflowN, "LeafOverflowN") + assert.Equal(t, 0, stats.KeyN, "KeyN") + assert.Equal(t, 1, stats.Depth, "Depth") + assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") + assert.Equal(t, 0, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { + // Incompatible page size + assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") + assert.Equal(t, 0, stats.LeafAlloc, "LeafAlloc") + } + assert.Equal(t, 1, stats.BucketN, "BucketN") + assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") + assert.Equal(t, pageHeaderSize, stats.InlineBucketInuse, "InlineBucketInuse") + return nil + }) + }) +} + // Ensure a bucket can calculate stats. func TestBucket_Stats_Nested(t *testing.T) { @@ -667,7 +701,7 @@ func TestBucket_Stats_Nested(t *testing.T) { assert.Equal(t, 122, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2474, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 2506, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") @@ -712,7 +746,7 @@ func TestBucket_Stats_Large(t *testing.T) { assert.Equal(t, 100000, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 27007, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2598436, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 2619092, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 77824, stats.BranchAlloc, "BranchAlloc") -- cgit v1.2.3 From 0716cdaec8e4a0d12085cda61d58ae275028670b Mon Sep 17 00:00:00 2001 From: Martin Kobetic Date: Mon, 12 May 2014 19:33:53 +0000 Subject: tweaks --- bucket.go | 3 ++- bucket_test.go | 21 +++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) (limited to 'bucket_test.go') diff --git a/bucket.go b/bucket.go index fdc9852..4d45b38 100644 --- a/bucket.go +++ b/bucket.go @@ -371,8 +371,9 @@ func (b *Bucket) Stats() BucketStats { b.forEachPage(func(p *page, depth int) { if (p.flags & leafPageFlag) != 0 { s.KeyN += int(p.count) - used := pageHeaderSize + (leafPageElementSize * int(p.count)) + used := pageHeaderSize if p.count != 0 { + used += leafPageElementSize * int(p.count-1) lastElement := p.leafPageElement(p.count - 1) used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) } diff --git a/bucket_test.go b/bucket_test.go index 11d1d77..0be02e5 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -560,15 +560,15 @@ func TestBucket_Put_KeyTooLarge(t *testing.T) { // Ensure a bucket can calculate stats. func TestBucket_Stats(t *testing.T) { withOpenDB(func(db *DB, path string) { + big_key := []byte("really-big-value") db.Update(func(tx *Tx) error { // Add bucket with fewer keys but one big value. - _, err := tx.CreateBucket([]byte("woojits")) + b, err := tx.CreateBucket([]byte("woojits")) assert.NoError(t, err) - b := tx.Bucket([]byte("woojits")) for i := 0; i < 500; i++ { b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - b.Put([]byte("really-big-value"), []byte(strings.Repeat("*", 10000))) + b.Put(big_key, []byte(strings.Repeat("*", 10000))) return nil }) @@ -583,7 +583,9 @@ func TestBucket_Stats(t *testing.T) { assert.Equal(t, 501, stats.KeyN, "KeyN") assert.Equal(t, 2, stats.Depth, "Depth") assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 21004, stats.LeafInuse, "LeafInuse") + used := pageHeaderSize + 501*leafPageElementSize + used += 10*2 + 90*4 + 400*6 + len(big_key) + 10000 + assert.Equal(t, used, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") @@ -628,7 +630,7 @@ func TestBucket_Stats_Small(t *testing.T) { } assert.Equal(t, 1, stats.BucketN, "BucketN") assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") - assert.Equal(t, bucketHeaderSize+pageHeaderSize+leafPageElementSize+6, stats.InlineBucketInuse, "InlineBucketInuse") + assert.Equal(t, pageHeaderSize+leafPageElementSize+6, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -701,7 +703,10 @@ func TestBucket_Stats_Nested(t *testing.T) { assert.Equal(t, 122, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2506, stats.LeafInuse, "LeafInuse") + baz := pageHeaderSize + 10*leafPageElementSize + 10*2 + foo := pageHeaderSize + 101*leafPageElementSize + 10*2 + 90*4 + 3 + bucketHeaderSize + bar := pageHeaderSize + 11*leafPageElementSize + 10*2 + 3 + bucketHeaderSize + baz + assert.Equal(t, foo+bar, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") @@ -709,7 +714,7 @@ func TestBucket_Stats_Nested(t *testing.T) { } assert.Equal(t, 3, stats.BucketN, "BucketN") assert.Equal(t, 1, stats.InlineBucketN, "InlineBucketN") - assert.Equal(t, 212, stats.InlineBucketInuse, "InlineBucketInuse") + assert.Equal(t, baz, stats.InlineBucketInuse, "InlineBucketInuse") return nil }) }) @@ -746,7 +751,7 @@ func TestBucket_Stats_Large(t *testing.T) { assert.Equal(t, 100000, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 27007, stats.BranchInuse, "BranchInuse") - assert.Equal(t, 2619092, stats.LeafInuse, "LeafInuse") + assert.Equal(t, 2598436, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 77824, stats.BranchAlloc, "BranchAlloc") -- cgit v1.2.3 From e728eb90da02e301aecbcabff96e691efcfe019c Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 14 May 2014 12:06:31 -0600 Subject: Minor stats fixes. --- bucket.go | 10 +++++++++- bucket_test.go | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 12 deletions(-) (limited to 'bucket_test.go') diff --git a/bucket.go b/bucket.go index 96218f1..877c310 100644 --- a/bucket.go +++ b/bucket.go @@ -371,11 +371,14 @@ func (b *Bucket) Stats() BucketStats { b.forEachPage(func(p *page, depth int) { if (p.flags & leafPageFlag) != 0 { s.KeyN += int(p.count) + // used totals the used bytes for the page used := pageHeaderSize + if p.count != 0 { // If page has any elements, add all element headers. used += leafPageElementSize * int(p.count-1) + // Add all element key, value sizes. // The computation takes advantage of the fact that the position // of the last element's key/value equals to the total of the sizes @@ -384,6 +387,7 @@ func (b *Bucket) Stats() BucketStats { lastElement := p.leafPageElement(p.count - 1) used += int(lastElement.pos + lastElement.ksize + lastElement.vsize) } + if b.root == 0 { // For inlined bucket just update the inline stats s.InlineBucketInuse += used @@ -408,9 +412,11 @@ func (b *Bucket) Stats() BucketStats { } else if (p.flags & branchPageFlag) != 0 { s.BranchPageN++ lastElement := p.branchPageElement(p.count - 1) + // used totals the used bytes for the page // Add header and all element headers. used := pageHeaderSize + (branchPageElementSize * int(p.count-1)) + // Add size of all keys and values. // Again, use the fact that last element's position equals to // the total of key, value sizes of all previous elements. @@ -418,11 +424,13 @@ func (b *Bucket) Stats() BucketStats { s.BranchInuse += used s.BranchOverflowN += int(p.overflow) } + // Keep track of maximum page depth. if depth+1 > s.Depth { s.Depth = (depth + 1) } }) + // Alloc stats can be computed from page counts and pageSize. s.BranchAlloc = (s.BranchPageN + s.BranchOverflowN) * pageSize s.LeafAlloc = (s.LeafPageN + s.LeafOverflowN) * pageSize @@ -568,7 +576,7 @@ func (b *Bucket) maxInlineBucketSize() int { func (b *Bucket) write() []byte { // Allocate the appropriate size. var n = b.rootNode - var value = make([]byte, bucketHeaderSize+pageHeaderSize+n.size()) + var value = make([]byte, bucketHeaderSize+n.size()) // Write a bucket header. var bucket = (*bucket)(unsafe.Pointer(&value[0])) diff --git a/bucket_test.go b/bucket_test.go index 0be02e5..f6bd13f 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -566,7 +566,7 @@ func TestBucket_Stats(t *testing.T) { b, err := tx.CreateBucket([]byte("woojits")) assert.NoError(t, err) for i := 0; i < 500; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + b.Put([]byte(fmt.Sprintf("%03d", i)), []byte(strconv.Itoa(i))) } b.Put(big_key, []byte(strings.Repeat("*", 10000))) @@ -582,15 +582,24 @@ func TestBucket_Stats(t *testing.T) { assert.Equal(t, 2, stats.LeafOverflowN, "LeafOverflowN") assert.Equal(t, 501, stats.KeyN, "KeyN") assert.Equal(t, 2, stats.Depth, "Depth") - assert.Equal(t, 125, stats.BranchInuse, "BranchInuse") - used := pageHeaderSize + 501*leafPageElementSize - used += 10*2 + 90*4 + 400*6 + len(big_key) + 10000 - assert.Equal(t, used, stats.LeafInuse, "LeafInuse") + + branchInuse := pageHeaderSize // branch page header + branchInuse += 6 * branchPageElementSize // branch elements + branchInuse += 6 * 3 // branch keys (6 3-byte keys) + assert.Equal(t, branchInuse, stats.BranchInuse, "BranchInuse") + + leafInuse := 6 * pageHeaderSize // leaf page header + leafInuse += 501 * leafPageElementSize // leaf elements + leafInuse += 500*3 + len(big_key) // leaf keys + leafInuse += 1*10 + 2*90 + 3*400 + 10000 // leaf values + assert.Equal(t, leafInuse, stats.LeafInuse, "LeafInuse") + if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 4096, stats.BranchAlloc, "BranchAlloc") assert.Equal(t, 32768, stats.LeafAlloc, "LeafAlloc") } + assert.Equal(t, 1, stats.BucketN, "BucketN") assert.Equal(t, 0, stats.InlineBucketN, "InlineBucketN") assert.Equal(t, 0, stats.InlineBucketInuse, "InlineBucketInuse") @@ -672,13 +681,12 @@ func TestBucket_Stats_EmptyBucket(t *testing.T) { // Ensure a bucket can calculate stats. func TestBucket_Stats_Nested(t *testing.T) { - withOpenDB(func(db *DB, path string) { db.Update(func(tx *Tx) error { b, err := tx.CreateBucket([]byte("foo")) assert.NoError(t, err) for i := 0; i < 100; i++ { - b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) + b.Put([]byte(fmt.Sprintf("%02d", i)), []byte(fmt.Sprintf("%02d", i))) } bar, err := b.CreateBucket([]byte("bar")) assert.NoError(t, err) @@ -692,7 +700,9 @@ func TestBucket_Stats_Nested(t *testing.T) { } return nil }) + mustCheck(db) + db.View(func(tx *Tx) error { b := tx.Bucket([]byte("foo")) stats := b.Stats() @@ -703,10 +713,22 @@ func TestBucket_Stats_Nested(t *testing.T) { assert.Equal(t, 122, stats.KeyN, "KeyN") assert.Equal(t, 3, stats.Depth, "Depth") assert.Equal(t, 0, stats.BranchInuse, "BranchInuse") - baz := pageHeaderSize + 10*leafPageElementSize + 10*2 - foo := pageHeaderSize + 101*leafPageElementSize + 10*2 + 90*4 + 3 + bucketHeaderSize - bar := pageHeaderSize + 11*leafPageElementSize + 10*2 + 3 + bucketHeaderSize + baz - assert.Equal(t, foo+bar, stats.LeafInuse, "LeafInuse") + + foo := pageHeaderSize // foo + foo += 101 * leafPageElementSize // foo leaf elements + foo += 100*2 + 100*2 // foo leaf key/values + foo += 3 + bucketHeaderSize // foo -> bar key/value + + bar := pageHeaderSize // bar + bar += 11 * leafPageElementSize // bar leaf elements + bar += 10 + 10 // bar leaf key/values + bar += 3 + bucketHeaderSize // bar -> baz key/value + + baz := pageHeaderSize // baz (inline) + baz += 10 * leafPageElementSize // baz leaf elements + baz += 10 + 10 // baz leaf key/values + + assert.Equal(t, foo+bar+baz, stats.LeafInuse, "LeafInuse") if os.Getpagesize() == 4096 { // Incompatible page size assert.Equal(t, 0, stats.BranchAlloc, "BranchAlloc") -- cgit v1.2.3