diff options
author | sasha-s <sasha@scaledinference.com> | 2015-03-26 16:47:24 -0700 |
---|---|---|
committer | sasha-s <sasha@scaledinference.com> | 2015-04-16 11:58:02 -0700 |
commit | bdc109bdc7dcb4ec98de9605c24fbd9afef5f3bc (patch) | |
tree | 38d96eae84b1726d713debd9725dcf2b27dc398b | |
parent | Add --path to bolt bench. (diff) | |
download | dedo-bdc109bdc7dcb4ec98de9605c24fbd9afef5f3bc.tar.gz dedo-bdc109bdc7dcb4ec98de9605c24fbd9afef5f3bc.tar.xz |
fix `slice bounds out of range`/maxAllocSize bugs
when accessing the node data we used to use cast to
*[maxAllocSize]byte, which breaks if we try to go across maxAllocSize boundary.
This leads to occasional panics.
Sample stacktrace:
```
panic: runtime error: slice bounds out of range
goroutine 1 [running]:
github.com/boltdb/bolt.(*node).write(0xc208010f50, 0xc27452a000)
$GOPATH/src/github.com/boltdb/bolt/node.go:228 +0x5a5
github.com/boltdb/bolt.(*node).spill(0xc208010f50, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/node.go:364 +0x506
github.com/boltdb/bolt.(*node).spill(0xc208010700, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/node.go:336 +0x12d
github.com/boltdb/bolt.(*node).spill(0xc208010620, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/node.go:336 +0x12d
github.com/boltdb/bolt.(*Bucket).spill(0xc22b6ae880, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/bucket.go:535 +0x1c4
github.com/boltdb/bolt.(*Bucket).spill(0xc22b6ae840, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/bucket.go:502 +0xac2
github.com/boltdb/bolt.(*Bucket).spill(0xc22f4e2018, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/bucket.go:502 +0xac2
github.com/boltdb/bolt.(*Tx).Commit(0xc22f4e2000, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/tx.go:150 +0x1ee
github.com/boltdb/bolt.(*DB).Update(0xc2080e4000, 0xc24d077508, 0x0, 0x0)
$GOPATH/src/github.com/boltdb/bolt/db.go:483 +0x169
```
It usually happens when working with large (50M/100M) values.
One way to reproduce it is to change maxAllocSize in bolt_amd64.go to 70000 and run the tests.
TestBucket_Put_Large crashes.
-rw-r--r-- | bolt_amd64.go | 4 | ||||
-rw-r--r-- | node.go | 12 | ||||
-rw-r--r-- | page.go | 6 | ||||
-rw-r--r-- | tx.go | 24 |
4 files changed, 34 insertions, 12 deletions
diff --git a/bolt_amd64.go b/bolt_amd64.go index cca6b7e..83f935f 100644 --- a/bolt_amd64.go +++ b/bolt_amd64.go @@ -5,3 +5,7 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Setting +// const maxAllocSize = 70000 +// reveals the index out of bound bug(s) @@ -220,12 +220,16 @@ func (n *node) write(p *page) { elem.pgid = item.pgid _assert(elem.pgid != p.id, "write: circular dependency occurred") } + lk, lv := len(item.key), len(item.value) + if len(b) < lk+lv { + b = (*[maxAllocSize]byte)(unsafe.Pointer(&b[0]))[:] + } // Write data for the element to the end of the page. copy(b[0:], item.key) - b = b[len(item.key):] + b = b[lk:] copy(b[0:], item.value) - b = b[len(item.value):] + b = b[lv:] } // DEBUG ONLY: n.dump() @@ -351,7 +355,9 @@ func (n *node) spill() error { } // Allocate contiguous space for the node. - p, err := tx.allocate((node.size() / tx.db.pageSize) + 1) + // sz := node.size() + n.pageElementSize()*len(n.inodes) + sz := node.size() + p, err := tx.allocate((sz / tx.db.pageSize) + 1) if err != nil { return err } @@ -96,7 +96,7 @@ type branchPageElement struct { // key returns a byte slice of the node key. func (n *branchPageElement) key() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return buf[n.pos : n.pos+n.ksize] + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize] } // leafPageElement represents a node on a leaf page. @@ -110,13 +110,13 @@ type leafPageElement struct { // key returns a byte slice of the node key. func (n *leafPageElement) key() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return buf[n.pos : n.pos+n.ksize] + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize] } // value returns a byte slice of the node value. func (n *leafPageElement) value() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return buf[n.pos+n.ksize : n.pos+n.ksize+n.vsize] + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize] } // PageInfo represents human readable information about a page. @@ -421,14 +421,26 @@ func (tx *Tx) write() error { // Write pages to disk in order. for _, p := range pages { size := (int(p.overflow) + 1) * tx.db.pageSize - buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:size] offset := int64(p.id) * int64(tx.db.pageSize) - if _, err := tx.db.ops.writeAt(buf, offset); err != nil { - return err + ptr := (*[maxAllocSize]byte)(unsafe.Pointer(p)) + for { + sz := size + if sz > maxAllocSize-1 { + sz = maxAllocSize - 1 + } + buf := ptr[:sz] + if _, err := tx.db.ops.writeAt(buf, offset); err != nil { + return err + } + // Update statistics. + tx.stats.Write++ + size -= sz + if size == 0 { + break + } + offset += int64(sz) + ptr = (*[maxAllocSize]byte)(unsafe.Pointer(&ptr[sz])) } - - // Update statistics. - tx.stats.Write++ } if !tx.db.NoSync || IgnoreNoSync { if err := fdatasync(tx.db); err != nil { |