diff options
-rw-r--r-- | README.md | 76 | ||||
-rw-r--r-- | bolt_unix.go | 11 | ||||
-rw-r--r-- | bolt_unix_solaris.go | 11 | ||||
-rw-r--r-- | bucket_test.go | 112 | ||||
-rw-r--r-- | db.go | 42 | ||||
-rw-r--r-- | db_test.go | 16 | ||||
-rw-r--r-- | tx.go | 12 | ||||
-rw-r--r-- | tx_test.go | 4 |
8 files changed, 196 insertions, 88 deletions
@@ -42,6 +42,7 @@ services every day. - [Database backups](#database-backups) - [Statistics](#statistics) - [Read-Only Mode](#read-only-mode) + - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) - [Resources](#resources) - [Comparison with other databases](#comparison-with-other-databases) - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) @@ -550,6 +551,81 @@ if err != nil { } ``` +### Mobile Use (iOS/Android) + +Bolt is able to run on mobile devices by leveraging the binding feature of the +[gomobile](https://github.com/golang/mobile) tool. Create a struct that will +contain your database logic and a reference to a *bolt.DB with a initializing +contstructor that takes in a filepath where the database file will be stored. +Neither Android nor iOS require extra permissions or cleanup from using this method. + +```go +func NewBoltDB(filepath string) *BoltDB { + db, err := bolt.Open(filepath+"/demo.db", 0600, nil) + if err != nil { + log.Fatal(err) + } + + return &BoltDB{db} +} + +type BoltDB struct { + db *bolt.DB + ... +} + +func (b *BoltDB) Path() string { + return b.db.Path() +} + +func (b *BoltDB) Close() { + b.db.Close() +} +``` + +Database logic should be defined as methods on this wrapper struct. + +To initialize this struct from the native language (both platforms now sync +their local storage to the cloud. These snippets disable that functionality for the +database file): +####Android +```java +String path; +if (android.os.Build.VERSION.SDK_INT >=android.os.Build.VERSION_CODES.LOLLIPOP){ + path = getNoBackupFilesDir().getAbsolutePath(); +} else{ + path = getFilesDir().getAbsolutePath(); +} +Boltmobiledemo.BoltDB boltDB = Boltmobiledemo.NewBoltDB(path) +``` + +####iOS +```objc +- (void)demo { + NSString* path = [NSSearchPathForDirectoriesInDomains(NSLibraryDirectory, + NSUserDomainMask, + YES) objectAtIndex:0]; + GoBoltmobiledemoBoltDB * demo = GoBoltmobiledemoNewBoltDB(path); + [self addSkipBackupAttributeToItemAtPath:demo.path]; + //Some DB Logic would go here + [demo close]; +} + +- (BOOL)addSkipBackupAttributeToItemAtPath:(NSString *) filePathString +{ + NSURL* URL= [NSURL fileURLWithPath: filePathString]; + assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]); + + NSError *error = nil; + BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES] + forKey: NSURLIsExcludedFromBackupKey error: &error]; + if(!success){ + NSLog(@"Error excluding %@ from backup %@", [URL lastPathComponent], error); + } + return success; +} + +``` ## Resources diff --git a/bolt_unix.go b/bolt_unix.go index 251680b..4b0723a 100644 --- a/bolt_unix.go +++ b/bolt_unix.go @@ -46,17 +46,6 @@ func funlock(f *os.File) error { // mmap memory maps a DB's data file. func mmap(db *DB, sz int) error { - // Truncate and fsync to ensure file size metadata is flushed. - // https://github.com/boltdb/bolt/issues/284 - if !db.NoGrowSync && !db.readOnly { - if err := db.file.Truncate(int64(sz)); err != nil { - return fmt.Errorf("file resize error: %s", err) - } - if err := db.file.Sync(); err != nil { - return fmt.Errorf("file sync error: %s", err) - } - } - // Map the data file to memory. b, err := syscall.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) if err != nil { diff --git a/bolt_unix_solaris.go b/bolt_unix_solaris.go index 214009b..1c4e48d 100644 --- a/bolt_unix_solaris.go +++ b/bolt_unix_solaris.go @@ -56,17 +56,6 @@ func funlock(f *os.File) error { // mmap memory maps a DB's data file. func mmap(db *DB, sz int) error { - // Truncate and fsync to ensure file size metadata is flushed. - // https://github.com/boltdb/bolt/issues/284 - if !db.NoGrowSync && !db.readOnly { - if err := db.file.Truncate(int64(sz)); err != nil { - return fmt.Errorf("file resize error: %s", err) - } - if err := db.file.Sync(); err != nil { - return fmt.Errorf("file sync error: %s", err) - } - } - // Map the data file to memory. b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) if err != nil { diff --git a/bucket_test.go b/bucket_test.go index 480ed08..a02c367 100644 --- a/bucket_test.go +++ b/bucket_test.go @@ -456,10 +456,10 @@ func TestBucket_Nested(t *testing.T) { if err := db.View(func(tx *bolt.Tx) error { var b = tx.Bucket([]byte("widgets")) if v := b.Bucket([]byte("foo")).Get([]byte("baz")); !bytes.Equal(v, []byte("yyyy")) { - t.Fatalf("unexpected value: %v") + t.Fatalf("unexpected value: %v", v) } - if !bytes.Equal(b.Get([]byte("bar")), []byte("xxxx")) { - t.Fatalf("unexpected value: %v") + if v := b.Get([]byte("bar")); !bytes.Equal(v, []byte("xxxx")) { + t.Fatalf("unexpected value: %v", v) } for i := 0; i < 10000; i++ { if v := b.Get([]byte(strconv.Itoa(i))); !bytes.Equal(v, []byte(strconv.Itoa(i))) { @@ -1118,18 +1118,18 @@ func TestBucket_Stats(t *testing.T) { // Only check allocations for 4KB pages. if os.Getpagesize() == 4096 { if stats.BranchAlloc != 4096 { - t.Fatalf("unexpected BranchAlloc:", stats.BranchAlloc) + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) } else if stats.LeafAlloc != 36864 { - t.Fatalf("unexpected LeafAlloc:", stats.LeafAlloc) + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) } } if stats.BucketN != 1 { - t.Fatalf("unexpected BucketN:", stats.BucketN) + t.Fatalf("unexpected BucketN: %d", stats.BucketN) } else if stats.InlineBucketN != 0 { - t.Fatalf("unexpected InlineBucketN:", stats.InlineBucketN) + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) } else if stats.InlineBucketInuse != 0 { - t.Fatalf("unexpected InlineBucketInuse:", stats.InlineBucketInuse) + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) } return nil @@ -1178,27 +1178,27 @@ func TestBucket_Stats_RandomFill(t *testing.T) { if err := db.View(func(tx *bolt.Tx) error { stats := tx.Bucket([]byte("woojits")).Stats() if stats.KeyN != 100000 { - t.Fatalf("unexpected KeyN", stats.KeyN) + t.Fatalf("unexpected KeyN: %d", stats.KeyN) } if stats.BranchPageN != 98 { - t.Fatalf("unexpected BranchPageN", stats.BranchPageN) + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN", stats.BranchOverflowN) + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) } else if stats.BranchInuse != 130984 { - t.Fatalf("unexpected BranchInuse", stats.BranchInuse) + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) } else if stats.BranchAlloc != 401408 { - t.Fatalf("unexpected BranchAlloc", stats.BranchAlloc) + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) } if stats.LeafPageN != 3412 { - t.Fatalf("unexpected LeafPageN", stats.LeafPageN) + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN", stats.LeafOverflowN) + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) } else if stats.LeafInuse != 4742482 { - t.Fatalf("unexpected LeafInuse", stats.LeafInuse) + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) } else if stats.LeafAlloc != 13975552 { - t.Fatalf("unexpected LeafAlloc", stats.LeafAlloc) + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) } return nil }); err != nil { @@ -1232,37 +1232,37 @@ func TestBucket_Stats_Small(t *testing.T) { b := tx.Bucket([]byte("whozawhats")) stats := b.Stats() if stats.BranchPageN != 0 { - t.Fatalf("unexpected BranchPageN: ", stats.BranchPageN) + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: ", stats.BranchOverflowN) + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) } else if stats.LeafPageN != 0 { - t.Fatalf("unexpected LeafPageN: ", stats.LeafPageN) + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: ", stats.LeafOverflowN) + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) } else if stats.KeyN != 1 { - t.Fatalf("unexpected KeyN: ", stats.KeyN) + t.Fatalf("unexpected KeyN: %d", stats.KeyN) } else if stats.Depth != 1 { - t.Fatalf("unexpected Depth: ", stats.Depth) + t.Fatalf("unexpected Depth: %d", stats.Depth) } else if stats.BranchInuse != 0 { - t.Fatalf("unexpected BranchInuse: ", stats.BranchInuse) + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) } else if stats.LeafInuse != 0 { - t.Fatalf("unexpected LeafInuse: ", stats.LeafInuse) + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) } if os.Getpagesize() == 4096 { if stats.BranchAlloc != 0 { - t.Fatalf("unexpected BranchAlloc: ", stats.BranchAlloc) + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) } else if stats.LeafAlloc != 0 { - t.Fatalf("unexpected LeafAlloc: ", stats.LeafAlloc) + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) } } if stats.BucketN != 1 { - t.Fatalf("unexpected BucketN: ", stats.BucketN) + t.Fatalf("unexpected BucketN: %d", stats.BucketN) } else if stats.InlineBucketN != 1 { - t.Fatalf("unexpected InlineBucketN: ", stats.InlineBucketN) + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) } else if stats.InlineBucketInuse != 16+16+6 { - t.Fatalf("unexpected InlineBucketInuse: ", stats.InlineBucketInuse) + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) } return nil @@ -1291,37 +1291,37 @@ func TestBucket_Stats_EmptyBucket(t *testing.T) { b := tx.Bucket([]byte("whozawhats")) stats := b.Stats() if stats.BranchPageN != 0 { - t.Fatalf("unexpected BranchPageN: ", stats.BranchPageN) + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: ", stats.BranchOverflowN) + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) } else if stats.LeafPageN != 0 { - t.Fatalf("unexpected LeafPageN: ", stats.LeafPageN) + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: ", stats.LeafOverflowN) + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) } else if stats.KeyN != 0 { - t.Fatalf("unexpected KeyN: ", stats.KeyN) + t.Fatalf("unexpected KeyN: %d", stats.KeyN) } else if stats.Depth != 1 { - t.Fatalf("unexpected Depth: ", stats.Depth) + t.Fatalf("unexpected Depth: %d", stats.Depth) } else if stats.BranchInuse != 0 { - t.Fatalf("unexpected BranchInuse: ", stats.BranchInuse) + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) } else if stats.LeafInuse != 0 { - t.Fatalf("unexpected LeafInuse: ", stats.LeafInuse) + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) } if os.Getpagesize() == 4096 { if stats.BranchAlloc != 0 { - t.Fatalf("unexpected BranchAlloc: ", stats.BranchAlloc) + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) } else if stats.LeafAlloc != 0 { - t.Fatalf("unexpected LeafAlloc: ", stats.LeafAlloc) + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) } } if stats.BucketN != 1 { - t.Fatalf("unexpected BucketN: ", stats.BucketN) + t.Fatalf("unexpected BucketN: %d", stats.BucketN) } else if stats.InlineBucketN != 1 { - t.Fatalf("unexpected InlineBucketN: ", stats.InlineBucketN) + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) } else if stats.InlineBucketInuse != 16 { - t.Fatalf("unexpected InlineBucketInuse: ", stats.InlineBucketInuse) + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) } return nil @@ -1377,19 +1377,19 @@ func TestBucket_Stats_Nested(t *testing.T) { b := tx.Bucket([]byte("foo")) stats := b.Stats() if stats.BranchPageN != 0 { - t.Fatalf("unexpected BranchPageN: ", stats.BranchPageN) + t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: ", stats.BranchOverflowN) + t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) } else if stats.LeafPageN != 2 { - t.Fatalf("unexpected LeafPageN: ", stats.LeafPageN) + t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: ", stats.LeafOverflowN) + t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) } else if stats.KeyN != 122 { - t.Fatalf("unexpected KeyN: ", stats.KeyN) + t.Fatalf("unexpected KeyN: %d", stats.KeyN) } else if stats.Depth != 3 { - t.Fatalf("unexpected Depth: ", stats.Depth) + t.Fatalf("unexpected Depth: %d", stats.Depth) } else if stats.BranchInuse != 0 { - t.Fatalf("unexpected BranchInuse: ", stats.BranchInuse) + t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) } foo := 16 // foo (pghdr) @@ -1407,23 +1407,23 @@ func TestBucket_Stats_Nested(t *testing.T) { baz += 10 + 10 // baz leaf key/values if stats.LeafInuse != foo+bar+baz { - t.Fatalf("unexpected LeafInuse: ", stats.LeafInuse) + t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) } if os.Getpagesize() == 4096 { if stats.BranchAlloc != 0 { - t.Fatalf("unexpected BranchAlloc: ", stats.BranchAlloc) + t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) } else if stats.LeafAlloc != 8192 { - t.Fatalf("unexpected LeafAlloc: ", stats.LeafAlloc) + t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) } } if stats.BucketN != 3 { - t.Fatalf("unexpected BucketN: ", stats.BucketN) + t.Fatalf("unexpected BucketN: %d", stats.BucketN) } else if stats.InlineBucketN != 1 { - t.Fatalf("unexpected InlineBucketN: ", stats.InlineBucketN) + t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) } else if stats.InlineBucketInuse != baz { - t.Fatalf("unexpected InlineBucketInuse: ", stats.InlineBucketInuse) + t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) } return nil @@ -26,13 +26,14 @@ const magic uint32 = 0xED0CDAED // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when // syncing changes to a file. This is required as some operating systems, // such as OpenBSD, do not have a unified buffer cache (UBC) and writes -// must be synchronzied using the msync(2) syscall. +// must be synchronized using the msync(2) syscall. const IgnoreNoSync = runtime.GOOS == "openbsd" // Default values if not set in a DB instance. const ( DefaultMaxBatchSize int = 1000 DefaultMaxBatchDelay = 10 * time.Millisecond + DefaultAllocSize = 16 * 1024 * 1024 ) // DB represents a collection of buckets persisted to a file on disk. @@ -85,11 +86,17 @@ type DB struct { // Do not change concurrently with calls to Batch. MaxBatchDelay time.Duration + // AllocSize is the amount of space allocated when the database + // needs to create new pages. This is done to amortize the cost + // of truncate() and fsync() when growing the data file. + AllocSize int + path string file *os.File dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int + filesz int // current on disk file size meta0 *meta meta1 *meta pageSize int @@ -147,6 +154,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // Set default values for later DB operations. db.MaxBatchSize = DefaultMaxBatchSize db.MaxBatchDelay = DefaultMaxBatchDelay + db.AllocSize = DefaultAllocSize flag := os.O_RDWR if options.ReadOnly { @@ -410,7 +418,7 @@ func (db *DB) close() error { // will cause the calls to block and be serialized until the current write // transaction finishes. // -// Transactions should not be depedent on one another. Opening a read +// Transactions should not be dependent on one another. Opening a read // transaction and a write transaction in the same goroutine can cause the // writer to deadlock because the database periodically needs to re-mmap itself // as it grows and it cannot do that while a read transaction is open. @@ -798,6 +806,36 @@ func (db *DB) allocate(count int) (*page, error) { return p, nil } +// grow grows the size of the database to the given sz. +func (db *DB) grow(sz int) error { + // Ignore if the new size is less than available file size. + if sz <= db.filesz { + return nil + } + + // If the data is smaller than the alloc size then only allocate what's needed. + // Once it goes over the allocation size then allocate in chunks. + if db.datasz < db.AllocSize { + sz = db.datasz + } else { + sz += db.AllocSize + } + + // Truncate and fsync to ensure file size metadata is flushed. + // https://github.com/boltdb/bolt/issues/284 + if !db.NoGrowSync && !db.readOnly { + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("file resize error: %s", err) + } + if err := db.file.Sync(); err != nil { + return fmt.Errorf("file sync error: %s", err) + } + } + + db.filesz = sz + return nil +} + func (db *DB) IsReadOnly() bool { return db.readOnly } @@ -229,6 +229,8 @@ func TestOpen_Size(t *testing.T) { path := db.Path() defer db.MustClose() + pagesize := db.Info().PageSize + // Insert until we get above the minimum 4MB size. if err := db.Update(func(tx *bolt.Tx) error { b, _ := tx.CreateBucketIfNotExists([]byte("data")) @@ -273,7 +275,8 @@ func TestOpen_Size(t *testing.T) { } // Compare the original size with the new size. - if sz != newSz { + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { t.Fatalf("unexpected file growth: %d => %d", sz, newSz) } } @@ -290,6 +293,8 @@ func TestOpen_Size_Large(t *testing.T) { path := db.Path() defer db.MustClose() + pagesize := db.Info().PageSize + // Insert until we get above the minimum 4MB size. var index uint64 for i := 0; i < 10000; i++ { @@ -338,7 +343,8 @@ func TestOpen_Size_Large(t *testing.T) { } // Compare the original size with the new size. - if sz != newSz { + // db size might increase by a few page sizes due to the new small update. + if sz < newSz-5*int64(pagesize) { t.Fatalf("unexpected file growth: %d => %d", sz, newSz) } } @@ -904,11 +910,11 @@ func TestDB_Stats(t *testing.T) { stats := db.Stats() if stats.TxStats.PageCount != 2 { - t.Fatalf("unexpected TxStats.PageCount", stats.TxStats.PageCount) + t.Fatalf("unexpected TxStats.PageCount: %d", stats.TxStats.PageCount) } else if stats.FreePageN != 0 { - t.Fatalf("unexpected FreePageN != 0", stats.FreePageN) + t.Fatalf("unexpected FreePageN != 0: %d", stats.FreePageN) } else if stats.PendingPageN != 2 { - t.Fatalf("unexpected PendingPageN != 2", stats.PendingPageN) + t.Fatalf("unexpected PendingPageN != 2: %d", stats.PendingPageN) } } @@ -168,6 +168,8 @@ func (tx *Tx) Commit() error { // Free the old root bucket. tx.meta.root.root = tx.root.root + opgid := tx.meta.pgid + // Free the freelist and allocate new pages for it. This will overestimate // the size of the freelist but not underestimate the size (which would be bad). tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) @@ -182,6 +184,14 @@ func (tx *Tx) Commit() error { } tx.meta.freelist = p.id + // If the high water mark has moved up then attempt to grow the database. + if tx.meta.pgid > opgid { + if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { + tx.rollback() + return err + } + } + // Write dirty pages to disk. startTime = time.Now() if err := tx.write(); err != nil { @@ -505,7 +515,7 @@ func (tx *Tx) writeMeta() error { } // page returns a reference to the page with a given id. -// If page has been written to then a temporary bufferred page is returned. +// If page has been written to then a temporary buffered page is returned. func (tx *Tx) page(id pgid) *page { // Check the dirty pages first. if tx.pages != nil { @@ -571,7 +571,7 @@ func TestTx_CopyFile_Error_Meta(t *testing.T) { if err := db.View(func(tx *bolt.Tx) error { return tx.Copy(&failWriter{}) }); err == nil || err.Error() != "meta copy: error injected for tests" { - t.Fatal("unexpected error: %s", err) + t.Fatalf("unexpected error: %v", err) } } @@ -598,7 +598,7 @@ func TestTx_CopyFile_Error_Normal(t *testing.T) { if err := db.View(func(tx *bolt.Tx) error { return tx.Copy(&failWriter{3 * db.Info().PageSize}) }); err == nil || err.Error() != "error injected for tests" { - t.Fatal("unexpected error: %s", err) + t.Fatalf("unexpected error: %v", err) } } |