From e67705ed6348675b7bae405ebeb37bb69b53a96d Mon Sep 17 00:00:00 2001
From: Xiang Li <xiangli.cs@gmail.com>
Date: Wed, 4 Nov 2015 15:12:18 -0800
Subject: do not grow dbsize agressively

Only grow the database size when the high watermark increases.
We also grows the database size a little bit aggressively to
save a few ftruncates.

I have tested this on various environments. The performance impact
is ignorable with 16MB over allocation. Without over allocation,
the performance might decrease 100% when each Tx.Commit needs a new
page on a very slow disk (seek time dominates the total write).
---
 db.go | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

(limited to 'db.go')

diff --git a/db.go b/db.go
index d39c4aa..fd920a0 100644
--- a/db.go
+++ b/db.go
@@ -84,6 +84,7 @@ type DB struct {
 	dataref  []byte // mmap'ed readonly, write throws SEGV
 	data     *[maxMapSize]byte
 	datasz   int
+	filesz   int // current on disk file size
 	meta0    *meta
 	meta1    *meta
 	pageSize int
@@ -655,6 +656,37 @@ func (db *DB) allocate(count int) (*page, error) {
 	return p, nil
 }
 
+// growSize grows the size of the database to the given sz.
+func (db *DB) growSize(sz int) error {
+	if sz <= db.filesz {
+		return nil
+	}
+
+	// over allocate 16MB to avoid calling Truncate aggressively
+	// for efficiency
+	overAllocation := 16 * 1024 * 1024
+	sz = sz + overAllocation
+
+	// do not over allocate
+	if sz > db.datasz {
+		sz = db.datasz
+	}
+
+	// Truncate and fsync to ensure file size metadata is flushed.
+	// https://github.com/boltdb/bolt/issues/284
+	if !db.NoGrowSync && !db.readOnly {
+		if err := db.file.Truncate(int64(sz)); err != nil {
+			return fmt.Errorf("file resize error: %s", err)
+		}
+		if err := db.file.Sync(); err != nil {
+			return fmt.Errorf("file sync error: %s", err)
+		}
+	}
+
+	db.filesz = sz
+	return nil
+}
+
 func (db *DB) IsReadOnly() bool {
 	return db.readOnly
 }
-- 
cgit v1.2.3


From a122e1c02bae30d40559b4daf028e579fcc2017c Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Mon, 11 Jan 2016 15:40:23 -0700
Subject: add AllocSize, minor grow() refactor

This commit moves `overAllocation` to a configurable `DB.AllocSize`
field and performs minor cosmetic clean up.
---
 db.go | 24 +++++++++++++++---------
 tx.go |  6 +++++-
 2 files changed, 20 insertions(+), 10 deletions(-)

(limited to 'db.go')

diff --git a/db.go b/db.go
index a4b8efb..911055d 100644
--- a/db.go
+++ b/db.go
@@ -33,6 +33,7 @@ const IgnoreNoSync = runtime.GOOS == "openbsd"
 const (
 	DefaultMaxBatchSize  int = 1000
 	DefaultMaxBatchDelay     = 10 * time.Millisecond
+	DefaultAllocSize         = 16 * 1024 * 1024
 )
 
 // DB represents a collection of buckets persisted to a file on disk.
@@ -85,6 +86,11 @@ type DB struct {
 	// Do not change concurrently with calls to Batch.
 	MaxBatchDelay time.Duration
 
+	// AllocSize is the amount of space allocated when the database
+	// needs to create new pages. This is done to amortize the cost
+	// of truncate() and fsync() when growing the data file.
+	AllocSize int
+
 	path     string
 	file     *os.File
 	dataref  []byte // mmap'ed readonly, write throws SEGV
@@ -148,6 +154,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	// Set default values for later DB operations.
 	db.MaxBatchSize = DefaultMaxBatchSize
 	db.MaxBatchDelay = DefaultMaxBatchDelay
+	db.AllocSize = DefaultAllocSize
 
 	flag := os.O_RDWR
 	if options.ReadOnly {
@@ -799,20 +806,19 @@ func (db *DB) allocate(count int) (*page, error) {
 	return p, nil
 }
 
-// growSize grows the size of the database to the given sz.
-func (db *DB) growSize(sz int) error {
+// grow grows the size of the database to the given sz.
+func (db *DB) grow(sz int) error {
+	// Ignore if the new size is less than available file size.
 	if sz <= db.filesz {
 		return nil
 	}
 
-	// over allocate 16MB to avoid calling Truncate aggressively
-	// for efficiency
-	overAllocation := 16 * 1024 * 1024
-	sz = sz + overAllocation
-
-	// do not over allocate
-	if sz > db.datasz {
+	// If the data is smaller than the alloc size then only allocate what's needed.
+	// Once it goes over the allocation size then allocate in chunks.
+	if db.datasz < db.AllocSize {
 		sz = db.datasz
+	} else {
+		sz += db.AllocSize
 	}
 
 	// Truncate and fsync to ensure file size metadata is flushed.
diff --git a/tx.go b/tx.go
index 395dce3..52b68ef 100644
--- a/tx.go
+++ b/tx.go
@@ -184,8 +184,12 @@ func (tx *Tx) Commit() error {
 	}
 	tx.meta.freelist = p.id
 
+	// If the high water mark has moved up then attempt to grow the database.
 	if tx.meta.pgid > opgid {
-		tx.db.growSize(int(tx.meta.pgid+1) * tx.db.pageSize)
+		if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
+			tx.rollback()
+			return err
+		}
 	}
 
 	// Write dirty pages to disk.
-- 
cgit v1.2.3