From 834b38e3e742f99b682311f61afc223407eca757 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 28 Jan 2015 11:29:27 -0500 Subject: Fix mmap resize calculation. This commit fixes an issue where the database would grow whenever it was opened. This was caused by a recent change that performed a truncation when the database grew. Now there are fixed growth sizes for the database (1MB, 2MB, 4MB, 8MB, etc) up to 1GB and then the database will grow by 1GB when it resizes. See also: 6bb25854a183f3d3bfa50096f910d3a3984e9834 --- db.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'db.go') diff --git a/db.go b/db.go index f3fe278..535e566 100644 --- a/db.go +++ b/db.go @@ -12,9 +12,6 @@ import ( "unsafe" ) -// The smallest size that the mmap can be. -const minMmapSize = 1 << 22 // 4MB - // The largest step that can be taken when remapping the mmap. const maxMmapStep = 1 << 30 // 1GB @@ -222,15 +219,21 @@ func (db *DB) munmap() error { // mmapSize determines the appropriate size for the mmap given the current size // of the database. The minimum size is 4MB and doubles until it reaches 1GB. func (db *DB) mmapSize(size int) int { - if size <= minMmapSize { - return minMmapSize - } else if size < maxMmapStep { - size *= 2 - } else { - size += maxMmapStep + // Double the size from 1MB until 1GB. + for i := uint(20); i < 30; i++ { + if size <= 1< 0 { + size -= remainder } // Ensure that the mmap size is a multiple of the page size. + // This should always be true since we're incrementing in MBs. if (size % db.pageSize) != 0 { size = ((size / db.pageSize) + 1) * db.pageSize } -- cgit v1.2.3 From dacc1873d170e88019c30f9dab1a364b45cc7516 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 28 Jan 2015 13:03:30 -0500 Subject: Fix mmap step & max size check. This commit adds fixes suggested by @tv42 for the mmap step fix in 834b38e: * Check max size before calculating the new the mmap size. * Fix mmap step loop to go to 1GB instead of 512MB. --- db.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'db.go') diff --git a/db.go b/db.go index 535e566..3198bca 100644 --- a/db.go +++ b/db.go @@ -171,11 +171,9 @@ func (db *DB) mmap(minsz int) error { if size < minsz { size = minsz } - size = db.mmapSize(size) - - // Verify the map size is not above the maximum allowed. - if size > maxMapSize { - return fmt.Errorf("mmap too large") + size, err = db.mmapSize(size) + if err != nil { + return err } // Dereference all mmap references before unmapping. @@ -218,14 +216,20 @@ func (db *DB) munmap() error { // mmapSize determines the appropriate size for the mmap given the current size // of the database. The minimum size is 4MB and doubles until it reaches 1GB. -func (db *DB) mmapSize(size int) int { +// Returns an error if the new mmap size is greater than the max allowed. +func (db *DB) mmapSize(size int) (int, error) { // Double the size from 1MB until 1GB. - for i := uint(20); i < 30; i++ { + for i := uint(20); i <= 30; i++ { if size <= 1< maxMapSize-maxMmapStep { + return 0, fmt.Errorf("mmap too large") + } + // If larger than 1GB then grow by 1GB at a time. size += maxMmapStep if remainder := size % maxMmapStep; remainder > 0 { @@ -238,7 +242,7 @@ func (db *DB) mmapSize(size int) int { size = ((size / db.pageSize) + 1) * db.pageSize } - return size + return size, nil } // init creates a new database file and initializes its meta pages. -- cgit v1.2.3 From 338d8e78e27972269c81f8c11bca5bfc281c84ae Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 28 Jan 2015 16:27:06 -0500 Subject: Fix max mmap check for 32-bit arch. --- db.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'db.go') diff --git a/db.go b/db.go index 3198bca..64de16e 100644 --- a/db.go +++ b/db.go @@ -225,24 +225,30 @@ func (db *DB) mmapSize(size int) (int, error) { } } - // Verify the map size is not above the maximum allowed. - if size > maxMapSize-maxMmapStep { + // Verify the requested size is not above the maximum allowed. + if size > maxMapSize { return 0, fmt.Errorf("mmap too large") } // If larger than 1GB then grow by 1GB at a time. - size += maxMmapStep - if remainder := size % maxMmapStep; remainder > 0 { - size -= remainder + sz := int64(size) + int64(maxMmapStep) + if remainder := sz % int64(maxMmapStep); remainder > 0 { + sz -= remainder } // Ensure that the mmap size is a multiple of the page size. // This should always be true since we're incrementing in MBs. - if (size % db.pageSize) != 0 { - size = ((size / db.pageSize) + 1) * db.pageSize + pageSize := int64(db.pageSize) + if (sz % pageSize) != 0 { + sz = ((sz / pageSize) + 1) * pageSize } - return size, nil + // If we've exceeded the max size then only grow up to the max size. + if sz > maxMapSize { + sz = maxMapSize + } + + return int(sz), nil } // init creates a new database file and initializes its meta pages. -- cgit v1.2.3