Use mremap on Linux instead of munmap+mmap

z/file.go

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package z
 
 import (
@@ -152,24 +168,6 @@ func (m *MmapFile) Delete() error {
 	return os.Remove(m.Fd.Name())
 }
 
-// Truncate would truncate the mmapped file to the given size. On Linux and
-// others, we could directly just truncate the underlying file, but in Windows,
-// we can't do that. So, unmap first, then truncate, then re-map.
-func (m *MmapFile) Truncate(maxSz int64) error {
-	if err := m.Sync(); err != nil {
-		return fmt.Errorf("while sync file: %s, error: %v\n", m.Fd.Name(), err)
-	}
-	if err := Munmap(m.Data); err != nil {
-		return fmt.Errorf("while munmap file: %s, error: %v\n", m.Fd.Name(), err)
-	}
-	if err := m.Fd.Truncate(maxSz); err != nil {
-		return fmt.Errorf("while truncate file: %s, error: %v\n", m.Fd.Name(), err)
-	}
-	var err error
-	m.Data, err = Mmap(m.Fd, true, maxSz) // Mmap up to max size.
-	return err
-}
-
 // Close would close the file. It would also truncate the file if maxSz >= 0.
 func (m *MmapFile) Close(maxSz int64) error {
 	// Badger can set the m.Data directly, without setting any Fd. In that case, this should be a

z/file_default.go

@@ -0,0 +1,39 @@
+// +build !linux
+
+/*
+ * Copyright 2020 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package z
+
+import "fmt"
+
+// Truncate would truncate the mmapped file to the given size. On Linux, we truncate
+// the underlying file and then call mremap, but on other systems, we unmap first,
+// then truncate, then re-map.
+func (m *MmapFile) Truncate(maxSz int64) error {
+	if err := m.Sync(); err != nil {
+		return fmt.Errorf("while sync file: %s, error: %v\n", m.Fd.Name(), err)
+	}
+	if err := Munmap(m.Data); err != nil {
+		return fmt.Errorf("while munmap file: %s, error: %v\n", m.Fd.Name(), err)
+	}
+	if err := m.Fd.Truncate(maxSz); err != nil {
+		return fmt.Errorf("while truncate file: %s, error: %v\n", m.Fd.Name(), err)
+	}
+	var err error
+	m.Data, err = Mmap(m.Fd, true, maxSz) // Mmap up to max size.
+	return err
+}

z/file_linux.go

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package z
+
+import (
+	"fmt"
+)
+
+// Truncate would truncate the mmapped file to the given size. On Linux, we truncate
+// the underlying file and then call mremap, but on other systems, we unmap first,
+// then truncate, then re-map.
+func (m *MmapFile) Truncate(maxSz int64) error {
+	if err := m.Sync(); err != nil {
+		return fmt.Errorf("while sync file: %s, error: %v\n", m.Fd.Name(), err)
+	}
+	if err := m.Fd.Truncate(maxSz); err != nil {
+		return fmt.Errorf("while truncate file: %s, error: %v\n", m.Fd.Name(), err)
+	}
+
+	var err error
+	m.Data, err = mremap(m.Data, int(maxSz)) // Mmap up to max size.
+	return err
+}

z/mmap_linux.go

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2020 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package z
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// mmap uses the mmap system call to memory-map a file. If writable is true,
+// memory protection of the pages is set so that they may be written to as well.
+func mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
+	mtype := unix.PROT_READ
+	if writable {
+		mtype |= unix.PROT_WRITE
+	}
+	return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
+}
+
+// mremap is a Linux-specific system call to remap pages in memory. This can be used in place of munmap + mmap.
+func mremap(data []byte, size int) ([]byte, error) {
+	// taken from <https://github.com/torvalds/linux/blob/f8394f232b1eab649ce2df5c5f15b0e528c92091/include/uapi/linux/mman.h#L8>
+	const MREMAP_MAYMOVE = 0x1
+
+	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
+	mmapAddr, mmapSize, errno := unix.Syscall6(
+		unix.SYS_MREMAP,
+		header.Data,
+		uintptr(header.Len),
+		uintptr(size),
+		uintptr(MREMAP_MAYMOVE),
+		0,
+		0,
+	)
+	if errno != 0 {
+		return nil, errno
+	}
+	if mmapSize != uintptr(size) {
+		return nil, fmt.Errorf("mremap size mismatch: requested: %d got: %d", size, mmapSize)
+	}
+
+	header.Data = mmapAddr
+	header.Cap = size
+	header.Len = size
+	return data, nil
+}
+
+// munmap unmaps a previously mapped slice.
+//
+// unix.Munmap maintains an internal list of mmapped addresses, and only calls munmap
+// if the address is present in that list. If we use mremap, this list is not updated.
+// To bypass this, we call munmap ourselves.
+func munmap(data []byte) error {
+	if len(data) == 0 || len(data) != cap(data) {
+		return unix.EINVAL
+	}
+	_, _, errno := unix.Syscall(
+		unix.SYS_MUNMAP,
+		uintptr(unsafe.Pointer(&data[0])),
+		uintptr(len(data)),
+		0,
+	)
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+// madvise uses the madvise system call to give advise about the use of memory
+// when using a slice that is memory-mapped to a file. Set the readahead flag to
+// false if page references are expected in random order.
+func madvise(b []byte, readahead bool) error {
+	flags := unix.MADV_NORMAL
+	if !readahead {
+		flags = unix.MADV_RANDOM
+	}
+	return unix.Madvise(b, flags)
+}
+
+// msync writes any modified data to persistent storage.
+func msync(b []byte) error {
+	return unix.Msync(b, unix.MS_SYNC)
+}

z/mmap_unix.go

@@ -1,4 +1,4 @@
-// +build !windows,!darwin,!plan9
+// +build !windows,!darwin,!plan9,!linux
 
 /*
  * Copyright 2019 Dgraph Labs, Inc. and Contributors