feat: Add badgerDB support. (#353)

See https://github.com/dgraph-io/badger
Slide: https://github.com/gopherchina/conference/blob/master/2018/1.5%20Badger_%20Fast%20Key-Value%20DB%20in%20Go.pdf

vendor/github.com/dgraph-io/badger/y/error.go

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+// This file contains some functions for error handling. Note that we are moving
+// towards using x.Trace, i.e., rpc tracing using net/tracer. But for now, these
+// functions are useful for simple checks logged on one machine.
+// Some common use cases are:
+// (1) You receive an error from external lib, and would like to check/log fatal.
+//     For this, use x.Check, x.Checkf. These will check for err != nil, which is
+//     more common in Go. If you want to check for boolean being true, use
+//		   x.Assert, x.Assertf.
+// (2) You receive an error from external lib, and would like to pass on with some
+//     stack trace information. In this case, use x.Wrap or x.Wrapf.
+// (3) You want to generate a new error with stack trace info. Use x.Errorf.
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/pkg/errors"
+)
+
+var debugMode = true
+
+// Check logs fatal if err != nil.
+func Check(err error) {
+	if err != nil {
+		log.Fatalf("%+v", Wrap(err))
+	}
+}
+
+// Check2 acts as convenience wrapper around Check, using the 2nd argument as error.
+func Check2(_ interface{}, err error) {
+	Check(err)
+}
+
+// AssertTrue asserts that b is true. Otherwise, it would log fatal.
+func AssertTrue(b bool) {
+	if !b {
+		log.Fatalf("%+v", errors.Errorf("Assert failed"))
+	}
+}
+
+// AssertTruef is AssertTrue with extra info.
+func AssertTruef(b bool, format string, args ...interface{}) {
+	if !b {
+		log.Fatalf("%+v", errors.Errorf(format, args...))
+	}
+}
+
+// Wrap wraps errors from external lib.
+func Wrap(err error) error {
+	if !debugMode {
+		return err
+	}
+	return errors.Wrap(err, "")
+}
+
+// Wrapf is Wrap with extra info.
+func Wrapf(err error, format string, args ...interface{}) error {
+	if !debugMode {
+		if err == nil {
+			return nil
+		}
+		return fmt.Errorf(format+" error: %+v", append(args, err)...)
+	}
+	return errors.Wrapf(err, format, args...)
+}

vendor/github.com/dgraph-io/badger/y/file_dsync.go

@@ -0,0 +1,25 @@
+// +build !dragonfly,!freebsd,!windows
+
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+import "golang.org/x/sys/unix"
+
+func init() {
+	datasyncFileFlag = unix.O_DSYNC
+}

vendor/github.com/dgraph-io/badger/y/file_nodsync.go

@@ -0,0 +1,25 @@
+// +build dragonfly freebsd windows
+
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+import "syscall"
+
+func init() {
+	datasyncFileFlag = syscall.O_SYNC
+}

vendor/github.com/dgraph-io/badger/y/iterator.go

@@ -0,0 +1,264 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+import (
+	"bytes"
+	"container/heap"
+	"encoding/binary"
+
+	"github.com/pkg/errors"
+)
+
+// ValueStruct represents the value info that can be associated with a key, but also the internal
+// Meta field.
+type ValueStruct struct {
+	Meta      byte
+	UserMeta  byte
+	ExpiresAt uint64
+	Value     []byte
+
+	Version uint64 // This field is not serialized. Only for internal usage.
+}
+
+func sizeVarint(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+
+// EncodedSize is the size of the ValueStruct when encoded
+func (v *ValueStruct) EncodedSize() uint16 {
+	sz := len(v.Value) + 2 // meta, usermeta.
+	if v.ExpiresAt == 0 {
+		return uint16(sz + 1)
+	}
+
+	enc := sizeVarint(v.ExpiresAt)
+	return uint16(sz + enc)
+}
+
+// Decode uses the length of the slice to infer the length of the Value field.
+func (v *ValueStruct) Decode(b []byte) {
+	v.Meta = b[0]
+	v.UserMeta = b[1]
+	var sz int
+	v.ExpiresAt, sz = binary.Uvarint(b[2:])
+	v.Value = b[2+sz:]
+}
+
+// Encode expects a slice of length at least v.EncodedSize().
+func (v *ValueStruct) Encode(b []byte) {
+	b[0] = v.Meta
+	b[1] = v.UserMeta
+	sz := binary.PutUvarint(b[2:], v.ExpiresAt)
+	copy(b[2+sz:], v.Value)
+}
+
+// EncodeTo should be kept in sync with the Encode function above. The reason
+// this function exists is to avoid creating byte arrays per key-value pair in
+// table/builder.go.
+func (v *ValueStruct) EncodeTo(buf *bytes.Buffer) {
+	buf.WriteByte(v.Meta)
+	buf.WriteByte(v.UserMeta)
+	var enc [binary.MaxVarintLen64]byte
+	sz := binary.PutUvarint(enc[:], v.ExpiresAt)
+	buf.Write(enc[:sz])
+	buf.Write(v.Value)
+}
+
+// Iterator is an interface for a basic iterator.
+type Iterator interface {
+	Next()
+	Rewind()
+	Seek(key []byte)
+	Key() []byte
+	Value() ValueStruct
+	Valid() bool
+
+	// All iterators should be closed so that file garbage collection works.
+	Close() error
+}
+
+type elem struct {
+	itr      Iterator
+	nice     int
+	reversed bool
+}
+
+type elemHeap []*elem
+
+func (eh elemHeap) Len() int            { return len(eh) }
+func (eh elemHeap) Swap(i, j int)       { eh[i], eh[j] = eh[j], eh[i] }
+func (eh *elemHeap) Push(x interface{}) { *eh = append(*eh, x.(*elem)) }
+func (eh *elemHeap) Pop() interface{} {
+	// Remove the last element, because Go has already swapped 0th elem <-> last.
+	old := *eh
+	n := len(old)
+	x := old[n-1]
+	*eh = old[0 : n-1]
+	return x
+}
+func (eh elemHeap) Less(i, j int) bool {
+	cmp := CompareKeys(eh[i].itr.Key(), eh[j].itr.Key())
+	if cmp < 0 {
+		return !eh[i].reversed
+	}
+	if cmp > 0 {
+		return eh[i].reversed
+	}
+	// The keys are equal. In this case, lower nice take precedence. This is important.
+	return eh[i].nice < eh[j].nice
+}
+
+// MergeIterator merges multiple iterators.
+// NOTE: MergeIterator owns the array of iterators and is responsible for closing them.
+type MergeIterator struct {
+	h        elemHeap
+	curKey   []byte
+	reversed bool
+
+	all []Iterator
+}
+
+// NewMergeIterator returns a new MergeIterator from a list of Iterators.
+func NewMergeIterator(iters []Iterator, reversed bool) *MergeIterator {
+	m := &MergeIterator{all: iters, reversed: reversed}
+	m.h = make(elemHeap, 0, len(iters))
+	m.initHeap()
+	return m
+}
+
+func (s *MergeIterator) storeKey(smallest Iterator) {
+	if cap(s.curKey) < len(smallest.Key()) {
+		s.curKey = make([]byte, 2*len(smallest.Key()))
+	}
+	s.curKey = s.curKey[:len(smallest.Key())]
+	copy(s.curKey, smallest.Key())
+}
+
+// initHeap checks all iterators and initializes our heap and array of keys.
+// Whenever we reverse direction, we need to run this.
+func (s *MergeIterator) initHeap() {
+	s.h = s.h[:0]
+	for idx, itr := range s.all {
+		if !itr.Valid() {
+			continue
+		}
+		e := &elem{itr: itr, nice: idx, reversed: s.reversed}
+		s.h = append(s.h, e)
+	}
+	heap.Init(&s.h)
+	for len(s.h) > 0 {
+		it := s.h[0].itr
+		if it == nil || !it.Valid() {
+			heap.Pop(&s.h)
+			continue
+		}
+		s.storeKey(s.h[0].itr)
+		break
+	}
+}
+
+// Valid returns whether the MergeIterator is at a valid element.
+func (s *MergeIterator) Valid() bool {
+	if s == nil {
+		return false
+	}
+	if len(s.h) == 0 {
+		return false
+	}
+	return s.h[0].itr.Valid()
+}
+
+// Key returns the key associated with the current iterator
+func (s *MergeIterator) Key() []byte {
+	if len(s.h) == 0 {
+		return nil
+	}
+	return s.h[0].itr.Key()
+}
+
+// Value returns the value associated with the iterator.
+func (s *MergeIterator) Value() ValueStruct {
+	if len(s.h) == 0 {
+		return ValueStruct{}
+	}
+	return s.h[0].itr.Value()
+}
+
+// Next returns the next element. If it is the same as the current key, ignore it.
+func (s *MergeIterator) Next() {
+	if len(s.h) == 0 {
+		return
+	}
+
+	smallest := s.h[0].itr
+	smallest.Next()
+
+	for len(s.h) > 0 {
+		smallest = s.h[0].itr
+		if !smallest.Valid() {
+			heap.Pop(&s.h)
+			continue
+		}
+
+		heap.Fix(&s.h, 0)
+		smallest = s.h[0].itr
+		if smallest.Valid() {
+			if !bytes.Equal(smallest.Key(), s.curKey) {
+				break
+			}
+			smallest.Next()
+		}
+	}
+	if !smallest.Valid() {
+		return
+	}
+	s.storeKey(smallest)
+}
+
+// Rewind seeks to first element (or last element for reverse iterator).
+func (s *MergeIterator) Rewind() {
+	for _, itr := range s.all {
+		itr.Rewind()
+	}
+	s.initHeap()
+}
+
+// Seek brings us to element with key >= given key.
+func (s *MergeIterator) Seek(key []byte) {
+	for _, itr := range s.all {
+		itr.Seek(key)
+	}
+	s.initHeap()
+}
+
+// Close implements y.Iterator
+func (s *MergeIterator) Close() error {
+	for _, itr := range s.all {
+		if err := itr.Close(); err != nil {
+			return errors.Wrap(err, "MergeIterator")
+		}
+	}
+	return nil
+}

vendor/github.com/dgraph-io/badger/y/metrics.go

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+import "expvar"
+
+var (
+	// LSMSize has size of the LSM in bytes
+	LSMSize *expvar.Map
+	// VlogSize has size of the value log in bytes
+	VlogSize *expvar.Map
+	// PendingWrites tracks the number of pending writes.
+	PendingWrites *expvar.Map
+
+	// These are cumulative
+
+	// NumReads has cumulative number of reads
+	NumReads *expvar.Int
+	// NumWrites has cumulative number of writes
+	NumWrites *expvar.Int
+	// NumBytesRead has cumulative number of bytes read
+	NumBytesRead *expvar.Int
+	// NumBytesWritten has cumulative number of bytes written
+	NumBytesWritten *expvar.Int
+	// NumLSMGets is number of LMS gets
+	NumLSMGets *expvar.Map
+	// NumLSMBloomHits is number of LMS bloom hits
+	NumLSMBloomHits *expvar.Map
+	// NumGets is number of gets
+	NumGets *expvar.Int
+	// NumPuts is number of puts
+	NumPuts *expvar.Int
+	// NumBlockedPuts is number of blocked puts
+	NumBlockedPuts *expvar.Int
+	// NumMemtableGets is number of memtable gets
+	NumMemtableGets *expvar.Int
+)
+
+// These variables are global and have cumulative values for all kv stores.
+func init() {
+	NumReads = expvar.NewInt("badger_disk_reads_total")
+	NumWrites = expvar.NewInt("badger_disk_writes_total")
+	NumBytesRead = expvar.NewInt("badger_read_bytes")
+	NumBytesWritten = expvar.NewInt("badger_written_bytes")
+	NumLSMGets = expvar.NewMap("badger_lsm_level_gets_total")
+	NumLSMBloomHits = expvar.NewMap("badger_lsm_bloom_hits_total")
+	NumGets = expvar.NewInt("badger_gets_total")
+	NumPuts = expvar.NewInt("badger_puts_total")
+	NumBlockedPuts = expvar.NewInt("badger_blocked_puts_total")
+	NumMemtableGets = expvar.NewInt("badger_memtable_gets_total")
+	LSMSize = expvar.NewMap("badger_lsm_size_bytes")
+	VlogSize = expvar.NewMap("badger_vlog_size_bytes")
+	PendingWrites = expvar.NewMap("badger_pending_writes_total")
+}

vendor/github.com/dgraph-io/badger/y/mmap_unix.go

@@ -0,0 +1,63 @@
+// +build !windows
+
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+import (
+	"os"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// Mmap uses the mmap system call to memory-map a file. If writable is true,
+// memory protection of the pages is set so that they may be written to as well.
+func Mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
+	mtype := unix.PROT_READ
+	if writable {
+		mtype |= unix.PROT_WRITE
+	}
+	return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
+}
+
+// Munmap unmaps a previously mapped slice.
+func Munmap(b []byte) error {
+	return unix.Munmap(b)
+}
+
+// Madvise uses the madvise system call to give advise about the use of memory
+// when using a slice that is memory-mapped to a file. Set the readahead flag to
+// false if page references are expected in random order.
+func Madvise(b []byte, readahead bool) error {
+	flags := unix.MADV_NORMAL
+	if !readahead {
+		flags = unix.MADV_RANDOM
+	}
+	return madvise(b, flags)
+}
+
+// This is required because the unix package does not support the madvise system call on OS X.
+func madvise(b []byte, advice int) (err error) {
+	_, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])),
+		uintptr(len(b)), uintptr(advice))
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}

vendor/github.com/dgraph-io/badger/y/mmap_windows.go

@@ -0,0 +1,90 @@
+// +build windows
+
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"unsafe"
+)
+
+func Mmap(fd *os.File, write bool, size int64) ([]byte, error) {
+	protect := syscall.PAGE_READONLY
+	access := syscall.FILE_MAP_READ
+
+	if write {
+		protect = syscall.PAGE_READWRITE
+		access = syscall.FILE_MAP_WRITE
+	}
+	fi, err := fd.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	// Truncate the database to the size of the mmap.
+	if fi.Size() < size {
+		if err := fd.Truncate(size); err != nil {
+			return nil, fmt.Errorf("truncate: %s", err)
+		}
+	}
+
+	// Open a file mapping handle.
+	sizelo := uint32(size >> 32)
+	sizehi := uint32(size) & 0xffffffff
+
+	handler, err := syscall.CreateFileMapping(syscall.Handle(fd.Fd()), nil,
+		uint32(protect), sizelo, sizehi, nil)
+	if err != nil {
+		return nil, os.NewSyscallError("CreateFileMapping", err)
+	}
+
+	// Create the memory map.
+	addr, err := syscall.MapViewOfFile(handler, uint32(access), 0, 0, uintptr(size))
+	if addr == 0 {
+		return nil, os.NewSyscallError("MapViewOfFile", err)
+	}
+
+	// Close mapping handle.
+	if err := syscall.CloseHandle(syscall.Handle(handler)); err != nil {
+		return nil, os.NewSyscallError("CloseHandle", err)
+	}
+
+	// Slice memory layout
+	// Copied this snippet from golang/sys package
+	var sl = struct {
+		addr uintptr
+		len  int
+		cap  int
+	}{addr, int(size), int(size)}
+
+	// Use unsafe to turn sl into a []byte.
+	data := *(*[]byte)(unsafe.Pointer(&sl))
+
+	return data, nil
+}
+
+func Munmap(b []byte) error {
+	return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&b[0])))
+}
+
+func Madvise(b []byte, readahead bool) error {
+	// Do Nothing. We don’t care about this setting on Windows
+	return nil
+}

vendor/github.com/dgraph-io/badger/y/y.go

@@ -0,0 +1,209 @@
+/*
+ * Copyright 2017 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package y
+
+import (
+	"bytes"
+	"encoding/binary"
+	"hash/crc32"
+	"math"
+	"os"
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+// ErrEOF indicates an end of file when trying to read from a memory mapped file
+// and encountering the end of slice.
+var ErrEOF = errors.New("End of mapped region")
+
+const (
+	// Sync indicates that O_DSYNC should be set on the underlying file,
+	// ensuring that data writes do not return until the data is flushed
+	// to disk.
+	Sync = 1 << iota
+	// ReadOnly opens the underlying file on a read-only basis.
+	ReadOnly
+)
+
+var (
+	// This is O_DSYNC (datasync) on platforms that support it -- see file_unix.go
+	datasyncFileFlag = 0x0
+
+	// CastagnoliCrcTable is a CRC32 polynomial table
+	CastagnoliCrcTable = crc32.MakeTable(crc32.Castagnoli)
+)
+
+// OpenExistingFile opens an existing file, errors if it doesn't exist.
+func OpenExistingFile(filename string, flags uint32) (*os.File, error) {
+	openFlags := os.O_RDWR
+	if flags&ReadOnly != 0 {
+		openFlags = os.O_RDONLY
+	}
+
+	if flags&Sync != 0 {
+		openFlags |= datasyncFileFlag
+	}
+	return os.OpenFile(filename, openFlags, 0)
+}
+
+// CreateSyncedFile creates a new file (using O_EXCL), errors if it already existed.
+func CreateSyncedFile(filename string, sync bool) (*os.File, error) {
+	flags := os.O_RDWR | os.O_CREATE | os.O_EXCL
+	if sync {
+		flags |= datasyncFileFlag
+	}
+	return os.OpenFile(filename, flags, 0666)
+}
+
+// OpenSyncedFile creates the file if one doesn't exist.
+func OpenSyncedFile(filename string, sync bool) (*os.File, error) {
+	flags := os.O_RDWR | os.O_CREATE
+	if sync {
+		flags |= datasyncFileFlag
+	}
+	return os.OpenFile(filename, flags, 0666)
+}
+
+// OpenTruncFile opens the file with O_RDWR | O_CREATE | O_TRUNC
+func OpenTruncFile(filename string, sync bool) (*os.File, error) {
+	flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC
+	if sync {
+		flags |= datasyncFileFlag
+	}
+	return os.OpenFile(filename, flags, 0666)
+}
+
+// SafeCopy does append(a[:0], src...).
+func SafeCopy(a []byte, src []byte) []byte {
+	return append(a[:0], src...)
+}
+
+// Copy copies a byte slice and returns the copied slice.
+func Copy(a []byte) []byte {
+	b := make([]byte, len(a))
+	copy(b, a)
+	return b
+}
+
+// KeyWithTs generates a new key by appending ts to key.
+func KeyWithTs(key []byte, ts uint64) []byte {
+	out := make([]byte, len(key)+8)
+	copy(out, key)
+	binary.BigEndian.PutUint64(out[len(key):], math.MaxUint64-ts)
+	return out
+}
+
+// ParseTs parses the timestamp from the key bytes.
+func ParseTs(key []byte) uint64 {
+	if len(key) <= 8 {
+		return 0
+	}
+	return math.MaxUint64 - binary.BigEndian.Uint64(key[len(key)-8:])
+}
+
+// CompareKeys checks the key without timestamp and checks the timestamp if keyNoTs
+// is same.
+// a<timestamp> would be sorted higher than aa<timestamp> if we use bytes.compare
+// All keys should have timestamp.
+func CompareKeys(key1 []byte, key2 []byte) int {
+	AssertTrue(len(key1) > 8 && len(key2) > 8)
+	if cmp := bytes.Compare(key1[:len(key1)-8], key2[:len(key2)-8]); cmp != 0 {
+		return cmp
+	}
+	return bytes.Compare(key1[len(key1)-8:], key2[len(key2)-8:])
+}
+
+// ParseKey parses the actual key from the key bytes.
+func ParseKey(key []byte) []byte {
+	if key == nil {
+		return nil
+	}
+
+	AssertTruef(len(key) > 8, "key=%q", key)
+	return key[:len(key)-8]
+}
+
+// SameKey checks for key equality ignoring the version timestamp suffix.
+func SameKey(src, dst []byte) bool {
+	if len(src) != len(dst) {
+		return false
+	}
+	return bytes.Equal(ParseKey(src), ParseKey(dst))
+}
+
+// Slice holds a reusable buf, will reallocate if you request a larger size than ever before.
+// One problem is with n distinct sizes in random order it'll reallocate log(n) times.
+type Slice struct {
+	buf []byte
+}
+
+// Resize reuses the Slice's buffer (or makes a new one) and returns a slice in that buffer of
+// length sz.
+func (s *Slice) Resize(sz int) []byte {
+	if cap(s.buf) < sz {
+		s.buf = make([]byte, sz)
+	}
+	return s.buf[0:sz]
+}
+
+// Closer holds the two things we need to close a goroutine and wait for it to finish: a chan
+// to tell the goroutine to shut down, and a WaitGroup with which to wait for it to finish shutting
+// down.
+type Closer struct {
+	closed  chan struct{}
+	waiting sync.WaitGroup
+}
+
+// NewCloser constructs a new Closer, with an initial count on the WaitGroup.
+func NewCloser(initial int) *Closer {
+	ret := &Closer{closed: make(chan struct{})}
+	ret.waiting.Add(initial)
+	return ret
+}
+
+// AddRunning Add()'s delta to the WaitGroup.
+func (lc *Closer) AddRunning(delta int) {
+	lc.waiting.Add(delta)
+}
+
+// Signal signals the HasBeenClosed signal.
+func (lc *Closer) Signal() {
+	close(lc.closed)
+}
+
+// HasBeenClosed gets signaled when Signal() is called.
+func (lc *Closer) HasBeenClosed() <-chan struct{} {
+	return lc.closed
+}
+
+// Done calls Done() on the WaitGroup.
+func (lc *Closer) Done() {
+	lc.waiting.Done()
+}
+
+// Wait waits on the WaitGroup.  (It waits for NewCloser's initial value, AddRunning, and Done
+// calls to balance out.)
+func (lc *Closer) Wait() {
+	lc.waiting.Wait()
+}
+
+// SignalAndWait calls Signal(), then Wait().
+func (lc *Closer) SignalAndWait() {
+	lc.Signal()
+	lc.Wait()
+}