Switch Badger Options to reasonable settings for p and w directories. (…

…#2605)

* Pick Badger Options which use LSM tree a bit more, so it would colocate values below 1KB with the keys. This allows easier reclamation of value log space, reducing the size of data consumption by Dgraph. A test run of 21M data set, resulted in 3.3GB, compared to 5.7GB with previous settings. The number of value log files also decreased considerably due to easier cleanup.

* This change removes the need to expose `--badger.options=ssd,hdd` options. We've chosen a middle ground setting, which should work in most cases.

* Remove the none option, which is confusing.
* Output the options used to open WAL as well.
* Pull in Badger updates.

dgraph/cmd/server/run.go

@@ -62,16 +62,13 @@ func init() {
 		"Directory to store posting lists.")
 
 	// Options around how to set up Badger.
-	flag.String("badger.options", defaults.BadgerOptions,
-		"[ssd, hdd] Specifies which Badger options to use. SSD decreases write amplification"+
-			" and increases read IOPS. HDD does the opposite.")
 	flag.String("badger.tables", defaults.BadgerTables,
-		"[none, ram, mmap, disk] Specifies how Badger LSM tree is stored. "+
+		"[ram, mmap, disk] Specifies how Badger LSM tree is stored. "+
 			"Option sequence consume most to least RAM while providing best to worst read "+
 			"performance respectively.")
 	flag.String("badger.vlog", defaults.BadgerVlog,
-		"[none, mmap, disk] Specifies how Badger Value log is stored."+
-			" mmap consumes more RAM, but provides better performance in some cases.")
+		"[mmap, disk] Specifies how Badger Value log is stored."+
+			" mmap consumes more RAM, but provides better performance.")
 
 	flag.StringP("wal", "w", defaults.WALDir,
 		"Directory to store raft write-ahead logs.")
@@ -278,9 +275,8 @@ var shutdownCh chan struct{}
 
 func run() {
 	config := edgraph.Options{
-		BadgerTables:  Server.Conf.GetString("badger.tables"),
-		BadgerVlog:    Server.Conf.GetString("badger.vlog"),
-		BadgerOptions: Server.Conf.GetString("badger.options"),
+		BadgerTables: Server.Conf.GetString("badger.tables"),
+		BadgerVlog:   Server.Conf.GetString("badger.vlog"),
 
 		PostingDir: Server.Conf.GetString("postings"),
 		WALDir:     Server.Conf.GetString("wal"),

dgraph/cmd/zero/run.go

@@ -177,6 +177,7 @@ func run() {
 	kvOpt.Truncate = true
 	kvOpt.Dir = opts.w
 	kvOpt.ValueDir = opts.w
+	kvOpt.ValueLogFileSize = 64 << 20
 	kv, err := badger.Open(kvOpt)
 	x.Checkf(err, "Error while opening WAL store")
 	defer kv.Close()

edgraph/config.go

@@ -22,12 +22,11 @@ import (
 )
 
 type Options struct {
-	PostingDir    string
-	BadgerTables  string
-	BadgerVlog    string
-	BadgerOptions string
-	WALDir        string
-	Nomutations   bool
+	PostingDir   string
+	BadgerTables string
+	BadgerVlog   string
+	WALDir       string
+	Nomutations  bool
 
 	AllottedMemory float64
 
@@ -48,12 +47,11 @@ type Options struct {
 var Config Options
 
 var DefaultConfig = Options{
-	PostingDir:    "p",
-	BadgerTables:  "mmap",
-	BadgerVlog:    "none",
-	BadgerOptions: "ssd",
-	WALDir:        "w",
-	Nomutations:   false,
+	PostingDir:   "p",
+	BadgerTables: "mmap",
+	BadgerVlog:   "mmap",
+	WALDir:       "w",
+	Nomutations:  false,
 
 	// User must specify this.
 	AllottedMemory: -1.0,
@@ -105,7 +103,6 @@ func setConfVar(conf Options) {
 	// This is so we can find these options in /debug/vars.
 	x.Conf.Set("badger.tables", newStr(conf.BadgerTables))
 	x.Conf.Set("badger.vlog", newStr(conf.BadgerVlog))
-	x.Conf.Set("badger.options", newStr(conf.BadgerOptions))
 
 	x.Conf.Set("posting_dir", newStr(conf.PostingDir))
 	x.Conf.Set("wal_dir", newStr(conf.WALDir))

edgraph/server.go

@@ -97,43 +97,14 @@ func (s *ServerState) runVlogGC(store *badger.DB) {
 	}
 }
 
-func (s *ServerState) initStorage() {
-	// Write Ahead Log directory
-	x.Checkf(os.MkdirAll(Config.WALDir, 0700), "Error while creating WAL dir.")
-	kvOpt := badger.LSMOnlyOptions
-	kvOpt.SyncWrites = true
-	kvOpt.Truncate = true
-	kvOpt.Dir = Config.WALDir
-	kvOpt.ValueDir = Config.WALDir
-	kvOpt.TableLoadingMode = options.MemoryMap
-
-	var err error
-	s.WALstore, err = badger.Open(kvOpt)
-	x.Checkf(err, "Error while creating badger KV WAL store")
-
-	// Postings directory
-	// All the writes to posting store should be synchronous. We use batched writers
-	// for posting lists, so the cost of sync writes is amortized.
-	x.Check(os.MkdirAll(Config.PostingDir, 0700))
-	x.Printf("Setting Badger option: %s", Config.BadgerOptions)
-	var opt badger.Options
-	switch Config.BadgerOptions {
-	case "ssd":
-		opt = badger.DefaultOptions
-	case "hdd":
-		opt = badger.LSMOnlyOptions
-	default:
-		x.Fatalf("Invalid Badger options")
-	}
+func setBadgerOptions(opt badger.Options, dir string) badger.Options {
 	opt.SyncWrites = true
 	opt.Truncate = true
-	opt.Dir = Config.PostingDir
-	opt.ValueDir = Config.PostingDir
-	opt.NumVersionsToKeep = math.MaxInt32
+	opt.Dir = dir
+	opt.ValueDir = dir
 
-	x.Printf("Setting Badger table load option: %s", Config.BadgerTables)
+	glog.Infof("Setting Badger table load option: %s", Config.BadgerTables)
 	switch Config.BadgerTables {
-	case "none": // Use default based on BadgerOptions.
 	case "mmap":
 		opt.TableLoadingMode = options.MemoryMap
 	case "ram":
@@ -144,23 +115,48 @@ func (s *ServerState) initStorage() {
 		x.Fatalf("Invalid Badger Tables options")
 	}
 
-	x.Printf("Setting Badger value log load option: %s", Config.BadgerVlog)
+	glog.Infof("Setting Badger value log load option: %s", Config.BadgerVlog)
 	switch Config.BadgerVlog {
-	case "none": // Use default based on BadgerOptions.
 	case "mmap":
 		opt.ValueLogLoadingMode = options.MemoryMap
 	case "disk":
 		opt.ValueLogLoadingMode = options.FileIO
 	default:
 		x.Fatalf("Invalid Badger Value log options")
 	}
+	return opt
+}
+
+func (s *ServerState) initStorage() {
+	var err error
+	{
+		// Write Ahead Log directory
+		x.Checkf(os.MkdirAll(Config.WALDir, 0700), "Error while creating WAL dir.")
+		opt := badger.LSMOnlyOptions
+		opt.ValueLogMaxEntries = 1000 // Allow for easy space reclamation.
+		opt = setBadgerOptions(opt, Config.WALDir)
+
+		glog.Infof("Opening write-ahead log BadgerDB with options: %+v\n", opt)
+		s.WALstore, err = badger.Open(opt)
+		x.Checkf(err, "Error while creating badger KV WAL store")
+	}
+	{
+		// Postings directory
+		// All the writes to posting store should be synchronous. We use batched writers
+		// for posting lists, so the cost of sync writes is amortized.
+		x.Check(os.MkdirAll(Config.PostingDir, 0700))
+		opt := badger.DefaultOptions
+		opt.ValueThreshold = 1 << 10 // 1KB
+		opt.NumVersionsToKeep = math.MaxInt32
+		opt = setBadgerOptions(opt, Config.PostingDir)
+
+		glog.Infof("Opening postings BadgerDB with options: %+v\n", opt)
+		s.Pstore, err = badger.OpenManaged(opt)
+		x.Checkf(err, "Error while creating badger KV posting store")
+	}
 
-	x.Printf("Opening postings Badger DB with options: %+v\n", opt)
-	s.Pstore, err = badger.OpenManaged(opt)
-	x.Checkf(err, "Error while creating badger KV posting store")
 	s.vlogTicker = time.NewTicker(1 * time.Minute)
 	s.mandatoryVlogTicker = time.NewTicker(10 * time.Minute)
-
 	go s.runVlogGC(s.Pstore)
 	go s.runVlogGC(s.WALstore)
 }

vendor/vendor.json

@@ -163,50 +163,50 @@
 			"revisionTime": "2016-09-07T16:21:46Z"
 		},
 		{
-			"checksumSHA1": "otgzUXMMH6BbCjO0uu8gqvMP8rA=",
+			"checksumSHA1": "JbgJNqZaQFtV8KXrUjDva5m2blI=",
 			"path": "github.com/dgraph-io/badger",
-			"revision": "d22c0e8a900d591da50fd59c07db79d75866a397",
-			"revisionTime": "2018-09-19T17:44:01Z",
+			"revision": "502db7429a172f57aa5843fba65e59c4ed73ff73",
+			"revisionTime": "2018-09-20T18:49:27Z",
 			"version": "HEAD",
 			"versionExact": "HEAD"
 		},
 		{
 			"checksumSHA1": "oOuT7ebEiZ1ViHLKdFxKFOvobAQ=",
 			"path": "github.com/dgraph-io/badger/options",
-			"revision": "d22c0e8a900d591da50fd59c07db79d75866a397",
-			"revisionTime": "2018-09-19T17:44:01Z",
+			"revision": "502db7429a172f57aa5843fba65e59c4ed73ff73",
+			"revisionTime": "2018-09-20T18:49:27Z",
 			"version": "HEAD",
 			"versionExact": "HEAD"
 		},
 		{
 			"checksumSHA1": "gGTDnTVVw5kcT2P5NXZV1YSckOU=",
 			"path": "github.com/dgraph-io/badger/protos",
-			"revision": "d22c0e8a900d591da50fd59c07db79d75866a397",
-			"revisionTime": "2018-09-19T17:44:01Z",
+			"revision": "502db7429a172f57aa5843fba65e59c4ed73ff73",
+			"revisionTime": "2018-09-20T18:49:27Z",
 			"version": "HEAD",
 			"versionExact": "HEAD"
 		},
 		{
 			"checksumSHA1": "00T6XbLV4d95J7hm6kTXDReaQHM=",
 			"path": "github.com/dgraph-io/badger/skl",
-			"revision": "d22c0e8a900d591da50fd59c07db79d75866a397",
-			"revisionTime": "2018-09-19T17:44:01Z",
+			"revision": "502db7429a172f57aa5843fba65e59c4ed73ff73",
+			"revisionTime": "2018-09-20T18:49:27Z",
 			"version": "HEAD",
 			"versionExact": "HEAD"
 		},
 		{
 			"checksumSHA1": "I33KkP2lnYqJDasvvsAlebzkeko=",
 			"path": "github.com/dgraph-io/badger/table",
-			"revision": "d22c0e8a900d591da50fd59c07db79d75866a397",
-			"revisionTime": "2018-09-19T17:44:01Z",
+			"revision": "502db7429a172f57aa5843fba65e59c4ed73ff73",
+			"revisionTime": "2018-09-20T18:49:27Z",
 			"version": "HEAD",
 			"versionExact": "HEAD"
 		},
 		{
 			"checksumSHA1": "v2pJQ5NbS034cLP+GM1WLlGnByY=",
 			"path": "github.com/dgraph-io/badger/y",
-			"revision": "d22c0e8a900d591da50fd59c07db79d75866a397",
-			"revisionTime": "2018-09-19T17:44:01Z",
+			"revision": "502db7429a172f57aa5843fba65e59c4ed73ff73",
+			"revisionTime": "2018-09-20T18:49:27Z",
 			"version": "HEAD",
 			"versionExact": "HEAD"
 		},
@@ -273,8 +273,8 @@
 		{
 			"checksumSHA1": "GaJLoEuMGnP5ofXvuweAI4wx06U=",
 			"path": "github.com/golang/protobuf/proto",
-			"revision": "6e3d092c77c3bae25acf4925942b568e742c1c20",
-			"revisionTime": "2018-09-17T22:48:16Z"
+			"revision": "7716a980bceefe9a0213654823d9ea62d23b87a2",
+			"revisionTime": "2018-09-19T22:46:59Z"
 		},
 		{
 			"checksumSHA1": "z4copNgeTN77OymdDKqLaIK/vSI=",