Add Filestore.Verify config option to control when block are verified.

The default value of "ifchanged" and blocks are only verified when
the backing file has changed.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

filestore/datastore.go

@@ -13,10 +13,13 @@ import (
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
 	k "github.com/ipfs/go-ipfs/blocks/key"
 	//mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
-	//b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
+	b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
+	logging "gx/ipfs/QmaDNZ4QMdBdku1YZWBysufYyoQt1negQGNav6PLYarbY8/go-log"
 )
 
+var log = logging.Logger("filestore")
+
 const (
 	VerifyNever     = 0
 	VerifyIfChanged = 1
@@ -28,8 +31,8 @@ type Datastore struct {
 	verify int
 }
 
-func New(d ds.Datastore, fileStorePath string) (*Datastore, error) {
-	return &Datastore{d, VerifyIfChanged}, nil
+func New(d ds.Datastore, fileStorePath string, verify int) (*Datastore, error) {
+	return &Datastore{d, verify}, nil
 }
 
 func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
@@ -131,6 +134,9 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
 	if val == nil {
 		return nil, errors.New("Nil DataObj")
 	} else if val.NoBlockData() {
+		if verify != VerifyIfChanged {
+			update = false
+		}
 		file, err := os.Open(val.FilePath)
 		if err != nil {
 			return nil, err
@@ -150,7 +156,7 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
 			}
 			data, err = reconstruct(val.Data, buf)
 		}
-		if err != nil  && err != io.EOF && err != io.ErrUnexpectedEOF {
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			return nil, err
 		}
 		modtime := val.ModTime
@@ -163,19 +169,20 @@ func (d *Datastore) GetData(key ds.Key, val *DataObj, verify int, update bool) (
 		}
 		invalid := val.Invalid() || err != nil
 		if err == nil && (verify == VerifyAlways || (verify == VerifyIfChanged && modtime != val.ModTime)) {
-			//println("verifying")
+			log.Debugf("verifying block %s\n", b58.Encode(key.Bytes()[1:]))
 			newKey := k.Key(u.Hash(data)).DsKey()
 			invalid = newKey != key
 		}
 		if update && (invalid != val.Invalid() || modtime != val.ModTime) {
-			//println("updating")
+			log.Debugf("updating block %s\n", b58.Encode(key.Bytes()[1:]))
 			newVal := *val
 			newVal.SetInvalid(invalid)
 			newVal.ModTime = modtime
 			// ignore errors as they are nonfatal
 			_ = d.put(key, &newVal)
 		}
 		if invalid {
+			log.Debugf("invalid block %s\n", b58.Encode(key.Bytes()[1:]))
 			return nil, InvalidBlock{}
 		} else {
 			return data, nil

repo/config/config.go

@@ -19,6 +19,7 @@ var log = logging.Logger("config")
 type Config struct {
 	Identity         Identity              // local node's peer identity
 	Datastore        Datastore             // local node's storage
+	Filestore        Filestore             // local node's filestore
 	Addresses        Addresses             // local node's addresses
 	Mounts           Mounts                // local node's mount points
 	Discovery        Discovery             // local node's discovery mechanisms

repo/config/datastore.go

@@ -38,3 +38,7 @@ type S3Datastore struct {
 func DataStorePath(configroot string) (string, error) {
 	return Path(configroot, DefaultDataStoreDirectory)
 }
+
+type Filestore struct {
+	Verify string // one of "always", "ifchanged", "never"
+}

repo/fsrepo/defaultds.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"strings"
 
 	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/flatfs"
@@ -71,15 +72,10 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, *filestore.Datastore, erro
 
 	var fileStore *filestore.Datastore
 	if useFileStore {
-		fileStorePath := path.Join(r.path, fileStoreDir)
-		fileStoreDB, err := levelds.NewDatastore(fileStorePath, &levelds.Options{
-			Compression: ldbopts.NoCompression,
-		})
+		fileStore, err = r.newFilestore()
 		if err != nil {
-			return nil, nil, fmt.Errorf("unable to open filestore: %v", err)
+			return nil, nil, err
 		}
-		fileStore, _ = filestore.New(fileStoreDB, "")
-		//fileStore.(io.Closer).Close()
 		blocksStore = multi.New(fileStore, metricsBlocks, nil, nil)
 	}
 
@@ -111,3 +107,27 @@ func initDefaultDatastore(repoPath string, conf *config.Config) error {
 	}
 	return nil
 }
+
+func (r *FSRepo) newFilestore() (*filestore.Datastore, error) {
+	fileStorePath := path.Join(r.path, fileStoreDir)
+	fileStoreDB, err := levelds.NewDatastore(fileStorePath, &levelds.Options{
+		Compression: ldbopts.NoCompression,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to open filestore: %v", err)
+	}
+	verify := filestore.VerifyIfChanged
+	switch strings.ToLower(r.config.Filestore.Verify) {
+	case "never":
+		verify = filestore.VerifyNever
+	case "":
+	case "ifchanged":
+	case "if changed":
+		verify = filestore.VerifyIfChanged
+	case "always":
+		verify = filestore.VerifyAlways
+	default:
+		return nil, fmt.Errorf("invalid value for Filestore.Verify: %s", r.config.Filestore.Verify)
+	}
+	return filestore.New(fileStoreDB, "", verify)
+}

test/sharness/t0262-filestore-config.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Copyright (c) 2014 Christian Couder
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test filestore"
+
+. lib/test-filestore-lib.sh
+. lib/test-lib.sh
+
+test_init_ipfs
+
+test_add_cat_file "add --no-copy" "."
+
+export IPFS_LOGGING=debug
+export IPFS_LOGGING_FMT=nocolor
+
+HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH"
+
+test_expect_success "file always checked" '
+  ipfs config Filestore.Verify always 2> log &&
+  ipfs cat "$HASH" 2> log &&
+  grep -q "verifying block $HASH" log &&
+  ! grep -q "updating block $HASH" log
+'
+
+test_expect_success "file checked after change" '
+  ipfs config Filestore.Verify ifchanged 2> log &&
+  echo "HELLO WORLDS!" >mountdir/hello.txt &&
+  test_must_fail ipfs cat "$HASH" 2> log &&
+  grep -q "verifying block $HASH" log &&
+  grep -q "updating block $HASH" log
+'
+
+test_expect_success "file never checked" '
+  echo "Hello Worlds!" >mountdir/hello.txt &&
+  ipfs add "$dir"/mountdir/hello.txt >actual 2> log &&
+  ipfs config Filestore.Verify never 2> log &&
+  echo "HELLO Worlds!" >mountdir/hello.txt &&
+  ( ipfs cat "$HASH" || true ) 2> log &&
+  grep -q "BlockService GetBlock" log && # Make sure we are still logging
+  ! grep -q "verifying block $HASH" log &&
+  ! grep -q "updating block $HASH" log 
+'
+
+test_done