From 23f9abc163f1d5b13b2a319bb27ad9e7a110e921 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sat, 28 May 2016 14:21:39 -0400 Subject: [PATCH] "filestore ls": enhance License: MIT Signed-off-by: Kevin Atkinson --- core/commands/filestore.go | 98 +++++++++++++++++++++++++++----- filestore/util/common.go | 22 ++++++- test/sharness/t0260-filestore.sh | 14 +++-- 3 files changed, 116 insertions(+), 18 deletions(-) diff --git a/core/commands/filestore.go b/core/commands/filestore.go index b6f28a7a4157..843b36281644 100644 --- a/core/commands/filestore.go +++ b/core/commands/filestore.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "path/filepath" + "strings" //ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore" //bs "github.com/ipfs/go-ipfs/blocks/blockstore" @@ -31,7 +32,7 @@ var FileStoreCmd = &cmds.Command{ "unpinned": fsUnpinned, "rm-dups": rmDups, "upgrade": fsUpgrade, - "mv": moveIntoFilestore, + "mv": moveIntoFilestore, }, } @@ -39,8 +40,18 @@ var lsFileStore = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List objects in filestore", ShortDescription: ` -List objects in the filestore. If --quiet is specified only the -hashes are printed, otherwise the fields are as follows: +List objects in the filestore. If one or more is specified only +list those specific objects, otherwise list all objects. An can +either be a multihash, or an absolute path. If the path ends in '/' +than it is assumed to be a directory and all paths with that directory +are included. + +If --all is specified list all matching blocks are lists, otherwise +only blocks representing the a file root is listed. A file root is any +block that represents a complete file. + +If --quiet is specified only the hashes are printed, otherwise the +fields are as follows: [] where is one of" leaf: to indicate a node where the contents are stored @@ -50,12 +61,15 @@ where is one of" invld: a leaf node that has been found invalid and is the part of the file the object represents. The part represented starts at and continues for bytes. -If is the special value "-" than the "leaf" or "root" node -represents the whole file. +If is the special value "-" indicates a file root. `, }, + Arguments: []cmds.Argument{ + cmds.StringArg("obj", false, true, "Hash or filename to list."), + }, Options: []cmds.Option{ cmds.BoolOption("quiet", "q", "Write just hashes of objects."), + cmds.BoolOption("all", "a", "List everything, not just file roots."), }, Run: func(req cmds.Request, res cmds.Response) { _, fs, err := extractFilestore(req) @@ -68,12 +82,49 @@ represents the whole file. res.SetError(err, cmds.ErrNormal) return } + all, _, err := res.Request().Option("all").Bool() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + objs := req.Arguments() + keys := make([]k.Key, 0) + paths := make([]string, 0) + for _, obj := range objs { + if filepath.IsAbs(obj) { + paths = append(paths, obj) + } else { + keys = append(keys, k.B58KeyDecode(obj)) + } + } + if len(keys) > 0 && len(paths) > 0 { + res.SetError(errors.New("Cannot specify both hashes and paths."), cmds.ErrNormal) + return + } + + var ch <-chan fsutil.ListRes + if len(keys) > 0 { + ch, _ = fsutil.ListByKey(fs, keys) + } else if all && len(paths) == 0 && quiet { + ch, _ = fsutil.ListKeys(fs) + } else if all && len(paths) == 0 { + ch, _ = fsutil.ListAll(fs) + } else if !all && len(paths) == 0 { + ch, _ = fsutil.ListWholeFile(fs) + } else if all { + ch, _ = fsutil.List(fs, func(r fsutil.ListRes) bool { + return pathMatch(paths, r.FilePath) + }) + } else { + ch, _ = fsutil.List(fs, func(r fsutil.ListRes) bool { + return r.WholeFile() && pathMatch(paths, r.FilePath) + }) + } + if quiet { - ch, _ := fsutil.ListKeys(fs) - res.SetOutput(&chanWriter{ch, "", 0, false}) + res.SetOutput(&chanWriter{ch: ch, quiet: true}) } else { - ch, _ := fsutil.ListAll(fs) - res.SetOutput(&chanWriter{ch, "", 0, false}) + res.SetOutput(&chanWriter{ch: ch}) } }, Marshalers: cmds.MarshalerMap{ @@ -83,6 +134,22 @@ represents the whole file. }, } +func pathMatch(match_list []string, path string) bool { + for _, to_match := range match_list { + if to_match[len(to_match)-1] == filepath.Separator { + if strings.HasPrefix(path, to_match) { + return true + } + } else { + if to_match == path { + return true + } + } + } + return false + +} + var lsFiles = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "List files in filestore", @@ -121,6 +188,7 @@ type chanWriter struct { buf string offset int errors bool + quiet bool } func (w *chanWriter) Read(p []byte) (int, error) { @@ -134,7 +202,11 @@ func (w *chanWriter) Read(p []byte) (int, error) { } else if fsutil.AnError(res.Status) { w.errors = true } - w.buf = res.Format() + if w.quiet { + w.buf = fmt.Sprintf("%s\n", res.MHash()) + } else { + w.buf = res.Format() + } } sz := copy(p, w.buf[w.offset:]) w.offset += sz @@ -255,10 +327,10 @@ The --verbose option specifies what to output. The current values are: } if basic { ch, _ := fsutil.VerifyBasic(fs, level, verbose) - res.SetOutput(&chanWriter{ch, "", 0, false}) + res.SetOutput(&chanWriter{ch: ch}) } else { ch, _ := fsutil.VerifyFull(node, fs, level, verbose, skipOrphans) - res.SetOutput(&chanWriter{ch, "", 0, false}) + res.SetOutput(&chanWriter{ch: ch}) } }, Marshalers: cmds.MarshalerMap{ @@ -562,7 +634,7 @@ copy is not removed. Use "filestore rm-dups" to remove the old copy. path = mhash } if offline { - path,err = filepath.Abs(path) + path, err = filepath.Abs(path) if err != nil { res.SetError(err, cmds.ErrNormal) return diff --git a/filestore/util/common.go b/filestore/util/common.go index a43028cdd19f..68f793527716 100644 --- a/filestore/util/common.go +++ b/filestore/util/common.go @@ -143,7 +143,11 @@ func List(d *Datastore, filter func(ListRes) bool) (<-chan ListRes, error) { } key := ds.NewKey(r.Key) val, _ := d.GetDirect(key) - out <- ListRes{key, val, 0} + res := ListRes{key, val, 0} + keep := filter(res) + if keep { + out <- res + } } }() return out, nil @@ -157,6 +161,22 @@ func ListWholeFile(d *Datastore) (<-chan ListRes, error) { return List(d, func(r ListRes) bool { return r.WholeFile() }) } +func ListByKey(fs *Datastore, keys []k.Key) (<-chan ListRes, error) { + out := make(chan ListRes, 128) + + go func() { + defer close(out) + for _, key := range keys { + dsKey := key.DsKey() + dataObj, err := fs.GetDirect(dsKey) + if err == nil { + out <- ListRes{dsKey, dataObj, 0} + } + } + }() + return out, nil +} + func verify(d *Datastore, key ds.Key, val *DataObj, level int) int { status := 0 _, err := d.GetData(key, val, level, true) diff --git a/test/sharness/t0260-filestore.sh b/test/sharness/t0260-filestore.sh index 2a81f333b788..0a80510e47b5 100755 --- a/test/sharness/t0260-filestore.sh +++ b/test/sharness/t0260-filestore.sh @@ -24,7 +24,7 @@ test_expect_success "fail after file move" ' # check "ipfs filestore " cmd by using state left by add commands -cat < ls_expect +cat < ls_expect_all QmQ8jJxa1Ts9fKsyUXcdYRHHUkuhJ69f82CF8BNX14ovLT QmQNcknfZjsABxg2bwxZQ9yqoUZW5dtAfCK3XY4eadjnxZ QmQnNhFzUjVRMHxafWaV2z7XZV8no9xJTdybMZbhgZ7776 @@ -49,7 +49,13 @@ QmfAGX7cH2G16Wb6tzVgVjwJtphCz3SeuRqvFmGuVY3C7D QmfYBbC153rBir5ECS2rzrKVUEer6rgqbRpriX2BviJHq1 EOF +cat < ls_expect +QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb +QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH +EOF + test_expect_success "testing filestore ls" ' + ipfs filestore ls -q -a | LC_ALL=C sort > ls_actual_all && ipfs filestore ls -q | LC_ALL=C sort > ls_actual && test_cmp ls_expect ls_actual ' @@ -62,7 +68,7 @@ test_expect_success "testing filestore verify" ' test_expect_success "tesing re-adding file after change" ' ipfs add --no-copy mountdir/hello.txt && - ipfs filestore ls -q | grep -q QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN + ipfs filestore ls -q -a | grep -q QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN ' cat < ls_expect @@ -72,7 +78,7 @@ EOF test_expect_success "tesing filestore clean invalid" ' ipfs filestore clean invalid > rm-invalid-output && - ipfs filestore ls -q | LC_ALL=C sort > ls_actual && + ipfs filestore ls -q -a | LC_ALL=C sort > ls_actual && test_cmp ls_expect ls_actual ' @@ -82,7 +88,7 @@ EOF test_expect_success "tesing filestore clean incomplete" ' ipfs filestore clean incomplete > rm-invalid-output && - ipfs filestore ls -q | LC_ALL=C sort > ls_actual && + ipfs filestore ls -q -a | LC_ALL=C sort > ls_actual && test_cmp ls_expect ls_actual '