From 32e09a67789811d4c0358ec7f174d28e8befe139 Mon Sep 17 00:00:00 2001
From: Kevin Atkinson <k@kevina.org>
Date: Thu, 12 Oct 2017 02:43:05 -0400
Subject: [PATCH] filestore util: Add 'filestore rm' command.
License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>
---
core/commands/filestore.go | 60 +++++++++++++++++
filestore/remove.go | 91 ++++++++++++++++++++++++++
test/sharness/t0271-filestore-utils.sh | 28 +++++++-
3 files changed, 176 insertions(+), 3 deletions(-)
create mode 100644 filestore/remove.go
diff --git a/core/commands/filestore.go b/core/commands/filestore.go
index a75df8918..bdb2aba2c 100644
--- a/core/commands/filestore.go
+++ b/core/commands/filestore.go
@@ -6,6 +6,8 @@ import (
"io"
"os"
+ bs "github.com/ipfs/go-ipfs/blocks/blockstore"
+ butil "github.com/ipfs/go-ipfs/blocks/blockstore/util"
oldCmds "github.com/ipfs/go-ipfs/commands"
"github.com/ipfs/go-ipfs/core"
e "github.com/ipfs/go-ipfs/core/commands/e"
@@ -22,6 +24,7 @@ var FileStoreCmd = &cmds.Command{
},
Subcommands: map[string]*cmds.Command{
"ls": lsFileStore,
+ "rm": rmFileStore,
},
OldSubcommands: map[string]*oldCmds.Command{
"verify": verifyFileStore,
@@ -236,6 +239,63 @@ var dupsFileStore = &oldCmds.Command{
Type: RefWrapper{},
}
+var rmFileStore = &cmds.Command{
+ Helptext: cmdkit.HelpText{
+ Tagline: "Remove IPFS block(s) from just the filestore or blockstore.",
+ ShortDescription: `
+Remove blocks from either the filestore or the main blockstore.
+`,
+ },
+ Arguments: []cmdkit.Argument{
+ cmdkit.StringArg("hash", true, true, "CID's of block(s) to remove."),
+ },
+ Options: []cmdkit.Option{
+ cmdkit.BoolOption("force", "f", "Ignore nonexistent blocks."),
+ cmdkit.BoolOption("quiet", "q", "Write minimal output."),
+ cmdkit.BoolOption("non-filestore", "Remove non-filestore blocks"),
+ },
+ Run: func(req cmds.Request, res cmds.ResponseEmitter) {
+ n, fs, err := getFilestore(req.InvocContext())
+ if err != nil {
+ res.SetError(err, cmdkit.ErrNormal)
+ return
+ }
+ hashes := req.Arguments()
+ force, _, _ := req.Option("force").Bool()
+ quiet, _, _ := req.Option("quiet").Bool()
+ nonFilestore, _, _ := req.Option("non-filestore").Bool()
+ prefix := filestore.FilestorePrefix.String()
+ if nonFilestore {
+ prefix = bs.BlockPrefix.String()
+ }
+ cids := make([]*cid.Cid, 0, len(hashes))
+ for _, hash := range hashes {
+ c, err := cid.Decode(hash)
+ if err != nil {
+ res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmdkit.ErrNormal)
+ return
+ }
+
+ cids = append(cids, c)
+ }
+ ch, err := filestore.RmBlocks(fs, n.Blockstore, n.Pinning, cids, butil.RmBlocksOpts{
+ Prefix: prefix,
+ Quiet: quiet,
+ Force: force,
+ })
+ if err != nil {
+ res.SetError(err, cmdkit.ErrNormal)
+ return
+ }
+ err = res.Emit(ch)
+ if err != nil {
+ log.Error(err)
+ }
+ },
+ PostRun: blockRmCmd.PostRun,
+ Type: butil.RemovedBlock{},
+}
+
type getNoder interface {
GetNode() (*core.IpfsNode, error)
}
diff --git a/filestore/remove.go b/filestore/remove.go
new file mode 100644
index 000000000..594bc97a4
--- /dev/null
+++ b/filestore/remove.go
@@ -0,0 +1,91 @@
+package filestore
+
+import (
+ "fmt"
+
+ bs "github.com/ipfs/go-ipfs/blocks/blockstore"
+ u "github.com/ipfs/go-ipfs/blocks/blockstore/util"
+ "github.com/ipfs/go-ipfs/pin"
+
+ cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
+ ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
+)
+
+// RmBlocks removes blocks from either the filestore or the
+// blockstore. It is similar to blockstore_util.RmBlocks but allows
+// the removal of pinned block from one store if it is also in the
+// other.
+func RmBlocks(fs *Filestore, lock bs.GCLocker, pins pin.Pinner, cids []*cid.Cid, opts u.RmBlocksOpts) (<-chan interface{}, error) {
+ // make the channel large enough to hold any result to avoid
+ // blocking while holding the GCLock
+ out := make(chan interface{}, len(cids))
+
+ var blocks deleter
+ switch opts.Prefix {
+ case FilestorePrefix.String():
+ blocks = fs.fm
+ case bs.BlockPrefix.String():
+ blocks = fs.bs
+ default:
+ return nil, fmt.Errorf("unknown prefix: %s", opts.Prefix)
+ }
+
+ go func() {
+ defer close(out)
+
+ unlocker := lock.GCLock()
+ defer unlocker.Unlock()
+
+ stillOkay := filterPinned(fs, pins, out, cids, blocks)
+
+ for _, c := range stillOkay {
+ err := blocks.DeleteBlock(c)
+ if err != nil && opts.Force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
+ // ignore non-existent blocks
+ } else if err != nil {
+ out <- &u.RemovedBlock{Hash: c.String(), Error: err.Error()}
+ } else if !opts.Quiet {
+ out <- &u.RemovedBlock{Hash: c.String()}
+ }
+ }
+ }()
+ return out, nil
+}
+
+type deleter interface {
+ DeleteBlock(c *cid.Cid) error
+}
+
+func filterPinned(fs *Filestore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, foundIn deleter) []*cid.Cid {
+ stillOkay := make([]*cid.Cid, 0, len(cids))
+ res, err := pins.CheckIfPinned(cids...)
+ if err != nil {
+ out <- &u.RemovedBlock{Error: fmt.Sprintf("pin check failed: %s", err)}
+ return nil
+ }
+ for _, r := range res {
+ if !r.Pinned() || availableElsewhere(fs, foundIn, r.Key) {
+ stillOkay = append(stillOkay, r.Key)
+ } else {
+ out <- &u.RemovedBlock{
+ Hash: r.Key.String(),
+ Error: r.String(),
+ }
+ }
+ }
+ return stillOkay
+}
+
+func availableElsewhere(fs *Filestore, foundIn deleter, c *cid.Cid) bool {
+ switch {
+ case fs.fm == foundIn:
+ have, _ := fs.bs.Has(c)
+ return have
+ case fs.bs == foundIn:
+ have, _ := fs.fm.Has(c)
+ return have
+ default:
+ // programmer error
+ panic("invalid pointer for foundIn")
+ }
+}
diff --git a/test/sharness/t0271-filestore-utils.sh b/test/sharness/t0271-filestore-utils.sh
index c93f51e9a..9d776df41 100755
--- a/test/sharness/t0271-filestore-utils.sh
+++ b/test/sharness/t0271-filestore-utils.sh
@@ -153,7 +153,7 @@ test_filestore_verify() {
test_init_dataset
}
-test_filestore_dups() {
+test_filestore_dups_and_rm() {
# make sure the filestore is in a clean state
test_filestore_state
@@ -163,6 +163,28 @@ test_filestore_dups() {
echo "$FILE1_HASH" > dups_expect
test_cmp dups_expect dups_actual
'
+
+ test_expect_success "remove non-filestore block of dup ok" '
+ ipfs filestore rm --non-filestore $FILE1_HASH &&
+ ipfs filestore dups > dups_actual &&
+ test_cmp /dev/null dups_actual
+ '
+
+ test_expect_success "block still in filestore" '
+ ipfs filestore ls $FILE1_HASH | grep -q file1
+ '
+
+ test_expect_success "remove non-duplicate pinned block not ok" '
+ test_must_fail ipfs filestore rm $FILE1_HASH 2>&1 | tee rm_err &&
+ grep -q pinned rm_err
+ '
+
+ test_expect_success "remove filestore block of dup ok" '
+ ipfs add --raw-leaves somedir/file1 &&
+ ipfs filestore rm $FILE1_HASH &&
+ ipfs filestore dups > dups_actual &&
+ test_cmp /dev/null dups_actual
+ '
}
#
@@ -175,7 +197,7 @@ test_filestore_adds
test_filestore_verify
-test_filestore_dups
+test_filestore_dups_and_rm
#
# With daemon
@@ -191,7 +213,7 @@ test_filestore_adds
test_filestore_verify
-test_filestore_dups
+test_filestore_dups_and_rm
test_kill_ipfs_daemon
--
GitLab