Skip to content
Snippets Groups Projects
Commit 32e09a67 authored by Kevin Atkinson's avatar Kevin Atkinson
Browse files

filestore util: Add 'filestore rm' command.


License: MIT
Signed-off-by: default avatarKevin Atkinson <k@kevina.org>
parent 96b87238
No related branches found
No related tags found
No related merge requests found
......@@ -6,6 +6,8 @@ import (
"io"
"os"
bs "github.com/ipfs/go-ipfs/blocks/blockstore"
butil "github.com/ipfs/go-ipfs/blocks/blockstore/util"
oldCmds "github.com/ipfs/go-ipfs/commands"
"github.com/ipfs/go-ipfs/core"
e "github.com/ipfs/go-ipfs/core/commands/e"
......@@ -22,6 +24,7 @@ var FileStoreCmd = &cmds.Command{
},
Subcommands: map[string]*cmds.Command{
"ls": lsFileStore,
"rm": rmFileStore,
},
OldSubcommands: map[string]*oldCmds.Command{
"verify": verifyFileStore,
......@@ -236,6 +239,63 @@ var dupsFileStore = &oldCmds.Command{
Type: RefWrapper{},
}
var rmFileStore = &cmds.Command{
Helptext: cmdkit.HelpText{
Tagline: "Remove IPFS block(s) from just the filestore or blockstore.",
ShortDescription: `
Remove blocks from either the filestore or the main blockstore.
`,
},
Arguments: []cmdkit.Argument{
cmdkit.StringArg("hash", true, true, "CID's of block(s) to remove."),
},
Options: []cmdkit.Option{
cmdkit.BoolOption("force", "f", "Ignore nonexistent blocks."),
cmdkit.BoolOption("quiet", "q", "Write minimal output."),
cmdkit.BoolOption("non-filestore", "Remove non-filestore blocks"),
},
Run: func(req cmds.Request, res cmds.ResponseEmitter) {
n, fs, err := getFilestore(req.InvocContext())
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}
hashes := req.Arguments()
force, _, _ := req.Option("force").Bool()
quiet, _, _ := req.Option("quiet").Bool()
nonFilestore, _, _ := req.Option("non-filestore").Bool()
prefix := filestore.FilestorePrefix.String()
if nonFilestore {
prefix = bs.BlockPrefix.String()
}
cids := make([]*cid.Cid, 0, len(hashes))
for _, hash := range hashes {
c, err := cid.Decode(hash)
if err != nil {
res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmdkit.ErrNormal)
return
}
cids = append(cids, c)
}
ch, err := filestore.RmBlocks(fs, n.Blockstore, n.Pinning, cids, butil.RmBlocksOpts{
Prefix: prefix,
Quiet: quiet,
Force: force,
})
if err != nil {
res.SetError(err, cmdkit.ErrNormal)
return
}
err = res.Emit(ch)
if err != nil {
log.Error(err)
}
},
PostRun: blockRmCmd.PostRun,
Type: butil.RemovedBlock{},
}
type getNoder interface {
GetNode() (*core.IpfsNode, error)
}
......
package filestore
import (
"fmt"
bs "github.com/ipfs/go-ipfs/blocks/blockstore"
u "github.com/ipfs/go-ipfs/blocks/blockstore/util"
"github.com/ipfs/go-ipfs/pin"
cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
)
// RmBlocks removes blocks from either the filestore or the
// blockstore. It is similar to blockstore_util.RmBlocks but allows
// the removal of pinned block from one store if it is also in the
// other.
func RmBlocks(fs *Filestore, lock bs.GCLocker, pins pin.Pinner, cids []*cid.Cid, opts u.RmBlocksOpts) (<-chan interface{}, error) {
// make the channel large enough to hold any result to avoid
// blocking while holding the GCLock
out := make(chan interface{}, len(cids))
var blocks deleter
switch opts.Prefix {
case FilestorePrefix.String():
blocks = fs.fm
case bs.BlockPrefix.String():
blocks = fs.bs
default:
return nil, fmt.Errorf("unknown prefix: %s", opts.Prefix)
}
go func() {
defer close(out)
unlocker := lock.GCLock()
defer unlocker.Unlock()
stillOkay := filterPinned(fs, pins, out, cids, blocks)
for _, c := range stillOkay {
err := blocks.DeleteBlock(c)
if err != nil && opts.Force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
// ignore non-existent blocks
} else if err != nil {
out <- &u.RemovedBlock{Hash: c.String(), Error: err.Error()}
} else if !opts.Quiet {
out <- &u.RemovedBlock{Hash: c.String()}
}
}
}()
return out, nil
}
type deleter interface {
DeleteBlock(c *cid.Cid) error
}
func filterPinned(fs *Filestore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, foundIn deleter) []*cid.Cid {
stillOkay := make([]*cid.Cid, 0, len(cids))
res, err := pins.CheckIfPinned(cids...)
if err != nil {
out <- &u.RemovedBlock{Error: fmt.Sprintf("pin check failed: %s", err)}
return nil
}
for _, r := range res {
if !r.Pinned() || availableElsewhere(fs, foundIn, r.Key) {
stillOkay = append(stillOkay, r.Key)
} else {
out <- &u.RemovedBlock{
Hash: r.Key.String(),
Error: r.String(),
}
}
}
return stillOkay
}
func availableElsewhere(fs *Filestore, foundIn deleter, c *cid.Cid) bool {
switch {
case fs.fm == foundIn:
have, _ := fs.bs.Has(c)
return have
case fs.bs == foundIn:
have, _ := fs.fm.Has(c)
return have
default:
// programmer error
panic("invalid pointer for foundIn")
}
}
......@@ -153,7 +153,7 @@ test_filestore_verify() {
test_init_dataset
}
test_filestore_dups() {
test_filestore_dups_and_rm() {
# make sure the filestore is in a clean state
test_filestore_state
......@@ -163,6 +163,28 @@ test_filestore_dups() {
echo "$FILE1_HASH" > dups_expect
test_cmp dups_expect dups_actual
'
test_expect_success "remove non-filestore block of dup ok" '
ipfs filestore rm --non-filestore $FILE1_HASH &&
ipfs filestore dups > dups_actual &&
test_cmp /dev/null dups_actual
'
test_expect_success "block still in filestore" '
ipfs filestore ls $FILE1_HASH | grep -q file1
'
test_expect_success "remove non-duplicate pinned block not ok" '
test_must_fail ipfs filestore rm $FILE1_HASH 2>&1 | tee rm_err &&
grep -q pinned rm_err
'
test_expect_success "remove filestore block of dup ok" '
ipfs add --raw-leaves somedir/file1 &&
ipfs filestore rm $FILE1_HASH &&
ipfs filestore dups > dups_actual &&
test_cmp /dev/null dups_actual
'
}
#
......@@ -175,7 +197,7 @@ test_filestore_adds
test_filestore_verify
test_filestore_dups
test_filestore_dups_and_rm
#
# With daemon
......@@ -191,7 +213,7 @@ test_filestore_adds
test_filestore_verify
test_filestore_dups
test_filestore_dups_and_rm
test_kill_ipfs_daemon
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment