...
 
FROM golang:1.13-alpine as builder
WORKDIR /home
ENV GOPATH /go
ENV CGO_ENABLED 0
ENV GO111MODULE on
RUN \
apk add --no-cache git 'curl>7.61.0' && \
git clone https://github.com/minio/minio && \
curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
FROM arm32v7/alpine:3.10
LABEL maintainer="MinIO Inc <dev@min.io>"
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
COPY CREDITS /third_party/
COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
MINIO_SECRET_KEY_FILE=secret_key \
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh
EXPOSE 9000
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
CMD ["minio"]
FROM golang:1.13-alpine as builder
WORKDIR /home
ENV GOPATH /go
ENV CGO_ENABLED 0
ENV GO111MODULE on
RUN \
apk add --no-cache git 'curl>7.61.0' && \
git clone https://github.com/minio/minio && \
curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static .
FROM arm64v8/alpine:3.10
LABEL maintainer="MinIO Inc <dev@min.io>"
COPY dockerscripts/docker-entrypoint.sh /usr/bin/
COPY CREDITS /third_party/
COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
ENV MINIO_UPDATE off
ENV MINIO_ACCESS_KEY_FILE=access_key \
MINIO_SECRET_KEY_FILE=secret_key \
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
RUN \
apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
chmod +x /usr/bin/minio && \
chmod +x /usr/bin/docker-entrypoint.sh
EXPOSE 9000
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
VOLUME ["/data"]
CMD ["minio"]
......@@ -1058,15 +1058,18 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
// is a leaf or non-leaf entry.
func (fs *FSObjects) listDirFactory() ListDirFunc {
// listDir - lists all the entries at a given prefix and given entry in the prefix.
listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
var err error
entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
if err != nil && err != errFileNotFound {
logger.LogIf(context.Background(), err)
return
return false, nil
}
if len(entries) == 0 {
return true, nil
}
sort.Strings(entries)
return filterMatchingPrefix(entries, prefixEntry)
return false, filterMatchingPrefix(entries, prefixEntry)
}
// Return list factory instance.
......
......@@ -326,7 +326,7 @@ func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketIn
func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
// listDir - lists all the entries at a given prefix and given entry in the prefix.
listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, prefixDir))
if err != nil {
if os.IsNotExist(err) {
......@@ -341,6 +341,9 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
logger.LogIf(context.Background(), err)
return
}
if len(fis) == 0 {
return true, nil
}
for _, fi := range fis {
if fi.IsDir() {
entries = append(entries, fi.Name()+hdfsSeparator)
......@@ -348,7 +351,7 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
entries = append(entries, fi.Name())
}
}
return minio.FilterMatchingPrefix(entries, prefixEntry)
return false, minio.FilterMatchingPrefix(entries, prefixEntry)
}
// Return list factory instance.
......
......@@ -45,6 +45,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
// Will not store any objects in this bucket,
// Its to test ListObjects on an empty bucket.
"empty-bucket",
// Listing the case where the marker > last object.
"test-bucket-single-object",
}
for _, bucket := range testBuckets {
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
......@@ -72,6 +74,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
{testBuckets[1], "obj1", "obj1", nil},
{testBuckets[1], "obj2", "obj2", nil},
{testBuckets[1], "temporary/0/", "", nil},
{testBuckets[3], "A/B", "contentstring", nil},
}
for _, object := range testObjects {
md5Bytes := md5.Sum([]byte(object.content))
......@@ -445,6 +448,11 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
{Name: "temporary/0/"},
},
},
// ListObjectsResult-34 Listing with marker > last object should return empty
{
IsTruncated: false,
Objects: []ObjectInfo{},
},
}
testCases := []struct {
......@@ -559,6 +567,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
{"test-bucket-empty-dir", "", "", SlashSeparator, 10, resultCases[32], nil, true},
// Test listing a directory which contains an empty directory (64)
{"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true},
// Test listing with marker > last object such that response should be empty (65)
{"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true},
}
for i, testCase := range testCases {
......
......@@ -669,13 +669,16 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
ch = make(chan FileInfo)
go func() {
defer close(ch)
listDir := func(volume, dirPath, dirEntry string) (entries []string) {
listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) {
entries, err := s.ListDir(volume, dirPath, -1, leafFile)
if err != nil {
return
}
if len(entries) == 0 {
return true, nil
}
sort.Strings(entries)
return filterMatchingPrefix(entries, dirEntry)
return false, filterMatchingPrefix(entries, dirEntry)
}
walkResultCh := startTreeWalk(context.Background(), volume, dirPath, marker, recursive, listDir, endWalkCh)
......
......@@ -56,10 +56,10 @@ func filterMatchingPrefix(entries []string, prefixEntry string) []string {
}
// ListDirFunc - "listDir" function of type listDirFunc returned by listDirFactory() - explained below.
type ListDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string)
type ListDirFunc func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string)
// treeWalk walks directory tree recursively pushing TreeWalkResult into the channel as and when it encounters files.
func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir ListDirFunc, resultCh chan TreeWalkResult, endWalkCh <-chan struct{}, isEnd bool) (totalNum int, treeErr error) {
func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir ListDirFunc, resultCh chan TreeWalkResult, endWalkCh <-chan struct{}, isEnd bool) (emptyDir bool, treeErr error) {
// Example:
// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
// called with prefixDir="one/two/three/four/" and marker="five.txt"
......@@ -75,10 +75,10 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
}
}
entries := listDir(bucket, prefixDir, entryPrefixMatch)
emptyDir, entries := listDir(bucket, prefixDir, entryPrefixMatch)
// For an empty list return right here.
if len(entries) == 0 {
return 0, nil
if emptyDir {
return true, nil
}
// example:
......@@ -90,7 +90,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
entries = entries[idx:]
// For an empty list after search through the entries, return right here.
if len(entries) == 0 {
return 0, nil
return false, nil
}
for i, entry := range entries {
......@@ -123,16 +123,16 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
// markIsEnd is passed to this entry's treeWalk() so that treeWalker.end can be marked
// true at the end of the treeWalk stream.
markIsEnd := i == len(entries)-1 && isEnd
totalFound, err := doTreeWalk(ctx, bucket, pentry, prefixMatch, markerArg, recursive,
emptyDir, err := doTreeWalk(ctx, bucket, pentry, prefixMatch, markerArg, recursive,
listDir, resultCh, endWalkCh, markIsEnd)
if err != nil {
return 0, err
return false, err
}
// A nil totalFound means this is an empty directory that
// needs to be sent to the result channel, otherwise continue
// to the next entry.
if totalFound > 0 {
if !emptyDir {
continue
}
}
......@@ -141,13 +141,13 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
isEOF := ((i == len(entries)-1) && isEnd)
select {
case <-endWalkCh:
return 0, errWalkAbort
return false, errWalkAbort
case resultCh <- TreeWalkResult{entry: pentry, end: isEOF}:
}
}
// Everything is listed.
return len(entries), nil
return false, nil
}
// Initiate a new treeWalk in a goroutine.
......
......@@ -260,7 +260,7 @@ func TestListDir(t *testing.T) {
}
// Should list "file1" from fsDir1.
entries := listDir(volume, "", "")
_, entries := listDir(volume, "", "")
if len(entries) != 2 {
t.Fatal("Expected the number of entries to be 2")
}
......@@ -278,7 +278,7 @@ func TestListDir(t *testing.T) {
}
// Should list "file2" from fsDir2.
entries = listDir(volume, "", "")
_, entries = listDir(volume, "", "")
if len(entries) != 1 {
t.Fatal("Expected the number of entries to be 1")
}
......
......@@ -25,7 +25,7 @@ import (
// disks - used for doing disk.ListDir()
func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
// Returns sorted merged entries from all the disks.
listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string) {
listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, mergedEntries []string) {
for _, disk := range disks {
if disk == nil {
continue
......@@ -38,6 +38,10 @@ func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
continue
}
if len(entries) == 0 {
return true, nil
}
// Find elements in entries which are not in mergedEntries
for _, entry := range entries {
idx := sort.SearchStrings(mergedEntries, entry)
......@@ -54,7 +58,7 @@ func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
sort.Strings(mergedEntries)
}
}
return filterMatchingPrefix(mergedEntries, prefixEntry)
return false, filterMatchingPrefix(mergedEntries, prefixEntry)
}
return listDir
}
......
......@@ -5,7 +5,7 @@ version: '3.7'
# 9001 through 9004.
services:
minio1:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
volumes:
- data1-1:/data1
- data1-2:/data2
......@@ -22,7 +22,7 @@ services:
retries: 3
minio2:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
volumes:
- data2-1:/data1
- data2-2:/data2
......@@ -39,7 +39,7 @@ services:
retries: 3
minio3:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
volumes:
- data3-1:/data1
- data3-2:/data2
......@@ -56,7 +56,7 @@ services:
retries: 3
minio4:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
volumes:
- data4-1:/data1
- data4-2:/data2
......
......@@ -2,7 +2,7 @@ version: '3.7'
services:
minio1:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio1
volumes:
- minio1-data:/export
......@@ -29,7 +29,7 @@ services:
retries: 3
minio2:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio2
volumes:
- minio2-data:/export
......@@ -56,7 +56,7 @@ services:
retries: 3
minio3:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio3
volumes:
- minio3-data:/export
......@@ -83,7 +83,7 @@ services:
retries: 3
minio4:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio4
volumes:
- minio4-data:/export
......
......@@ -2,7 +2,7 @@ version: '3.7'
services:
minio1:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio1
volumes:
- minio1-data:/export
......@@ -33,7 +33,7 @@ services:
retries: 3
minio2:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio2
volumes:
- minio2-data:/export
......@@ -64,7 +64,7 @@ services:
retries: 3
minio3:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio3
volumes:
- minio3-data:/export
......@@ -95,7 +95,7 @@ services:
retries: 3
minio4:
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
hostname: minio4
volumes:
- minio4-data:/export
......
......@@ -30,7 +30,7 @@ spec:
value: "minio"
- name: MINIO_SECRET_KEY
value: "minio123"
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
# Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
args:
- server
......
......@@ -22,7 +22,7 @@ spec:
value: "minio"
- name: MINIO_SECRET_KEY
value: "minio123"
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
args:
- server
- http://minio-{0...3}.minio.default.svc.cluster.local/data
......
......@@ -24,7 +24,7 @@ spec:
containers:
- name: minio
# Pulls the default Minio image from Docker Hub
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
args:
- gateway
- gcs
......
......@@ -32,7 +32,7 @@ spec:
- name: data
mountPath: "/data"
# Pulls the lastest Minio image from Docker Hub
image: minio/minio:RELEASE.2020-03-09T18-26-53Z
image: minio/minio:RELEASE.2020-03-14T02-21-58Z
args:
- server
- /data
......
......@@ -39,15 +39,19 @@ $ go run docs/sts/web-identity.go -cid account -csec 072e7f00-4289-469c-9ab2-bbe
2018/12/26 17:49:36 listening on http://localhost:8888/
```
This will open the login page of keycloak, upon successful login, STS credentials will be printed on the screen, for example
This will open the login page of keycloak, upon successful login, STS credentials along with any buckets discovered using the credentials will be printed on the screen, for example:
```
##### Credentials
{
"accessKey": "6N2BALX7ELO827DXS3GK",
"secretKey": "23JKqAD+um8ObHqzfIh+bfqwG9V8qs9tFY6MqeFR",
"expiration": "2019-10-01T07:22:34Z",
"sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI2TjJCQUxYN0VMTzgyN0RYUzNHSyIsImFjciI6IjAiLCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjoxNTY5OTEwNTUyLCJhenAiOiJhY2NvdW50IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJleHAiOjE1Njk5MTQ1NTQsImlhdCI6MTU2OTkxMDk1NCwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgxL2F1dGgvcmVhbG1zL2RlbW8iLCJqdGkiOiJkOTk4YTBlZS01NDk2LTQ4OWYtYWJlMi00ZWE5MjJiZDlhYWYiLCJuYmYiOjAsInBvbGljeSI6InJlYWR3cml0ZSIsInByZWZlcnJlZF91c2VybmFtZSI6Im5ld3VzZXIxIiwic2Vzc2lvbl9zdGF0ZSI6IjJiYTAyYTI2LWE5MTUtNDUxNC04M2M1LWE0YjgwYjc4ZTgxNyIsInN1YiI6IjY4ZmMzODVhLTA5MjItNGQyMS04N2U5LTZkZTdhYjA3Njc2NSIsInR5cCI6IklEIn0._UG_-ZHgwdRnsp0gFdwChb7VlbPs-Gr_RNUz9EV7TggCD59qjCFAKjNrVHfOSVkKvYEMe0PvwfRKjnJl3A_mBA"
"buckets": [
"bucket-x"
],
"credentials": {
"AccessKeyID": "6N2BALX7ELO827DXS3GK",
"SecretAccessKey": "23JKqAD+um8ObHqzfIh+bfqwG9V8qs9tFY6MqeFR+xxx",
"SessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI2TjJCQUxYN0VMTzgyN0RYUzNHSyIsImFjciI6IjAiLCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjoxNTY5OTEwNTUyLCJhenAiOiJhY2NvdW50IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJleHAiOjE1Njk5MTQ1NTQsImlhdCI6MTU2OTkxMDk1NCwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgxL2F1dGgvcmVhbG1zL2RlbW8iLCJqdGkiOiJkOTk4YTBlZS01NDk2LTQ4OWYtYWJlMi00ZWE5MjJiZDlhYWYiLCJuYmYiOjAsInBvbGljeSI6InJlYWR3cml0ZSIsInByZWZlcnJlZF91c2VybmFtZSI6Im5ld3VzZXIxIiwic2Vzc2lvbl9zdGF0ZSI6IjJiYTAyYTI2LWE5MTUtNDUxNC04M2M1LWE0YjgwYjc4ZTgxNyIsInN1YiI6IjY4ZmMzODVhLTA5MjItNGQyMS04N2U5LTZkZTdhYjA3Njc2NSIsInR5cCI6IklEIn0._UG_-ZHgwdRnsp0gFdwChb7VlbPs-Gr_RNUz9EV7TggCD59qjCFAKjNrVHfOSVkKvYEMe0PvwfRKjnJl3A_mBA"",
"SignerType": 1
}
}
```
......
......@@ -143,6 +143,7 @@ func main() {
ddoc, err := parseDiscoveryDoc(configEndpoint)
if err != nil {
log.Println(fmt.Errorf("Failed to parse OIDC discovery document %s", err))
fmt.Println(err)
return
}
......@@ -163,10 +164,16 @@ func main() {
state := randomState()
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s", r.Method, r.RequestURI)
if r.RequestURI != "/" {
http.NotFound(w, r)
return
}
http.Redirect(w, r, config.AuthCodeURL(state), http.StatusFound)
})
http.HandleFunc("/oauth2/callback", func(w http.ResponseWriter, r *http.Request) {
log.Printf("%s %s", r.Method, r.RequestURI)
if r.URL.Query().Get("state") != state {
http.Error(w, "state did not match", http.StatusBadRequest)
return
......@@ -189,13 +196,11 @@ func main() {
sts, err := credentials.NewSTSWebIdentity(stsEndpoint, getWebTokenExpiry)
if err != nil {
log.Println(fmt.Errorf("Could not get STS credentials: %s", err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Uncomment this to use MinIO API operations by initializing minio
// client with obtained credentials.
opts := &minio.Options{
Creds: sts,
BucketLookup: minio.BucketLookupAuto,
......@@ -203,23 +208,40 @@ func main() {
u, err := url.Parse(stsEndpoint)
if err != nil {
log.Println(fmt.Errorf("Failed to parse STS Endpoint: %s", err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
clnt, err := minio.NewWithOptions(u.Host, opts)
if err != nil {
log.Println(fmt.Errorf("Error while initializing Minio client, %s", err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
buckets, err := clnt.ListBuckets()
if err != nil {
log.Println(fmt.Errorf("Error while listing buckets, %s", err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
creds, _ := sts.Get()
bucketNames := []string{}
for _, bucket := range buckets {
log.Println(bucket)
log.Println(fmt.Sprintf("Bucket discovered: %s", bucket.Name))
bucketNames = append(bucketNames, bucket.Name)
}
response := make(map[string]interface{})
response["credentials"] = creds
response["buckets"] = bucketNames
c, err := json.MarshalIndent(response, "", "\t")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Write(c)
})
address := fmt.Sprintf("localhost:%v", port)
......
......@@ -74,6 +74,9 @@ RUN build/worm/install.sh
COPY build/healthcheck /mint/build/healthcheck
RUN build/healthcheck/install.sh
COPY build/s3select /mint/build/s3select
RUN build/s3select/install.sh
COPY remove-packages.list /mint
COPY postinstall.sh /mint
RUN /mint/postinstall.sh
......
#!/bin/bash -e
#
# Mint (C) 2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
python -m pip install minio
## `s3select` tests
This directory serves as the location for Mint tests for s3select features. Top level `mint.sh` calls `run.sh` to execute tests.
## Adding new tests
New tests are added into `s3select/tests.py` as new functions.
## Running tests manually
- Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`
- Call `run.sh` with output log file and error log file. for example
```bash
export MINT_DATA_DIR=~/my-mint-dir
export MINT_MODE=core
export SERVER_ENDPOINT="play.min.io"
export ACCESS_KEY="Q3AM3UQ867SPQQA43P2F"
export SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
export ENABLE_HTTPS=1
export SERVER_REGION=us-east-1
./run.sh /tmp/output.log /tmp/error.log
```
#!/bin/bash
#
# Mint (C) 2020 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# handle command line arguments
if [ $# -ne 2 ]; then
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
exit -1
fi
output_log_file="$1"
error_log_file="$2"
# run path style tests
python "./tests.py" 1>>"$output_log_file" 2>"$error_log_file"
This diff is collapsed.
......@@ -113,6 +113,9 @@ type Reader struct {
// or the Unicode replacement character (0xFFFD).
Comma rune
// Quote is the single character used for marking fields limits
Quote []rune
// Comment, if not 0, is the comment character. Lines beginning with the
// Comment character without preceding whitespace are ignored.
// With leading whitespace the Comment character becomes part of the
......@@ -171,6 +174,7 @@ type Reader struct {
func NewReader(r io.Reader) *Reader {
return &Reader{
Comma: ',',
Quote: []rune(`"`),
r: bufio.NewReader(r),
}
}
......@@ -255,6 +259,13 @@ func nextRune(b []byte) rune {
return r
}
func encodeRune(r rune) []byte {
rlen := utf8.RuneLen(r)
p := make([]byte, rlen)
_ = utf8.EncodeRune(p, r)
return p
}
func (r *Reader) readRecord(dst []string) ([]string, error) {
if r.Comma == r.Comment || !validDelim(r.Comma) || (r.Comment != 0 && !validDelim(r.Comment)) {
return nil, errInvalidDelim
......@@ -280,9 +291,17 @@ func (r *Reader) readRecord(dst []string) ([]string, error) {
return nil, errRead
}
var quote rune
var quoteLen int
if len(r.Quote) > 0 {
quote = r.Quote[0]
quoteLen = utf8.RuneLen(quote)
}
encodedQuote := encodeRune(quote)
// Parse each field in the record.
var err error
const quoteLen = len(`"`)
commaLen := utf8.RuneLen(r.Comma)
recLine := r.numLine // Starting line for record
r.recordBuffer = r.recordBuffer[:0]
......@@ -292,7 +311,7 @@ parseField:
if r.TrimLeadingSpace {
line = bytes.TrimLeftFunc(line, unicode.IsSpace)
}
if len(line) == 0 || line[0] != '"' {
if len(line) == 0 || quoteLen == 0 || nextRune(line) != quote {
// Non-quoted string field
i := bytes.IndexRune(line, r.Comma)
field := line
......@@ -303,7 +322,7 @@ parseField:
}
// Check to make sure a quote does not appear in field.
if !r.LazyQuotes {
if j := bytes.IndexByte(field, '"'); j >= 0 {
if j := bytes.IndexRune(field, quote); j >= 0 {
col := utf8.RuneCount(fullLine[:len(fullLine)-len(line[j:])])
err = &ParseError{StartLine: recLine, Line: r.numLine, Column: col, Err: ErrBareQuote}
break parseField
......@@ -320,15 +339,15 @@ parseField:
// Quoted string field
line = line[quoteLen:]
for {
i := bytes.IndexByte(line, '"')
i := bytes.IndexRune(line, quote)
if i >= 0 {
// Hit next quote.
r.recordBuffer = append(r.recordBuffer, line[:i]...)
line = line[i+quoteLen:]
switch rn := nextRune(line); {
case rn == '"':
case rn == quote:
// `""` sequence (append quote).
r.recordBuffer = append(r.recordBuffer, '"')
r.recordBuffer = append(r.recordBuffer, encodedQuote...)
line = line[quoteLen:]
case rn == r.Comma:
// `",` sequence (end of field).
......@@ -341,7 +360,7 @@ parseField:
break parseField
case r.LazyQuotes:
// `"` sequence (bare quote).
r.recordBuffer = append(r.recordBuffer, '"')
r.recordBuffer = append(r.recordBuffer, encodedQuote...)
default:
// `"*` sequence (invalid non-escaped quote).
col := utf8.RuneCount(fullLine[:len(fullLine)-len(line)-quoteLen])
......
......@@ -28,15 +28,18 @@ import (
// the underlying io.Writer. Any errors that occurred should
// be checked by calling the Error method.
type Writer struct {
Comma rune // Field delimiter (set to ',' by NewWriter)
UseCRLF bool // True to use \r\n as the line terminator
w *bufio.Writer
Comma rune // Field delimiter (set to ',' by NewWriter)
Quote rune // Fields quote character
AlwaysQuote bool // True to quote all fields
UseCRLF bool // True to use \r\n as the line terminator
w *bufio.Writer
}
// NewWriter returns a new Writer that writes to w.
func NewWriter(w io.Writer) *Writer {
return &Writer{
Comma: ',',
Quote: '"',
w: bufio.NewWriter(w),
}
}
......@@ -59,19 +62,22 @@ func (w *Writer) Write(record []string) error {
// If we don't have to have a quoted field then just
// write out the field and continue to the next field.
if !w.fieldNeedsQuotes(field) {
if !w.AlwaysQuote && !w.fieldNeedsQuotes(field) {
if _, err := w.w.WriteString(field); err != nil {
return err
}
continue
}
if err := w.w.WriteByte('"'); err != nil {
if _, err := w.w.WriteRune(w.Quote); err != nil {
return err
}
specialChars := "\r\n" + string(w.Quote)
for len(field) > 0 {
// Search for special characters.
i := strings.IndexAny(field, "\"\r\n")
i := strings.IndexAny(field, specialChars)
if i < 0 {
i = len(field)
}
......@@ -85,9 +91,13 @@ func (w *Writer) Write(record []string) error {
// Encode the special character.
if len(field) > 0 {
var err error
switch field[0] {
case '"':
_, err = w.w.WriteString(`""`)
switch nextRune([]byte(field)) {
case w.Quote:
_, err = w.w.WriteRune(w.Quote)
if err != nil {
break
}
_, err = w.w.WriteRune(w.Quote)
case '\r':
if !w.UseCRLF {
err = w.w.WriteByte('\r')
......@@ -105,7 +115,7 @@ func (w *Writer) Write(record []string) error {
}
}
}
if err := w.w.WriteByte('"'); err != nil {
if _, err := w.w.WriteRune(w.Quote); err != nil {
return err
}
}
......@@ -158,7 +168,7 @@ func (w *Writer) fieldNeedsQuotes(field string) bool {
if field == "" {
return false
}
if field == `\.` || strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") {
if field == `\.` || strings.ContainsAny(field, "\r\n"+string(w.Quote)+string(w.Comma)) {
return true
}
......
......@@ -11,11 +11,13 @@ import (
)
var writeTests = []struct {
Input [][]string
Output string
Error error
UseCRLF bool
Comma rune
Input [][]string
Output string
Error error
UseCRLF bool
Comma rune
Quote rune
AlwaysQuote bool
}{
{Input: [][]string{{"abc"}}, Output: "abc\n"},
{Input: [][]string{{"abc"}}, Output: "abc\r\n", UseCRLF: true},
......@@ -46,6 +48,7 @@ var writeTests = []struct {
{Input: [][]string{{"a", "a", ""}}, Output: "a|a|\n", Comma: '|'},
{Input: [][]string{{",", ",", ""}}, Output: ",|,|\n", Comma: '|'},
{Input: [][]string{{"foo"}}, Comma: '"', Error: errInvalidDelim},
{Input: [][]string{{"a", "a", ""}}, Quote: '"', AlwaysQuote: true, Output: "\"a\"|\"a\"|\"\"\n", Comma: '|'},
}
func TestWrite(t *testing.T) {
......@@ -56,6 +59,10 @@ func TestWrite(t *testing.T) {
if tt.Comma != 0 {
f.Comma = tt.Comma
}
if tt.Quote != 0 {
f.Quote = tt.Quote
}
f.AlwaysQuote = tt.AlwaysQuote
err := f.WriteAll(tt.Input)
if err != tt.Error {
t.Errorf("Unexpected error:\ngot %v\nwant %v", err, tt.Error)
......
......@@ -18,8 +18,11 @@ package csv
import (
"encoding/xml"
"errors"
"fmt"
"io"
"strings"
"unicode/utf8"
)
const (
......@@ -55,68 +58,64 @@ func (args *ReaderArgs) IsEmpty() bool {
}
// UnmarshalXML - decodes XML data.
func (args *ReaderArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type subReaderArgs ReaderArgs
parsedArgs := subReaderArgs{}
if err := d.DecodeElement(&parsedArgs, &start); err != nil {
return err
func (args *ReaderArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
args.FileHeaderInfo = none
args.RecordDelimiter = defaultRecordDelimiter
args.FieldDelimiter = defaultFieldDelimiter
args.QuoteCharacter = defaultQuoteCharacter
args.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
args.CommentCharacter = defaultCommentCharacter
args.AllowQuotedRecordDelimiter = false
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()
if err != nil {
if err == io.EOF {
break
}
return err
}
switch se := t.(type) {
case xml.StartElement:
tagName := se.Name.Local
switch tagName {
case "AllowQuotedRecordDelimiter":
var b bool
if err = d.DecodeElement(&b, &se); err != nil {
return err
}
args.AllowQuotedRecordDelimiter = b
default:
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
}
switch tagName {
case "FileHeaderInfo":
args.FileHeaderInfo = strings.ToLower(s)
case "RecordDelimiter":
args.RecordDelimiter = s
case "FieldDelimiter":
args.FieldDelimiter = s
case "QuoteCharacter":
if utf8.RuneCountInString(s) > 1 {
return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
}
args.QuoteCharacter = s
// Not supported yet
case "QuoteEscapeCharacter":
case "Comments":
args.CommentCharacter = s
default:
return errors.New("unrecognized option")
}
}
}
}
parsedArgs.FileHeaderInfo = strings.ToLower(parsedArgs.FileHeaderInfo)
switch parsedArgs.FileHeaderInfo {
case "":
parsedArgs.FileHeaderInfo = none
case none, use, ignore:
default:
return errInvalidFileHeaderInfo(fmt.Errorf("invalid FileHeaderInfo '%v'", parsedArgs.FileHeaderInfo))
}
switch len([]rune(parsedArgs.RecordDelimiter)) {
case 0:
parsedArgs.RecordDelimiter = defaultRecordDelimiter
case 1, 2:
default:
return fmt.Errorf("invalid RecordDelimiter '%v'", parsedArgs.RecordDelimiter)
}
switch len([]rune(parsedArgs.FieldDelimiter)) {
case 0:
parsedArgs.FieldDelimiter = defaultFieldDelimiter
case 1:
default:
return fmt.Errorf("invalid FieldDelimiter '%v'", parsedArgs.FieldDelimiter)
}
switch parsedArgs.QuoteCharacter {
case "":
parsedArgs.QuoteCharacter = defaultQuoteCharacter
case defaultQuoteCharacter:
default:
return fmt.Errorf("unsupported QuoteCharacter '%v'", parsedArgs.QuoteCharacter)
}
switch parsedArgs.QuoteEscapeCharacter {
case "":
parsedArgs.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
case defaultQuoteEscapeCharacter:
default:
return fmt.Errorf("unsupported QuoteEscapeCharacter '%v'", parsedArgs.QuoteEscapeCharacter)
}
switch parsedArgs.CommentCharacter {
case "":
parsedArgs.CommentCharacter = defaultCommentCharacter
case defaultCommentCharacter:
default:
return fmt.Errorf("unsupported Comments '%v'", parsedArgs.CommentCharacter)
}
if parsedArgs.AllowQuotedRecordDelimiter {
return fmt.Errorf("flag AllowQuotedRecordDelimiter is unsupported at the moment")
}
*args = ReaderArgs(parsedArgs)
args.QuoteEscapeCharacter = args.QuoteCharacter
args.unmarshaled = true
return nil
}
......@@ -138,55 +137,54 @@ func (args *WriterArgs) IsEmpty() bool {
// UnmarshalXML - decodes XML data.
func (args *WriterArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type subWriterArgs WriterArgs
parsedArgs := subWriterArgs{}
if err := d.DecodeElement(&parsedArgs, &start); err != nil {
return err
}
parsedArgs.QuoteFields = strings.ToLower(parsedArgs.QuoteFields)
switch parsedArgs.QuoteFields {
case "":
parsedArgs.QuoteFields = asneeded
case always, asneeded:
default:
return errInvalidQuoteFields(fmt.Errorf("invalid QuoteFields '%v'", parsedArgs.QuoteFields))
}
switch len([]rune(parsedArgs.RecordDelimiter)) {
case 0:
parsedArgs.RecordDelimiter = defaultRecordDelimiter
case 1, 2:
default:
return fmt.Errorf("invalid RecordDelimiter '%v'", parsedArgs.RecordDelimiter)
}
switch len([]rune(parsedArgs.FieldDelimiter)) {
case 0:
parsedArgs.FieldDelimiter = defaultFieldDelimiter
case 1:
default:
return fmt.Errorf("invalid FieldDelimiter '%v'", parsedArgs.FieldDelimiter)
}
switch parsedArgs.QuoteCharacter {
case "":
parsedArgs.QuoteCharacter = defaultQuoteCharacter
case defaultQuoteCharacter:
default:
return fmt.Errorf("unsupported QuoteCharacter '%v'", parsedArgs.QuoteCharacter)
}
switch parsedArgs.QuoteEscapeCharacter {
case "":
parsedArgs.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
case defaultQuoteEscapeCharacter:
default:
return fmt.Errorf("unsupported QuoteEscapeCharacter '%v'", parsedArgs.QuoteEscapeCharacter)
args.QuoteFields = asneeded
args.RecordDelimiter = defaultRecordDelimiter
args.FieldDelimiter = defaultFieldDelimiter
args.QuoteCharacter = defaultQuoteCharacter
args.QuoteEscapeCharacter = defaultQuoteCharacter
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()
if err != nil {
if err == io.EOF {
break
}
return err
}
switch se := t.(type) {
case xml.StartElement:
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
}
switch se.Name.Local {
case "QuoteFields":
args.QuoteFields = strings.ToLower(s)
case "RecordDelimiter":
args.RecordDelimiter = s
case "FieldDelimiter":
args.FieldDelimiter = s
case "QuoteCharacter":
switch utf8.RuneCountInString(s) {
case 0:
args.QuoteCharacter = "\x00"
case 1:
args.QuoteCharacter = s
default:
return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
}
// Not supported yet
case "QuoteEscapeCharacter":
default:
return errors.New("unrecognized option")
}
}
}
*args = WriterArgs(parsedArgs)
args.QuoteEscapeCharacter = args.QuoteCharacter
args.unmarshaled = true
return nil
}
......@@ -294,6 +294,11 @@ func NewReader(readCloser io.ReadCloser, args *ReaderArgs) (*Reader, error) {
ret := csv.NewReader(r)
ret.Comma = []rune(args.FieldDelimiter)[0]
ret.Comment = []rune(args.CommentCharacter)[0]
ret.Quote = []rune{}
if len([]rune(args.QuoteCharacter)) > 0 {
// Add the first rune of args.QuoteChracter
ret.Quote = append(ret.Quote, []rune(args.QuoteCharacter)[0])
}
ret.FieldsPerRecord = -1
// If LazyQuotes is true, a quote may appear in an unquoted field and a
// non-doubled quote may appear in a quoted field.
......
......@@ -63,7 +63,7 @@ func TestRead(t *testing.T) {
if err != nil {
break
}
record.WriteCSV(&result, []rune(c.fieldDelimiter)[0])
record.WriteCSV(&result, []rune(c.fieldDelimiter)[0], '"', false)
result.Truncate(result.Len() - 1)
result.WriteString(c.recordDelimiter)
}
......@@ -243,7 +243,7 @@ func TestReadExtended(t *testing.T) {
}
if fields < 10 {
// Write with fixed delimiters, newlines.
err := record.WriteCSV(&result, ',')
err := record.WriteCSV(&result, ',', '"', false)
if err != nil {
t.Error(err)
}
......@@ -454,7 +454,7 @@ func TestReadFailures(t *testing.T) {
break
}
// Write with fixed delimiters, newlines.
err := record.WriteCSV(&result, ',')
err := record.WriteCSV(&result, ',', '"', false)
if err != nil {
t.Error(err)
}
......
......@@ -92,9 +92,11 @@ func (r *Record) Clone(dst sql.Record) sql.Record {
}
// WriteCSV - encodes to CSV data.
func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter rune) error {
func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter rune, quote rune, alwaysQuote bool) error {
w := csv.NewWriter(writer)
w.Comma = fieldDelimiter
w.AlwaysQuote = alwaysQuote
w.Quote = quote
if err := w.Write(r.csvRecord); err != nil {
return err
}
......
......@@ -17,7 +17,6 @@
package json
import (
"encoding/csv"
"encoding/json"
"errors"
"fmt"
......@@ -27,6 +26,7 @@ import (
"strings"
"github.com/bcicen/jstream"
csv "github.com/minio/minio/pkg/csvparser"
"github.com/minio/minio/pkg/s3select/sql"
)
......@@ -108,7 +108,7 @@ func (r *Record) Set(name string, value *sql.Value) (sql.Record, error) {
}
// WriteCSV - encodes to CSV data.
func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter rune) error {
func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter rune, quote rune, alwaysQuote bool) error {
var csvRecord []string
for _, kv := range r.KVS {
var columnValue string
......@@ -137,6 +137,8 @@ func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter rune) error {
w := csv.NewWriter(writer)
w.Comma = fieldDelimiter
w.Quote = quote
w.AlwaysQuote = alwaysQuote
if err := w.Write(csvRecord); err != nil {
return err
}
......
......@@ -353,7 +353,10 @@ func (s3Select *S3Select) marshal(buf *bytes.Buffer, record sql.Record) error {
}()
bufioWriter.Reset(buf)
err := record.WriteCSV(bufioWriter, []rune(s3Select.Output.CSVArgs.FieldDelimiter)[0])
err := record.WriteCSV(bufioWriter,
[]rune(s3Select.Output.CSVArgs.FieldDelimiter)[0],
[]rune(s3Select.Output.CSVArgs.QuoteCharacter)[0],
strings.ToLower(s3Select.Output.CSVArgs.QuoteFields) == "always")
if err != nil {
return err
}
......
......@@ -252,6 +252,7 @@ func TestJSONQueries(t *testing.T) {
</InputSerialization>
<OutputSerialization>
<CSV>
<QuoteCharacter>"</QuoteCharacter>
</CSV>
</OutputSerialization>
<RequestProgress>
......@@ -587,6 +588,7 @@ func TestCSVQueries2(t *testing.T) {
<CompressionType>NONE</CompressionType>
<CSV>
<FileHeaderInfo>USE</FileHeaderInfo>
<QuoteCharacter>"</QuoteCharacter>
</CSV>
</InputSerialization>
<OutputSerialization>
......
......@@ -131,11 +131,11 @@ func TestNDJSON(t *testing.T) {
t.Error(err)
}
var gotB, wantB bytes.Buffer
err = rec.WriteCSV(&gotB, ',')
err = rec.WriteCSV(&gotB, ',', '"', false)
if err != nil {
t.Error(err)
}
err = want.WriteCSV(&wantB, ',')
err = want.WriteCSV(&wantB, ',', '"', false)
if err != nil {
t.Error(err)
}
......
......@@ -17,10 +17,11 @@
package simdj
import (
"encoding/csv"
"fmt"
"io"
csv "github.com/minio/minio/pkg/csvparser"
"github.com/bcicen/jstream"
"github.com/minio/minio/pkg/s3select/json"
"github.com/minio/minio/pkg/s3select/sql"
......@@ -140,7 +141,7 @@ func (r *Record) Set(name string, value *sql.Value) (sql.Record, error) {
}
// WriteCSV - encodes to CSV data.
func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter rune) error {
func (r *Record) WriteCSV(writer io.Writer, fieldDelimiter, quote rune, alwaysQuote bool) error {
csvRecord := make([]string, 0, 10)
var tmp simdjson.Iter
obj := r.object
......@@ -173,6 +174,8 @@ allElems:
}
w := csv.NewWriter(writer)
w.Comma = fieldDelimiter
w.Quote = quote
w.AlwaysQuote = alwaysQuote
if err := w.Write(csvRecord); err != nil {
return err
}
......
......@@ -46,7 +46,7 @@ type Record interface {
// Set a value.
// Can return a different record type.
Set(name string, value *Value) (Record, error)
WriteCSV(writer io.Writer, fieldDelimiter rune) error
WriteCSV(writer io.Writer, fieldDelimiter, quote rune, alwaysQuote bool) error
WriteJSON(writer io.Writer) error
// Clone the record and if possible use the destination provided.
......