mirror of
https://github.com/pgsty/minio.git
synced 2026-07-22 21:50:22 +03:00
Add mint tests into MinIO repo (#7886)
This commit is contained in:
Harshavardhana
@@ -0,0 +1,409 @@
|
||||
/*
|
||||
* Mint, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|
||||
var randSource *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
const (
|
||||
letterIdxBits = 6 // 6 bits to represent a letter index
|
||||
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
|
||||
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
|
||||
)
|
||||
const (
|
||||
PASS = "PASS" // Indicate that a test passed
|
||||
FAIL = "FAIL" // Indicate that a test failed
|
||||
NA = "NA" // Indicate that a test is not applicable
|
||||
maxPartSize = int64(512 * 1000 * 1024)
|
||||
maxRetries = 1
|
||||
)
|
||||
|
||||
type mintJSONFormatter struct {
|
||||
}
|
||||
|
||||
func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
|
||||
data := make(log.Fields, len(entry.Data))
|
||||
for k, v := range entry.Data {
|
||||
switch v := v.(type) {
|
||||
case error:
|
||||
// Otherwise errors are ignored by `encoding/json`
|
||||
// https://github.com/sirupsen/logrus/issues/137
|
||||
data[k] = v.Error()
|
||||
default:
|
||||
data[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
|
||||
}
|
||||
return append(serialized, '\n'), nil
|
||||
}
|
||||
|
||||
// log successful test runs
|
||||
func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
// log with the fields as per mint
|
||||
fields := log.Fields{"name": "test worm mode", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": PASS}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
// log failed test runs
|
||||
func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
|
||||
// calculate the test case duration
|
||||
duration := time.Since(startTime)
|
||||
var fields log.Fields
|
||||
// log with the fields as per mint
|
||||
if err != nil {
|
||||
fields = log.Fields{"name": "test worm mode", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message, "error": err}
|
||||
} else {
|
||||
fields = log.Fields{"name": "test worm mode", "function": function, "args": args,
|
||||
"duration": duration.Nanoseconds() / 1000000, "status": FAIL, "alert": alert, "message": message}
|
||||
}
|
||||
return log.WithFields(fields)
|
||||
}
|
||||
|
||||
func randBucketName() string {
|
||||
b := make([]byte, 55)
|
||||
for i := range b {
|
||||
b[i] = charset[randSource.Intn(len(charset))]
|
||||
}
|
||||
return "bucket-" + string(b)
|
||||
}
|
||||
|
||||
func testPutDeletObject(s3Client *s3.S3) {
|
||||
startTime := time.Now()
|
||||
object := "testObject"
|
||||
function := "PutAndDelete"
|
||||
bucket := randBucketName()
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": bucket,
|
||||
"objectName": object,
|
||||
"expiry": expiry,
|
||||
}
|
||||
// First time bucket creation will be successful
|
||||
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON - CreateBucket Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
// First time put object will be successful
|
||||
putInput1 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("fileToUpload")),
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput1)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON - expected to pass but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
// Put Object
|
||||
putInput2 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("filetouploadSecondTime")),
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput2)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON Put is expected to fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Deleting the Object
|
||||
delObject := &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.DeleteObject(delObject)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON Delete is expected to fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
successLogger(function, args, startTime).Info()
|
||||
|
||||
}
|
||||
|
||||
func testCopyObject(s3Client *s3.S3) {
|
||||
startTime := time.Now()
|
||||
function := "CopyObject"
|
||||
object := "DestinationObject"
|
||||
object1 := "SourceObject"
|
||||
destinationBucket := randBucketName()
|
||||
sourceBucket := randBucketName()
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": destinationBucket,
|
||||
"objectName": object,
|
||||
"expiry": expiry,
|
||||
}
|
||||
// Create Destination bucket
|
||||
_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(destinationBucket),
|
||||
})
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON Destination Bucket Creation Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Put object on Destination bucket
|
||||
putInput1 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("file to Upload In Destination")),
|
||||
Bucket: aws.String(destinationBucket),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput1)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON PUT expected to pass but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Create Source bucket
|
||||
_, err1 := s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(sourceBucket),
|
||||
})
|
||||
if err1 != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON Source Bucket Creation Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Put object on Destination bucket
|
||||
putInput2 := &s3.PutObjectInput{
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("file content to copy ")),
|
||||
Bucket: aws.String(sourceBucket),
|
||||
Key: aws.String(object1),
|
||||
}
|
||||
_, err = s3Client.PutObject(putInput2)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON PUT expected to pass but got %v", err), err).Fatal()
|
||||
return
|
||||
}
|
||||
|
||||
// Test for Copy Object
|
||||
copyInput := &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destinationBucket),
|
||||
CopySource: aws.String(sourceBucket + "/" + object1),
|
||||
Key: aws.String(object),
|
||||
}
|
||||
|
||||
_, err = s3Client.CopyObject(copyInput)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON Copy Object should fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
successLogger(function, args, startTime).Info()
|
||||
|
||||
}
|
||||
|
||||
func testPutMultipart(s3Client *s3.S3) {
|
||||
bucket := randBucketName()
|
||||
startTime := time.Now()
|
||||
object := "testObject"
|
||||
expiry := 1 * time.Minute
|
||||
args := map[string]interface{}{
|
||||
"bucketName": bucket,
|
||||
"objectName": object,
|
||||
"expiry": expiry,
|
||||
}
|
||||
function := "PutMultiPart"
|
||||
file, err := os.Open("/mint/data/datafile-5-MB")
|
||||
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON err opening file", err).Fatal()
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
fileInfo, _ := file.Stat()
|
||||
size := fileInfo.Size()
|
||||
buffer := make([]byte, size)
|
||||
fileType := http.DetectContentType(buffer)
|
||||
file.Read(buffer)
|
||||
|
||||
path := file.Name()
|
||||
input := &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(path),
|
||||
ContentType: aws.String(fileType),
|
||||
}
|
||||
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON Destination Bucket Creation Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
// Upload for the first time
|
||||
resp, err := s3Client.CreateMultipartUpload(input)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON CreateMultipartUpload Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
var curr, partLength int64
|
||||
var remaining = size
|
||||
var completedParts []*s3.CompletedPart
|
||||
partNumber := 1
|
||||
for curr = 0; remaining != 0; curr += partLength {
|
||||
if remaining < maxPartSize {
|
||||
partLength = remaining
|
||||
} else {
|
||||
partLength = maxPartSize
|
||||
}
|
||||
completedPart, err := uploadPart(s3Client, resp, buffer[curr:curr+partLength], partNumber)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON uploadPart Failed", err).Fatal()
|
||||
err := abortMultipartUpload(s3Client, resp)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON abortMultipartUpload Failed", err).Fatal()
|
||||
}
|
||||
return
|
||||
}
|
||||
remaining -= partLength
|
||||
partNumber++
|
||||
completedParts = append(completedParts, completedPart)
|
||||
}
|
||||
_, err = completeMultipartUpload(s3Client, resp, completedParts)
|
||||
if err != nil {
|
||||
failureLog(function, args, startTime, "", "WORM_MODE ON completeMultipartUpload Failed", err).Fatal()
|
||||
return
|
||||
}
|
||||
// These tests should fail
|
||||
_, err = s3Client.CreateMultipartUpload(input)
|
||||
if err == nil {
|
||||
failureLog(function, args, startTime, "", fmt.Sprintf("WORM_MODE ON CreateMultipartUpload must fail, but it passed %v", nil), nil).Fatal()
|
||||
return
|
||||
}
|
||||
successLogger(function, args, startTime).Info()
|
||||
|
||||
}
|
||||
func completeMultipartUpload(svc *s3.S3, resp *s3.CreateMultipartUploadOutput, completedParts []*s3.CompletedPart) (*s3.CompleteMultipartUploadOutput, error) {
|
||||
completeInput := &s3.CompleteMultipartUploadInput{
|
||||
Bucket: resp.Bucket,
|
||||
Key: resp.Key,
|
||||
UploadId: resp.UploadId,
|
||||
MultipartUpload: &s3.CompletedMultipartUpload{
|
||||
Parts: completedParts,
|
||||
},
|
||||
}
|
||||
return svc.CompleteMultipartUpload(completeInput)
|
||||
}
|
||||
func uploadPart(svc *s3.S3, resp *s3.CreateMultipartUploadOutput, fileBytes []byte, partNumber int) (*s3.CompletedPart, error) {
|
||||
tryNum := 1
|
||||
partInput := &s3.UploadPartInput{
|
||||
Body: bytes.NewReader(fileBytes),
|
||||
Bucket: resp.Bucket,
|
||||
Key: resp.Key,
|
||||
PartNumber: aws.Int64(int64(partNumber)),
|
||||
UploadId: resp.UploadId,
|
||||
ContentLength: aws.Int64(int64(len(fileBytes))),
|
||||
}
|
||||
|
||||
for tryNum <= maxRetries {
|
||||
uploadResult, err := svc.UploadPart(partInput)
|
||||
if err != nil {
|
||||
if tryNum == maxRetries {
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
return nil, aerr
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
tryNum++
|
||||
} else {
|
||||
return &s3.CompletedPart{
|
||||
ETag: uploadResult.ETag,
|
||||
PartNumber: aws.Int64(int64(partNumber)),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func abortMultipartUpload(svc *s3.S3, resp *s3.CreateMultipartUploadOutput) error {
|
||||
abortInput := &s3.AbortMultipartUploadInput{
|
||||
Bucket: resp.Bucket,
|
||||
Key: resp.Key,
|
||||
UploadId: resp.UploadId,
|
||||
}
|
||||
_, err := svc.AbortMultipartUpload(abortInput)
|
||||
return err
|
||||
}
|
||||
|
||||
func main() {
|
||||
endpoint := os.Getenv("SERVER_ENDPOINT")
|
||||
accessKey := os.Getenv("ACCESS_KEY")
|
||||
secretKey := os.Getenv("SECRET_KEY")
|
||||
secure := os.Getenv("ENABLE_HTTPS")
|
||||
sdkEndpoint := "http://" + endpoint
|
||||
if secure == "1" {
|
||||
sdkEndpoint = "https://" + endpoint
|
||||
}
|
||||
creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
|
||||
newSession := session.New()
|
||||
s3Config := &aws.Config{
|
||||
Credentials: creds,
|
||||
Endpoint: aws.String(sdkEndpoint),
|
||||
Region: aws.String("us-east-1"),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
}
|
||||
|
||||
// Create an S3 service object in the default region.
|
||||
s3Client := s3.New(newSession, s3Config)
|
||||
|
||||
// Output to stdout instead of the default stderr
|
||||
log.SetOutput(os.Stdout)
|
||||
// create custom formatter
|
||||
mintFormatter := mintJSONFormatter{}
|
||||
// set custom formatter
|
||||
log.SetFormatter(&mintFormatter)
|
||||
// log Info or above -- success cases are Info level, failures are Fatal level
|
||||
log.SetLevel(log.InfoLevel)
|
||||
// execute tests
|
||||
// Test Put and Delete Object
|
||||
testPutDeletObject(s3Client)
|
||||
//testCopyObject
|
||||
testCopyObject(s3Client)
|
||||
// Test Multipart Upload
|
||||
testPutMultipart(s3Client)
|
||||
}
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Mint (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# handle command line arguments
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
output_log_file="$1"
|
||||
error_log_file="$2"
|
||||
|
||||
# run tests
|
||||
/mint/run/core/worm/worm 1>>"$output_log_file" 2>"$error_log_file"
|
||||
Reference in New Issue
Block a user