Go library to work with AWS(amazon web services) S3 and GCP(google cloud platform) cloud storage
go get -u github.com/AccelByte/common-blob-go
commonblobgo "github.com/AccelByte/common-blob-go"
To create a new storage client, use this function:
storage, err := storage, err := NewCloudStorage(
ctx,
isTesting,
bucketProvider,
bucketName,
awsS3Endpoint,
awsS3Region,
awsS3AccessKeyID,
awsS3SecretAccessKey,
gcpCredentialsJSON,
gcpStorageEmulatorHost,
)
NewCloudStorage
requires such parameters :
-
ctx context.Context : a context that could be cancelled to force-stop the initialization
-
isTesting bool : a flag to switch between external and in-docker-compose dependencies. Used from tests
-
bucketProvider string : provider type. Could be
aws
orgcp
-
bucketName string : the name of a bucket
-
awsS3Endpoint string : S3 endpoint. Used only from tests(required if bucketProvider==
aws
and isTesting ==true
) -
awsS3Region string : S3 region(required if bucketProvider==
aws
) -
awsS3AccessKeyID string : S3 Access key(required if bucketProvider==
aws
) -
awsS3SecretAccessKey string : S3 secret key(required if bucketProvider==
aws
) -
gcpCredentialsJSON string : GCP JSON credentials(optional if bucketProvider==
gcp
).-
If empty - the library will attempt to self-configure from k8s GCP API. The Client(service account) should have the role "Service Account Token Creator"
-
-
gcpStorageEmulatorHost string : GCP storage host. Used only from tests(required if bucketProvider==
gcp
and isTesting ==true
)
To enable cloud storage additional features:
storage, err := storage, err := NewCloudStorageWithOption(
ctx,
isTesting,
bucketProvider,
bucketName,
opts,
)
Cloud-specific parameter such as awsRegion
, gcpStorageEmulatorHost
, etc. has been moved to opts CloudStorageOption
.
Supported additional cloud storage feature:
opts.AWSEnableS3Accelerate
(default: false) : a boolean that indicate S3 bucket use accelerate endpoint. Not available in testing using localstack or using path-style S3 endpoint. Note: make sure to enable transfer accelerate in S3 bucket, please refer to this documentation.
type CloudStorage interface {
List(ctx context.Context, prefix string) *ListIterator // iterate over all objects in the folder
ListWithOptions(ctx context.Context, options *ListOptions) *ListIterator // iterate over objects in the folder based on ListOptions criteria
Get(ctx context.Context, key string) ([]byte, error) // get the object by a name
GetReader(ctx context.Context, key string) (io.ReadCloser, error) // get reader to operate with io.ReadCloser
Delete(ctx context.Context, key string) error // delete the object by a name
CreateBucket(ctx context.Context, bucketPrefix string, expirationTimeDays int64) error // create a bucket. Used only from tests
Close() // close connection
GetSignedURL(ctx context.Context, key string, expiry time.Duration) (string, error) // create signed URL
Write(ctx context.Context, key string, body []byte, contentType *string) error // write the object a file-name
GetWriter(ctx context.Context, key string) (io.WriteCloser, error) // get writer to operate with io.WriteCloser
Attributes(ctx context.Context, key string) (*Attributes, error) // get object attributes
Exists(ctx context.Context, key string) (bool, error) // check object existence
}
list := storage.List(ctx, bucketPrefix)
for {
item, err := list.Next(ctx)
if err == io.EOF {
break // no more object
}
// ...
if item.Key == fileName {
fileFound = true
}
}
list := storage.CloudStorage.ListWithOptions(parentCtx, &commonblobgo.ListOptions{
Prefix: bucketPrefix,
// An empty delimiter means that the bucket is treated as a single flat namespace.
//
// A non-empty delimiter means that any result with the delimiter in its key after Prefix is stripped will be returned with ListObject.IsDir = true,
// ListObject.Key truncated after the delimiter. These results represent "directories"
Delimiter: "/",
})
var directories []string
for {
item, err := list.Next(ctx)
if err == io.EOF {
break // no more object
}
// ...
if item.IsDir {
directories = append(directories, item.Key)
}
}
storedBody, err := storage.Get(ctx, fileName)
if err != nil {
return nil, err
}
fmt.Println(string(storedBody))
reader, err := storage.GetReader(ctx, fileName)
if err != nil {
return nil, err
}
defer reader.Close() // Important to prevent memory leaks
storedBody, err := ioutil.ReadAll(reader)
fmt.Println(string(storedBody))
reader, err := storage.GetRangeReader(ctx, fileName, offset, length)
if err != nil {
return nil, err
}
defer reader.Close() // Important to prevent memory leaks
storedBody, err := ioutil.ReadAll(reader)
fmt.Println(string(storedBody))
err = storage.Delete(ctx, fileName)
if err != nil {
return nil, err
}
err = storage.CreateBucket(ctx, bucketPrefix, 1)
if err != nil {
return nil, err
}
storage, err := storage, err := NewCloudStorage(
ctx,
isTesting,
bucketProvider,
bucketName,
awsS3Endpoint,
awsS3Region,
awsS3AccessKeyID,
awsS3SecretAccessKey,
gcpCredentialsJSON,
gcpStorageEmulatorHost,
)
defer storage.Close()
url, err := storage.GetSignedURL(ctx, fileName, time.Hour)
if err != nil {
return nil, err
}
fmt.Println(url)
err := storage.Write(ctx, fileName, bodyBytes, nil)
if err != nil {
return nil, err
}
body := []byte(`{"key": "value", "key2": "value2"}`)
writer, err := storage.GetWriter(ctx, fileName)
if err != nil {
return nil, err
}
_, err = writer.Write(body[:10])
if err != nil {
return nil, err
}
_, err = writer.Write(body[10:20])
if err != nil {
return nil, err
}
_, err = writer.Write(body[20:])
if err != nil {
return nil, err
}
err = writer.Close()
if err != nil {
return nil, err
}
attrs, err := storage.Attributes(ctx, fileName)
if err != nil {
return nil, err
}
fmt.Println(attrs.Size)
isExists, err := storage.Exists(ctx, fileName)
if err != nil {
return nil, err
}
if isExists {
fmt.Println("found")
}
Copyright © 2020, AccelByte Inc. Released under the Apache License, Version 2.0