Efficient file storage and management are crucial for modern applications. OpenStack Swift, an open-source object storage system, offers a scalable solution for storing and retrieving large amounts of data in the cloud. This DevTip explains how to upload files to OpenStack Swift using Go and the Gophercloud library, a robust Go SDK for interacting with OpenStack APIs.

Prerequisites

Before you begin, ensure you have:

  • Go installed (version 1.22 or higher)
  • Basic knowledge of Go programming
  • Access to an OpenStack Swift environment
  • OpenStack credentials (configurable via clouds.yaml)

Setting up the Go environment

First, create a new directory for your project and initialize a Go module:

mkdir swift-upload-example
cd swift-upload-example
 go mod init example.com/swift-upload

Installing Gophercloud

Gophercloud is an open-source Go SDK for working with OpenStack APIs. Install the v2 version using:

go get github.com/gophercloud/gophercloud/v2

Authenticating with OpenStack Swift

Create a new file named main.go. Below are two methods of authentication: using direct credentials and clouds.yaml.

package main

import (
	"context"
	"fmt"

	"github.com/gophercloud/gophercloud/v2"
	"github.com/gophercloud/gophercloud/v2/openstack"
	"github.com/gophercloud/utils/openstack/clientconfig"
)

func authenticateWithCredentials(ctx context.Context) (*gophercloud.ServiceClient, error) {
	opts := gophercloud.AuthOptions{
		IdentityEndpoint: "https://your-openstack-auth-url",
		Username:         "your-username",
		Password:         "your-password",
		TenantName:       "your-tenant-name",
		DomainName:       "your-domain-name",
	}

	provider, err := openstack.AuthenticatedClient(ctx, opts)
	if err != nil {
		return nil, fmt.Errorf("error creating OpenStack provider client: %w", err)
	}

	client, err := openstack.NewObjectStorageV1(ctx, provider, gophercloud.EndpointOpts{
		Region: "your-region",
	})
	if err != nil {
		return nil, fmt.Errorf("error creating Swift service client: %w", err)
	}

	return client, nil
}

func authenticateWithCloudsYAML(ctx context.Context) (*gophercloud.ServiceClient, error) {
	opts := &clientconfig.ClientOpts{
		Cloud: "openstack", // Name of the cloud in clouds.yaml
	}

	provider, err := clientconfig.AuthenticatedClient(ctx, opts)
	if err != nil {
		return nil, fmt.Errorf("error creating provider client: %w", err)
	}

	client, err := openstack.NewObjectStorageV1(ctx, provider, gophercloud.EndpointOpts{})
	if err != nil {
		return nil, fmt.Errorf("error creating Swift service client: %w", err)
	}

	return client, nil
}

Uploading files to Swift containers

Ensure container exists

Before uploading, verify that the target container exists. If it does not, create it with robust error handling.

package main

import (
	"context"
	"fmt"
	"strings"
	"github.com/gophercloud/gophercloud/v2/openstack/objectstorage/v1/containers"
)

func ensureContainer(ctx context.Context, client *gophercloud.ServiceClient, containerName string) error {
	result := containers.Get(client, containerName, nil)
	if result.Err == nil {
		return nil // Container exists
	}
	// Check if the error indicates that the container was not found (HTTP 404)
	if !strings.Contains(result.Err.Error(), "404") {
		return fmt.Errorf("error checking container %s: %w", containerName, result.Err)
	}

	// Create the container since it does not exist
	_, err := containers.Create(client, containerName, containers.CreateOpts{}).Extract()
	if err != nil {
		return fmt.Errorf("error creating container %s: %w", containerName, err)
	}

	return nil
}

Upload a file

Use the following function to upload a file with proper content type and error handling.

package main

import (
	"context"
	"fmt"
	"os"

	"github.com/gophercloud/gophercloud/v2/openstack/objectstorage/v1/objects"
)

func uploadFile(ctx context.Context, client *gophercloud.ServiceClient, containerName, objectName, filePath string) error {
	file, err := os.Open(filePath)
	if err != nil {
		return fmt.Errorf("error opening file: %w", err)
	}
	defer file.Close()

	stat, err := file.Stat()
	if err != nil {
		return fmt.Errorf("error getting file info: %w", err)
	}

	createOpts := objects.CreateOpts{
		Content:       file,
		ContentLength: stat.Size(),
		ContentType:   "application/octet-stream",
	}

	result := objects.Create(client, containerName, objectName, createOpts)
	if err := result.Err; err != nil {
		return fmt.Errorf("error uploading file: %w", err)
	}

	return nil
}

Retry logic for uploads

Implement a retry mechanism to handle transient errors when uploading files.

package main

import (
	"context"
	"fmt"
	"time"
)

func uploadWithRetry(ctx context.Context, client *gophercloud.ServiceClient, containerName, objectName, filePath string) error {
	backoff := []time.Duration{time.Second, 2 * time.Second, 5 * time.Second}

	for i, wait := range backoff {
		err := uploadFile(ctx, client, containerName, objectName, filePath)
		if err == nil {
			return nil
		}

		if i == len(backoff)-1 {
			return fmt.Errorf("failed after %d retries: %w", len(backoff), err)
		}

		time.Sleep(wait)
	}

	return nil
}

Upload an object with retry (using createopts)

For uploading segments or objects that do not originate from file paths, use this generalized retry function.

package main

import (
	"context"
	"fmt"
	"time"

	"github.com/gophercloud/gophercloud/v2/openstack/objectstorage/v1/objects"
)

func uploadObjectWithRetry(ctx context.Context, client *gophercloud.ServiceClient, containerName, objectName string, opts objects.CreateOpts) error {
	backoff := []time.Duration{time.Second, 2 * time.Second, 5 * time.Second}
	var err error

	for i, wait := range backoff {
		result := objects.Create(client, containerName, objectName, opts)
		err = result.Err
		if err == nil {
			return nil
		}
		if i == len(backoff)-1 {
			return fmt.Errorf("failed after %d retries: %w", len(backoff), err)
		}
		time.Sleep(wait)
	}

	return err
}

Handling large files with segmented uploads

For large files, divide the file into segments and upload each segment separately. Then, create a manifest object to assemble these segments.

package main

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"

	"github.com/gophercloud/gophercloud/v2/openstack/objectstorage/v1/objects"
)

func uploadLargeFile(ctx context.Context, client *gophercloud.ServiceClient, containerName, objectName, filePath string) error {
	file, err := os.Open(filePath)
	if err != nil {
		return fmt.Errorf("error opening file: %w", err)
	}
	defer file.Close()

	// Use a separate container for segments
	segmentsContainerName := containerName + "_segments"
	if err := ensureContainer(ctx, client, segmentsContainerName); err != nil {
		return fmt.Errorf("error ensuring segments container: %w", err)
	}

	buffer := make([]byte, 5*1024*1024) // 5 MB segments
	segmentNum := 0

	for {
		n, readErr := file.Read(buffer)
		if n > 0 {
			segmentName := fmt.Sprintf("%s/%08d", objectName, segmentNum)
			segmentOpts := objects.CreateOpts{
				Content:       bytes.NewReader(buffer[:n]),
				ContentLength: int64(n),
				ContentType:   "application/octet-stream",
			}
			if err := uploadObjectWithRetry(ctx, client, segmentsContainerName, segmentName, segmentOpts); err != nil {
				return fmt.Errorf("error uploading segment %d: %w", segmentNum, err)
			}
			segmentNum++
		}

		if readErr == io.EOF {
			break
		}
		if readErr != nil {
			return fmt.Errorf("error reading file: %w", readErr)
		}
	}

	// Create a manifest object that assembles the segments
	manifestOpts := objects.CreateOpts{
		ContentType: "application/octet-stream",
		Metadata: map[string]string{
			"X-Object-Manifest": fmt.Sprintf("%s/%s/", segmentsContainerName, objectName),
		},
	}
	if err := uploadObjectWithRetry(ctx, client, containerName, objectName, manifestOpts); err != nil {
		return fmt.Errorf("error creating manifest for large file: %w", err)
	}

	return nil
}

Handling errors and best practices

Rate limiting

Implement rate limiting to avoid overwhelming the Swift API. This example uses Golang's rate limiter.

package main

import (
	"context"
	"fmt"

	"golang.org/x/time/rate"
)

type RateLimitedClient struct {
	client  *gophercloud.ServiceClient
	limiter *rate.Limiter
}

func NewRateLimitedClient(client *gophercloud.ServiceClient, rps float64) *RateLimitedClient {
	return &RateLimitedClient{
		client:  client,
		limiter: rate.NewLimiter(rate.Limit(rps), 1),
	}
}

func (r *RateLimitedClient) UploadFile(ctx context.Context, containerName, objectName, filePath string) error {
	if err := r.limiter.Wait(ctx); err != nil {
		return fmt.Errorf("rate limit wait error: %w", err)
	}

	return uploadWithRetry(ctx, r.client, containerName, objectName, filePath)
}

Conclusion

Uploading files to OpenStack Swift using Go and Gophercloud v2 provides a flexible and efficient way to manage cloud storage. By incorporating proper error handling, retry logic, and rate limiting, you can build reliable systems that gracefully handle various scenarios.

For more advanced workflows, Transloadit offers a suite of file handling services. Explore our file exporting service for additional options to streamline your file processing.

Happy coding!