Breaking News: Grepper is joining You.com. Read the official announcement!

aws-sdk-go-v2 - Download Manager - Download All Objects in a Bucket

CL answered on March 1, 2023 Popularity 1/10 Helpfulness 1/10

answer aws-sdk-go-v2 - Download Manager - Download All Objects in a Bucket

related aws-sdk-go-v2 - Download Manager - Download a File

aws-sdk-go-v2 - Download Manager - Download All Objects in a Bucket

Comment

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"path/filepath"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
	"github.com/aws/aws-sdk-go-v2/service/s3"
)

var (
	Bucket         = "MyBucket" // Download from this bucket
	Prefix         = "logs/"    // Using this key prefix
	LocalDirectory = "s3logs"   // Into this directory
)

func main() {
	cfg, err := config.LoadDefaultConfig(context.TODO())
	if err != nil {
		log.Fatalln("error:", err)
	}

	client := s3.NewFromConfig(cfg)
	manager := manager.NewDownloader(client)

	paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
		Bucket: &Bucket,
		Prefix: &Prefix,
	})

	for paginator.HasMorePages() {
		page, err := paginator.NextPage(context.TODO())
		if err != nil {
			log.Fatalln("error:", err)
		}
		for _, obj := range page.Contents {
			if err := downloadToFile(manager, LocalDirectory, Bucket, aws.ToString(obj.Key)); err != nil {
				log.Fatalln("error:", err)
			}
		}
	}
}

func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key string) error {
	// Create the directories in the path
	file := filepath.Join(targetDirectory, key)
	if err := os.MkdirAll(filepath.Dir(file), 0775); err != nil {
		return err
	}

	// Set up the local file
	fd, err := os.Create(file)
	if err != nil {
		return err
	}
	defer fd.Close()

	// Download the file using the AWS SDK for Go
	fmt.Printf("Downloading s3://%s/%s to %s...\n", bucket, key, file)
	_, err = downloader.Download(context.TODO(), fd, &s3.GetObjectInput{Bucket: &bucket, Key: &key})

	return err
}

Popularity 1/10 Helpfulness 1/10 Language go

Source: aws.github.io

Tags: aws-sdk-go-v2 bucket download download-manager go

Link to this answer
Share Copy Link

Contributed on Mar 01 2023

0 Answers Avg Quality 2/10

Closely Related Answers

aws-sdk-go-v2 - Download Manager - Download a File

Comment

import "context"
import "github.com/aws/aws-sdk-go-v2/aws"
import "github.com/aws/aws-sdk-go-v2/config"
import "github.com/aws/aws-sdk-go-v2/service/s3"
import "github.com/aws/aws-sdk-go-v2/feature/s3/manager"

// ...

cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
	log.Println("error:", err)
	return
}

client := s3.NewFromConfig(cfg)

downloader := manager.NewDownloader(client)
numBytes, err := downloader.Download(context.TODO(), downloadFile, &s3.GetObjectInput{
	Bucket: aws.String("my-bucket"), 
	Key:    aws.String("my-key"),
})

The following example uses the Amazon S3 `Downloader` to download a file. Using `Downloader` is similar to the [s3.GetObject](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3#Client.GetObject) operation.

The `downloadFile` parameter is an `io.WriterAt` type. The `WriterAt` interface enables the `Downloader` to write multiple parts of the file in parallel.

Popularity 1/10 Helpfulness 1/10 Language go

Source: aws.github.io

Tags: download-manager file aws-sdk-go-v2 download go

Link to this answer
Share Copy Link

Contributed on Mar 01 2023

0 Answers Avg Quality 2/10

aws-sdk-go-v2 - Download Manager - Download All Objects in a Bucket

Contents

More Related Answers

aws-sdk-go-v2 - Download Manager - Download All Objects in a Bucket

Closely Related Answers

aws-sdk-go-v2 - Download Manager - Download a File

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.