how to process each row in s3 bucket using golang

Comment

0

package s3demo

import (
    "os"
    "context"
    "fmt"
    "io/ioutil"
    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/aws/awserr"
    "github.com/aws/aws-sdk-go-v2/aws/external"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

func GetObjectWithV2SDKDemo() {
    bucket := "YOUR_BUCKET"
    key := "YOUR_OBJECT_KEY"
    fileName := "YOUR_FILE_PATH"

    // may need AWS_PROFILE and AWS_REGION populated as environment variables
    cfg, err := external.LoadDefaultAWSConfig()
    if err != nil {
        panic("failed to load config, " + err.Error())
    }

    svc := s3.New(cfg)
    ctx := context.Background()
    req := svc.GetObjectRequest(&s3.GetObjectInput{
        Bucket: aws.String(bucket),
        Key:    aws.String(key),
    })

    resp, err := req.Send(ctx)
    if err != nil {
        panic(err)
    }

    s3objectBytes, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }
    // create file
    f, err := os.Create(fileName)
    defer f.Close()
    if err != nil {
        panic(err)
    }

    bytesWritten, err := f.Write(s3objectBytes)
    if err != nil {
        panic(err)
    }

    fmt.Printf("Fetched %d bytes for S3Object\n", bytesWritten)
    fmt.Printf("successfully downloaded data from %s/%s\n to file %s", bucket, key, fileName)
}

Popularity 1/10 Helpfulness 1/10 Language whatever

Source: stackoverflow.com

Tags: each process row s3-bucket using whatever

Share

Link to this answer
Share Copy Link

Contributed on Oct 06 2022

Pathi Prudhvi

0 Answers Avg Quality 2/10

how to process each row in s3 bucket using golang

Comment

0

package s3demo

import (
    "os"
    "context"
    "fmt"
    "io/ioutil"
    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/aws/awserr"
    "github.com/aws/aws-sdk-go-v2/aws/external"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

func GetObjectWithV2SDKDemo() {
    bucket := "YOUR_BUCKET"
    key := "YOUR_OBJECT_KEY"
    fileName := "YOUR_FILE_PATH"

    // may need AWS_PROFILE and AWS_REGION populated as environment variables
    cfg, err := external.LoadDefaultAWSConfig()
    if err != nil {
        panic("failed to load config, " + err.Error())
    }

    svc := s3.New(cfg)
    ctx := context.Background()
    req := svc.GetObjectRequest(&s3.GetObjectInput{
        Bucket: aws.String(bucket),
        Key:    aws.String(key),
    })

    resp, err := req.Send(ctx)
    if err != nil {
        panic(err)
    }

    s3objectBytes, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
    }
    // create file
    f, err := os.Create(fileName)
    defer f.Close()
    if err != nil {
        panic(err)
    }

    bytesWritten, err := f.Write(s3objectBytes)
    if err != nil {
        panic(err)
    }

    fmt.Printf("Fetched %d bytes for S3Object\n", bytesWritten)
    fmt.Printf("successfully downloaded data from %s/%s\n to file %s", bucket, key, fileName)
}

Popularity 1/10 Helpfulness 1/10 Language whatever

Source: stackoverflow.com

Tags: process row s3-bucket using each whatever

Share

Link to this answer
Share Copy Link

Contributed on Oct 06 2022

Pathi Prudhvi

0 Answers Avg Quality 2/10

how to process each row in s3 bucket using golang

Comment

0

func getDomains(path string) (lines []string, Error error) {

    file, err := os.Open(path)
    if err != nil {
        log.Fatalln(err)
    }

    defer file.Close()

    scanner := bufio.NewScanner(file)

    for scanner.Scan() {
        lines = append(lines, scanner.Text())
    }

    return lines, scanner.Err()
}

func Process(w http.ResponseWriter, r *http.Request) {


    urls := make(chan *Http, Threads*10)
    list, err := getDomains("**NEED A TEXT FILE FROM S3 HERE as an argument**")
    if err != nil {
        log.Fatalln(err)
    }

    var wg sync.WaitGroup
    for i := 0; i < Threads; i++ {
        wg.Add(1)
        go func() {
            for url := range urls {
                url.DNS()
            }

            wg.Done()
        }()
    }

    for i := 0; i < len(list); i++ {
        Progress := fmt.Sprintln(w, len(list))
        urls <- &Http{Url: list[i], Num: Progress}
    }

    close(urls)

    wg.Wait()

    fmt.Printf("\r%s", strings.Repeat(" ", 100))
    fmt.Fprintln(w, "\rTask completed.\n")
}

Popularity 1/10 Helpfulness 1/10 Language whatever

Source: stackoverflow.com

Tags: using whatever row each s3-bucket process

Share

Link to this answer
Share Copy Link

Contributed on Oct 06 2022

Pathi Prudhvi

0 Answers Avg Quality 2/10

how to process each row in s3 bucket using golang

Comment

0

func getDomains(path string) (lines []string, Error error) {

    file, err := os.Open(path)
    if err != nil {
        log.Fatalln(err)
    }

    defer file.Close()

    scanner := bufio.NewScanner(file)

    for scanner.Scan() {
        lines = append(lines, scanner.Text())
    }

    return lines, scanner.Err()
}

func Process(w http.ResponseWriter, r *http.Request) {


    urls := make(chan *Http, Threads*10)
    list, err := getDomains("**NEED A TEXT FILE FROM S3 HERE as an argument**")
    if err != nil {
        log.Fatalln(err)
    }

    var wg sync.WaitGroup
    for i := 0; i < Threads; i++ {
        wg.Add(1)
        go func() {
            for url := range urls {
                url.DNS()
            }

            wg.Done()
        }()
    }

    for i := 0; i < len(list); i++ {
        Progress := fmt.Sprintln(w, len(list))
        urls <- &Http{Url: list[i], Num: Progress}
    }

    close(urls)

    wg.Wait()

    fmt.Printf("\r%s", strings.Repeat(" ", 100))
    fmt.Fprintln(w, "\rTask completed.\n")
}

Popularity 1/10 Helpfulness 1/10 Language whatever

Source: stackoverflow.com

Tags: using whatever each process row s3-bucket

Share

Link to this answer
Share Copy Link

Contributed on Oct 06 2022

Pathi Prudhvi

0 Answers Avg Quality 2/10

how to process each row in s3 bucket using golang

Contents

More Related Answers

how to process each row in s3 bucket using golang

Closely Related Answers

how to process each row in s3 bucket using golang

how to process each row in s3 bucket using golang

how to process each row in s3 bucket using golang

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.