xxxxxxxxxx
package s3demo
import (
"os"
"context"
"fmt"
"io/ioutil"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/awserr"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func GetObjectWithV2SDKDemo() {
bucket := "YOUR_BUCKET"
key := "YOUR_OBJECT_KEY"
fileName := "YOUR_FILE_PATH"
// may need AWS_PROFILE and AWS_REGION populated as environment variables
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
panic("failed to load config, " + err.Error())
}
svc := s3.New(cfg)
ctx := context.Background()
req := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
resp, err := req.Send(ctx)
if err != nil {
panic(err)
}
s3objectBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
// create file
f, err := os.Create(fileName)
defer f.Close()
if err != nil {
panic(err)
}
bytesWritten, err := f.Write(s3objectBytes)
if err != nil {
panic(err)
}
fmt.Printf("Fetched %d bytes for S3Object\n", bytesWritten)
fmt.Printf("successfully downloaded data from %s/%s\n to file %s", bucket, key, fileName)
}
xxxxxxxxxx
package s3demo
import (
"os"
"context"
"fmt"
"io/ioutil"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/awserr"
"github.com/aws/aws-sdk-go-v2/aws/external"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func GetObjectWithV2SDKDemo() {
bucket := "YOUR_BUCKET"
key := "YOUR_OBJECT_KEY"
fileName := "YOUR_FILE_PATH"
// may need AWS_PROFILE and AWS_REGION populated as environment variables
cfg, err := external.LoadDefaultAWSConfig()
if err != nil {
panic("failed to load config, " + err.Error())
}
svc := s3.New(cfg)
ctx := context.Background()
req := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
resp, err := req.Send(ctx)
if err != nil {
panic(err)
}
s3objectBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
// create file
f, err := os.Create(fileName)
defer f.Close()
if err != nil {
panic(err)
}
bytesWritten, err := f.Write(s3objectBytes)
if err != nil {
panic(err)
}
fmt.Printf("Fetched %d bytes for S3Object\n", bytesWritten)
fmt.Printf("successfully downloaded data from %s/%s\n to file %s", bucket, key, fileName)
}
xxxxxxxxxx
func getDomains(path string) (lines []string, Error error) {
file, err := os.Open(path)
if err != nil {
log.Fatalln(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
func Process(w http.ResponseWriter, r *http.Request) {
urls := make(chan *Http, Threads*10)
list, err := getDomains("**NEED A TEXT FILE FROM S3 HERE as an argument**")
if err != nil {
log.Fatalln(err)
}
var wg sync.WaitGroup
for i := 0; i < Threads; i++ {
wg.Add(1)
go func() {
for url := range urls {
url.DNS()
}
wg.Done()
}()
}
for i := 0; i < len(list); i++ {
Progress := fmt.Sprintln(w, len(list))
urls <- &Http{Url: list[i], Num: Progress}
}
close(urls)
wg.Wait()
fmt.Printf("\r%s", strings.Repeat(" ", 100))
fmt.Fprintln(w, "\rTask completed.\n")
}
xxxxxxxxxx
func getDomains(path string) (lines []string, Error error) {
file, err := os.Open(path)
if err != nil {
log.Fatalln(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
func Process(w http.ResponseWriter, r *http.Request) {
urls := make(chan *Http, Threads*10)
list, err := getDomains("**NEED A TEXT FILE FROM S3 HERE as an argument**")
if err != nil {
log.Fatalln(err)
}
var wg sync.WaitGroup
for i := 0; i < Threads; i++ {
wg.Add(1)
go func() {
for url := range urls {
url.DNS()
}
wg.Done()
}()
}
for i := 0; i < len(list); i++ {
Progress := fmt.Sprintln(w, len(list))
urls <- &Http{Url: list[i], Num: Progress}
}
close(urls)
wg.Wait()
fmt.Printf("\r%s", strings.Repeat(" ", 100))
fmt.Fprintln(w, "\rTask completed.\n")
}