Category Archives: Go

Golang with decoupling S3 Buckets and clients

// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX - License - Identifier: Apache - 2.0
package main

import (
"context"
"fmt"
"strconv"


"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"

)

func main() {
cfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
panic("configuration error, " + err.Error())
}


client := s3.NewFromConfig(cfg)
// ListBuckets(client, cfg)
// BucketLocation(client, cfg.Region, "dnb-vedi")
// BucketLocation(client, cfg.Region, "ag-glue-shaping-test-code")
// cfg.Region = "us-east-1"
// client = s3.NewFromConfig(cfg)
// ListObjects(client, cfg, "dnb-vedi", "2022-06-10")
// ListObjects(client, cfg,  , "")
// if IsObjectExistsInS3Bucket(client, cfg, "ag-glue-shaping-test-code", "experiment-data/top50%_caac_fake_entity_newHash.csv") {
//  fmt.Println("%s exists %s", "ag-glue-shaping-test-code", "experiment-data/top50%_caac_fake_entity_newHash.csv")
// } else {
//  fmt.Println("%s not exists %s", "ag-glue-shaping-test-code", "experiment-data/top50%_caac_fake_entity_newHash.csv")
// }

// if IsObjectExistsInS3Bucket(client, cfg, "dnb-vedi", "2022-06-10/BEMFAB_FILE1.TXT.gz.pgp") {
//  fmt.Println("%s exists %s", "dnb-vedi", "2022-06-10/BEMFAB_FILE1.TXT.gz.pgp")
// } else {
//  fmt.Println("%s not exists %s", "dnb-vedi", "2022-06-10/BEMFAB_FILE1.TXT.gz.pgp")
// }

// if IsObjectExistsInS3Bucket(client, cfg, "dnb-vedi", "2022-06-10/") {
//  fmt.Println("%s exists %s", "dnb-vedi", "2022-06-10/")
// } else {
//  fmt.Println("%s not exists %s", "dnb-vedi", "2022-06-10")
// }
items, err := GetObjects(client, cfg, "dnb-vedi", "2022-06-10")
if err != nil {
    panic("configuration error, " + err.Error())
}
for _, item := range items {
    for k, v := range item {
        fmt.Printf("%s: %s ", k, v)
    }
    fmt.Println()
}

}

func IsObjectExistsInS3Bucket(client *s3.Client, cfg aws.Config, bucket, objectKey string) bool {
region, _ := GetBucketLocation(client, cfg, bucket)
cfg.Region = region
client = s3.NewFromConfig(cfg)


headObj := s3.HeadObjectInput{
    Bucket: aws.String(bucket),
    Key:    aws.String(objectKey),
}
_, err := client.HeadObject(context.TODO(), &headObj)
return err == nil

}

func GetObjects(client *s3.Client, cfg aws.Config, bucket, prefix string) ([]map[string]string, error) {
region, _ := GetBucketLocation(client, cfg, bucket)
cfg.Region = region
// fmt.Println(cfg.Region)
client = s3.NewFromConfig(cfg)
params := &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),
}


resp, err := client.ListObjectsV2(context.Background(), params)
if err != nil {
    fmt.Println("Got error retrieving list of objects:")
    return nil, err
}
var returnArray []map[string]string
for _, item := range resp.Contents {
    var obj = map[string]string{}
    obj["path"] = string(*item.Key)
    obj["lastmodified"] = (*item.LastModified).Format("2006-01-02T15:04:05")
    obj["size"] = strconv.FormatInt(item.Size, 10)
    obj["storageClass"] = string(item.StorageClass)
    returnArray = append(returnArray, obj)
}
return returnArray, nil

}

func ListObjects(client *s3.Client, cfg aws.Config, bucket, prefix string) {
region, _ := GetBucketLocation(client, cfg, bucket)
cfg.Region = region
// fmt.Println(cfg.Region)
client = s3.NewFromConfig(cfg)
params := &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),
}


resp, err := client.ListObjectsV2(context.Background(), params)
if err != nil {
    fmt.Println("Got error retrieving list of objects:")
    fmt.Println(err)
    return
}
for _, item := range resp.Contents {
    fmt.Println("Name:          ", *item.Key)
    fmt.Println("Last modified: ", *item.LastModified)
    fmt.Println("Size:          ", item.Size)
    fmt.Println("Storage class: ", item.StorageClass)
    fmt.Println("")
}

}

func ListBuckets(client *s3.Client, cfg aws.Config) {


param := &s3.ListBucketsInput{}

result, err := client.ListBuckets(context.TODO(), param)
if err != nil {
    fmt.Println("Got an error retrieving buckets:")
    fmt.Println(err)
    return
}

fmt.Println("Buckets:")
for _, bucket := range result.Buckets {
    fmt.Print(*bucket.Name + ": " + bucket.CreationDate.Format("2006-01-02 15:04:05 Monday") + ": ")
    re, _ := GetBucketLocation(client, cfg, *bucket.Name)
    fmt.Println(re)
}

}

func GetBucketLocation(client *s3.Client, cfg aws.Config, bucket string) (string, error) {
param := &s3.GetBucketLocationInput{
Bucket: aws.String(bucket),
}
locationOutput, err := client.GetBucketLocation(context.Background(), param)
if err != nil {
return "", err
}
// fmt.Println(locationOutput.LocationConstraint)
if locationOutput.LocationConstraint == "" {
return cfg.Region, nil
} else {
return string(locationOutput.LocationConstraint), nil
}
}