initial migration of bosh-s3cli patch proposal: support config from env

Author: schnoddelbotz

main.go

@@ -99,12 +99,15 @@ func main() {
 	// configure storage-cli config
 	common.InitConfig(parseLogLevel(*logLevel))
 
-	// check client config file exists
-	configFile, err := os.Open(*configPath)
-	if err != nil {
-		fatalLog("", err)
+	// try reading config file. if not provided, env will be tried as source.
+	var configFile *os.File
+	if *configPath != "" {
+		configFile, err := os.Open(*configPath)
+		if err != nil {
+			fatalLog("", err)
+		}
+		defer configFile.Close() //nolint:errcheck
 	}
-	defer configFile.Close() //nolint:errcheck
 
 	// create client
 	client, err := storage.NewStorageClient(*storageType, configFile)

s3/client/sdk.go

@@ -94,6 +94,9 @@ func NewAwsS3ClientWithApiOptions(
 			}
 			o.BaseEndpoint = aws.String(endpoint)
 		}
+		if c.Debug {
+			o.ClientLogMode = aws.LogResponse | aws.LogRequest
+		}
 		// Apply custom middlewares if provided
 		o.APIOptions = append(o.APIOptions, apiOptions...)
 	})

s3/config/config.go

@@ -1,10 +1,13 @@
 package config
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"os"
+	"strconv"
 	"strings"
 )
 
@@ -27,6 +30,7 @@ type S3Cli struct {
 	HostStyle                                 bool   `json:"host_style"`
 	SwiftAuthAccount                          string `json:"swift_auth_account"`
 	SwiftTempURLKey                           string `json:"swift_temp_url_key"`
+	Debug                                     bool   `json:"debug"`
 	RequestChecksumCalculationEnabled         bool
 	ResponseChecksumCalculationEnabled        bool
 	UploaderRequestChecksumCalculationEnabled bool
@@ -74,6 +78,44 @@ func newStaticCredentialsPresentError(desiredSource string) error {
 	return errorStaticCredentialsPresent{credentialsSource: desiredSource}
 }
 
+// NewReader provides an io.Reader on given configFile, using environment as fall-back.
+func NewReader(configFile *os.File) (io.Reader, error) {
+	if configFile != nil {
+		return configFile, nil
+	}
+	// Try reading config from env if no config file handle was provided
+	port := 443
+	if altPort, isset := os.LookupEnv("S3_PORT"); isset {
+		var err error
+		port, err = strconv.Atoi(altPort)
+		if err != nil {
+			return nil, err
+		}
+	}
+	c := S3Cli{
+		AccessKeyID: os.Getenv("S3_ACCESS_KEY_ID"),
+		BucketName:  os.Getenv("S3_BUCKET_NAME"),
+		// Fixate CredentialSource to StaticCredentialsSource, making S3_ACCESS_KEY_ID & S3_SECRET_ACCESS_KEY required
+		CredentialsSource: StaticCredentialsSource,
+		Host:              os.Getenv("S3_HOST"),
+		Port:              port,
+		Region:            os.Getenv("S3_REGION"),
+		SecretAccessKey:   os.Getenv("S3_SECRET_ACCESS_KEY"),
+		SSLVerifyPeer:     !(os.Getenv("S3_INSECURE_SSL") != ""),
+		// Use SSL/TLS/https:// unless S3_DISABLE_SSL is set
+		UseSSL: !(os.Getenv("S3_DISABLE_SSL") != ""),
+		// Use PathStyle=true by default (endpoint/bucket instead of DNS bucket.endpoint), if S3_USE_HOSTSTYLE unset. See client/sdk.go
+		HostStyle: os.Getenv("S3_USE_HOSTSTYLE") != "",
+		// Enable HTTP(S) request debugging if S3_DEBUG has non-empty value
+		Debug: os.Getenv("S3_DEBUG") != "",
+	}
+	json, err := json.Marshal(c)
+	if err != nil {
+		return nil, err
+	}
+	return bytes.NewReader(json), nil
+}
+
 // NewFromReader returns a new s3cli configuration struct from the contents of reader.
 // reader.Read() is expected to return valid JSON
 func NewFromReader(reader io.Reader) (S3Cli, error) {

storage/factory.go

@@ -72,7 +72,12 @@ var newGcsClient = func(configFile *os.File) (Storager, error) {
 }
 
 var newS3Client = func(configFile *os.File) (Storager, error) {
-	s3Config, err := s3config.NewFromReader(configFile)
+	configReader, err := s3config.NewReader(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	s3Config, err := s3config.NewFromReader(configReader)
 	if err != nil {
 		return nil, err
 	}