Support aws executable options

README.md

@@ -21,6 +21,7 @@ resources:
     secret_access_key: ((aws_secret_key)) # Optional
     bucket: my-bucket
     region: us-east-1 # Optional
+    aws_options: "--endpoint-url https://objects-us-east-1.dream.io" # Optional
 
 jobs:
 - name: publish-files
@@ -50,8 +51,8 @@ Given the following directory `test`:
 ```
 test
 ├── results
-│   ├── 1.json
-│   └── 2.json
+│   ├── 1.json
+│   └── 2.json
 └── scripts
     └── bad.sh
 ```
@@ -64,6 +65,9 @@ options:
 - "--include 'results/*'"
 ```
 
+The `aws_options` source parameter supplies options to the `aws` executable 
+(whereas the `options` parameter supplies options to the `aws sync` command.)
+
 ### Region
 Interacting with some AWS regions (like London) requires AWS Signature Version
 4. This options allows you to explicitly specify region where your bucket is

assets/check

@@ -7,10 +7,11 @@ source "$(dirname $0)/common.sh"
 payload=`cat`
 bucket=$(get_bucket)
 prefix=$(get_prefix)
+aws_options=$(echo "$payload" | jq -r '.source.aws_options // [] | join(" ")')
 
 export_aws_vars
 
 # Consider the most recent LastModified timestamp as the most recent version.
-timestamps=$(aws s3api list-objects --bucket $bucket --prefix "$prefix" --query 'Contents[].{LastModified: LastModified}')
+timestamps=$(aws $aws_options s3api list-objects --bucket $bucket --prefix "$prefix" --query 'Contents[].{LastModified: LastModified}')
 recent="$(echo $timestamps | jq -r 'max_by(.LastModified)')"
 echo "[$recent]"

assets/in

@@ -17,6 +17,7 @@ source "$(dirname $0)/common.sh"
 payload=`cat`
 bucket=$(get_bucket)
 prefix=$(get_prefix)
+aws_options=$(echo "$payload" | jq -r '.source.aws_options // [] | join(" ")')
 options=$(echo "$payload" | jq -r '.params.options // [] | join(" ")')
 skip_download=$(echo "$payload" | jq -r '.params.skip_download // false')
 
@@ -26,7 +27,7 @@ if [ "$skip_download" = "true" ]; then
   echo "Skipping download from S3"
 else
   echo "Downloading from S3..."
-  eval aws s3 sync "s3://$bucket/$prefix" $dest $options
+  eval aws $aws_options s3 sync "s3://$bucket/$prefix" $dest $options
   echo "Done."
 fi
 

assets/out

@@ -17,6 +17,7 @@ source "$(dirname $0)/common.sh"
 payload=`cat`
 bucket=$(get_bucket)
 prefix=$(get_prefix)
+aws_options=$(echo "$payload" | jq -r '.source.aws_options // [] | join(" ")')
 options=$(echo "$payload" | jq -r '.params.options // [] | join(" ")')
 source_dir=$(echo "$payload" | jq -r '.params.source_dir // "."')
 test -z "$source_dir" && { echo "Must supply params.source_dir" >&2; exit 1; }
@@ -27,7 +28,7 @@ cd $source/$source_dir
 
 bucket_uri="s3://$bucket/$prefix"
 echo "Uploading '$source_dir' to '$bucket_uri'..."
-eval aws s3 sync $source/$source_dir $bucket_uri $options
+eval aws $aws_options s3 sync $source/$source_dir $bucket_uri $options
 echo "Done."
 
 emit_version