Ax333l/s3-resource-simple
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Support aws executable options

Browse source
This commit is contained in:
Troy Kinsella 2021-03-24 17:57:12 -07:00
parent 1651f9acfd
commit 098d73bb99
4 changed files with 12 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
README.md
View file

@ -21,6 +21,7 @@ resources:
secret_access_key: ((aws_secret_key)) # Optional secret_access_key: ((aws_secret_key)) # Optional
bucket: my-bucket bucket: my-bucket
region: us-east-1 # Optional region: us-east-1 # Optional
aws_options: "--endpoint-url https://objects-us-east-1.dream.io" # Optional
jobs: jobs:
- name: publish-files - name: publish-files
@ -50,8 +51,8 @@ Given the following directory `test`:
``` ```
test test
├── results ├── results
│   ├── 1.json │ ├── 1.json
│   └── 2.json │ └── 2.json
└── scripts └── scripts
└── bad.sh └── bad.sh
``` ```
@ -64,6 +65,9 @@ options:
- "--include 'results/*'" - "--include 'results/*'"
``` ```
The `aws_options` source parameter supplies options to the `aws` executable
(whereas the `options` parameter supplies options to the `aws sync` command.)
### Region ### Region
Interacting with some AWS regions (like London) requires AWS Signature Version Interacting with some AWS regions (like London) requires AWS Signature Version
4. This options allows you to explicitly specify region where your bucket is 4. This options allows you to explicitly specify region where your bucket is

3
assets/check
View file

@ -7,10 +7,11 @@ source "$(dirname $0)/common.sh"
payload=`cat` payload=`cat`
bucket=$(get_bucket) bucket=$(get_bucket)
prefix=$(get_prefix) prefix=$(get_prefix)
aws_options=$(echo "$payload" | jq -r '.source.aws_options // [] | join(" ")')
export_aws_vars export_aws_vars
# Consider the most recent LastModified timestamp as the most recent version. # Consider the most recent LastModified timestamp as the most recent version.
timestamps=$(aws s3api list-objects --bucket $bucket --prefix "$prefix" --query 'Contents[].{LastModified: LastModified}') timestamps=$(aws $aws_options s3api list-objects --bucket $bucket --prefix "$prefix" --query 'Contents[].{LastModified: LastModified}')
recent="$(echo $timestamps | jq -r 'max_by(.LastModified)')" recent="$(echo $timestamps | jq -r 'max_by(.LastModified)')"
echo "[$recent]" echo "[$recent]"

3
assets/in
View file

@ -17,6 +17,7 @@ source "$(dirname $0)/common.sh"
payload=`cat` payload=`cat`
bucket=$(get_bucket) bucket=$(get_bucket)
prefix=$(get_prefix) prefix=$(get_prefix)
aws_options=$(echo "$payload" | jq -r '.source.aws_options // [] | join(" ")')
options=$(echo "$payload" | jq -r '.params.options // [] | join(" ")') options=$(echo "$payload" | jq -r '.params.options // [] | join(" ")')
skip_download=$(echo "$payload" | jq -r '.params.skip_download // false') skip_download=$(echo "$payload" | jq -r '.params.skip_download // false')
@ -26,7 +27,7 @@ if [ "$skip_download" = "true" ]; then
echo "Skipping download from S3" echo "Skipping download from S3"
else else
echo "Downloading from S3..." echo "Downloading from S3..."
eval aws s3 sync "s3://$bucket/$prefix" $dest $options eval aws $aws_options s3 sync "s3://$bucket/$prefix" $dest $options
echo "Done." echo "Done."
fi fi

3
assets/out
View file

@ -17,6 +17,7 @@ source "$(dirname $0)/common.sh"
payload=`cat` payload=`cat`
bucket=$(get_bucket) bucket=$(get_bucket)
prefix=$(get_prefix) prefix=$(get_prefix)
aws_options=$(echo "$payload" | jq -r '.source.aws_options // [] | join(" ")')
options=$(echo "$payload" | jq -r '.params.options // [] | join(" ")') options=$(echo "$payload" | jq -r '.params.options // [] | join(" ")')
source_dir=$(echo "$payload" | jq -r '.params.source_dir // "."') source_dir=$(echo "$payload" | jq -r '.params.source_dir // "."')
test -z "$source_dir" && { echo "Must supply params.source_dir" >&2; exit 1; } test -z "$source_dir" && { echo "Must supply params.source_dir" >&2; exit 1; }
@ -27,7 +28,7 @@ cd $source/$source_dir
bucket_uri="s3://$bucket/$prefix" bucket_uri="s3://$bucket/$prefix"
echo "Uploading '$source_dir' to '$bucket_uri'..." echo "Uploading '$source_dir' to '$bucket_uri'..."
eval aws s3 sync $source/$source_dir $bucket_uri $options eval aws $aws_options s3 sync $source/$source_dir $bucket_uri $options
echo "Done." echo "Done."
emit_version emit_version