Accept options for sync in pipeline.yml

Closes #1
Ref #3

README.md

@@ -19,6 +19,7 @@ resources:
     access_key_id: {{aws-access-key}}
     secret_access_key: {{aws-secret-key}}
     bucket: {{aws-bucket}}
+    options: [<optional, see note below>]
 jobs:
 - name: <job name>
   plan:
@@ -26,6 +27,30 @@ jobs:
   - put: <resource name>
 ```
 
+### Options
+The `options` parameter is synonymous with the options that `aws cli` accepts for `sync`.
+Please see
+[S3 Sync Options](http://docs.aws.amazon.com/cli/latest/reference/s3/sync.html#options)
+And pay special attention to the
+[Use of Exclude and Include Filters](http://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters)
+
+Given the following directory `test` we can upload _only_ the `results` directory by using the `options` JSON below.
+
+```
+test
+├── results
+│   ├── 1.json
+│   └── 2.json
+└── scripts
+    └── bad.sh
+```
+```
+options: [
+  "--exclude: '*'",
+  "--include: 'results/*'"
+]
+```
+
 ## Development
 
 Requires [Docker](https://www.docker.com/).

assets/in

@@ -1,8 +1,6 @@
 #!/bin/sh
 
-# http://concourse.ci/implementing-resources.html#in:-fetch-a-given-resource.
-
-# https://github.com/concourse/git-resource/blob/6fcfbd4/assets/out#L4-L16
+# Resource Impl: http://concourse.ci/implementing-resources.html#in:-fetch-a-given-resource
 set -e
 
 exec 3>&1 # make stdout available as fd 3 for the result
@@ -19,17 +17,14 @@ fi
 # parse incoming config data
 payload=`cat`
 bucket=$(echo "$payload" | jq -r '.source.bucket')
+options=$(echo "$payload" | jq -r '.source.options | join(" ")')
 
-
-echo "Downloading from S3..."
-
-# http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment
+# export for `aws` cli
 export AWS_ACCESS_KEY_ID=$(echo "$payload" | jq -r '.source.access_key_id')
 export AWS_SECRET_ACCESS_KEY=$(echo "$payload" | jq -r '.source.secret_access_key')
 
-aws s3 sync "s3://$bucket" $dest
-
+echo "Downloading from S3..."
+eval aws s3 sync "s3://$bucket" $dest $options
 echo "...done."
 
-
 source "$(dirname $0)/emit.sh" >&3

assets/out

@@ -1,8 +1,6 @@
 #!/bin/sh
 
-# http://concourse.ci/implementing-resources.html#out:-update-a-resource.
-
-# https://github.com/concourse/git-resource/blob/6fcfbd4/assets/out#L4-L16
+# Resource Impl: http://concourse.ci/implementing-resources.html#out:-update-a-resource.
 set -e
 
 exec 3>&1 # make stdout available as fd 3 for the result
@@ -11,30 +9,22 @@ exec 1>&2 # redirect all output to stderr for logging
 source=$1
 
 if [ -z "$source" ]; then
-  echo "usage: $0 <path/to/volume>"
+  echo "usage: $0 <full/path/to/dir>"
   exit 1
 fi
 #######################################
 
-
 # parse incoming config data
 payload=`cat`
 bucket=$(echo "$payload" | jq -r '.source.bucket')
+options=$(echo "$payload" | jq -r '.source.options | join(" ")')
 
-
-echo "Uploading to S3..."
-
-# http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment
+# export for `aws` cli
 export AWS_ACCESS_KEY_ID=$(echo "$payload" | jq -r '.source.access_key_id')
 export AWS_SECRET_ACCESS_KEY=$(echo "$payload" | jq -r '.source.secret_access_key')
 
-# TODO: Get options from the config.
-# http://docs.aws.amazon.com/cli/latest/reference/s3/sync.html#options
-# http://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters
-
-aws s3 sync $source "s3://$bucket"
-
+echo "Uploading to S3..."
+eval aws s3 sync $source "s3://$bucket" $options
 echo "...done."
 
-
 source "$(dirname $0)/emit.sh" >&3

config.example.json

@@ -2,6 +2,11 @@
   "source": {
     "access_key_id": "",
     "secret_access_key": "",
-    "bucket": ""
+    "bucket": "",
+    "options": [
+      "--dryrun",
+      "--exclude '*'",
+      "--include 'results/*'"
+    ]
   }
 }