AWS CLI S3コマンドメモ

参考

aws s3 help
aws s3 機能 help
aws s3 syncするシェルスクリプトでワイルドカードでexcludeしたときのメモ - Qiita
→excludeが意図した通りに動かず、はまる。。。上記URLで解消・感謝！！！
　

最初にやったこと

・ec2ユーザーにロール割り当て

・ec2インスタンスをロールをつけて起動(Amazon Linux)
・ec2インスタンスにSSHログイン(ec2-user)
　

動作確認スクリプト

#!/bin/bash

S3="aws s3"
BN=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1)
TF="./test"
TFN=$(basename $TF)
TD="./testdir"
TDN="$(basename ./testdir)"

com_exec() {
  MES="$1"
  COM="$2"
  echo "########## $MES"
  echo "[command]"
  echo "$COM"
  echo  ""
  echo "[output]"
  eval "$COM"
  echo ""
  echo ""
}

rm -rf $TF $TD

com_exec "create s3 bucket" "$S3 mb s3://$BN"
com_exec "check createed s3 bucket" "$S3 ls s3://$BN"
com_exec "make local file" "touch $TF"
com_exec "upload local file" "$S3 cp $TF s3://$BN/$TFN"
com_exec "check uploaded s3 file" "$S3 ls --recursive s3://$BN"
com_exec "download uploaded s3 file" "$S3 cp s3://$BN/$TFN $TF-dl"
com_exec "check downloaded local file" "ls $TF-dl"
com_exec "remove s3 file" "$S3 rm s3://$BN/$TFN"
com_exec "make local directory" "mkdir -p $TD"
com_exec "mv local file" "mv $TF-dl $TD/."
com_exec "upload local directory" "$S3 cp --recursive $TD s3://$BN/$TDN"
com_exec "check uploaded s3 directory" "$S3 ls --recursive s3://$BN"
com_exec "remove local directory" "rm -rf $TD"
com_exec "download s3 directory" "$S3 cp --recursive s3://$BN/$TDN $TD"
com_exec "check downloaded local directory" "find $TD"
com_exec "make local file" "touch $TD/$TF-2"
com_exec "sync local direcotry to s3" "$S3 sync $TD s3://$BN/$TDN"
com_exec "remove local file" "rm $TD/$TF-2"
com_exec "sync local direcotry to s3 with --delete" "$S3 sync --delete $TD s3://$BN/$TDN"
com_exec "make local file" "touch $TD/$TF-3 $TD/$TF-4"
com_exec "sync local direcotry to s3 with --exclude" "$S3 sync --exclude '$(basename $TF-3)' $TD s3://$BN/$TDN"
com_exec "remove s3 bucket" "$S3 rb --force s3://$BN"

動作確認スクリプト出力結果

########## create s3 bucket
[command]
aws s3 mb s3://3bihbmwqlzmh7evd

[output]
make_bucket: 3bihbmwqlzmh7evd


########## check createed s3 bucket
[command]
aws s3 ls s3://3bihbmwqlzmh7evd

[output]


########## make local file
[command]
touch ./test

[output]


########## upload local file
[command]
aws s3 cp ./test s3://3bihbmwqlzmh7evd/test

[output]
upload: ./test to s3://3bihbmwqlzmh7evd/test


########## check uploaded s3 file
[command]
aws s3 ls --recursive s3://3bihbmwqlzmh7evd

[output]
2018-11-04 03:45:48          0 test


########## download uploaded s3 file
[command]
aws s3 cp s3://3bihbmwqlzmh7evd/test ./test-dl

[output]
download: s3://3bihbmwqlzmh7evd/test to ./test-dl


########## check downloaded local file
[command]
ls ./test-dl

[output]
./test-dl


########## remove s3 file
[command]
aws s3 rm s3://3bihbmwqlzmh7evd/test

[output]
delete: s3://3bihbmwqlzmh7evd/test


########## make local directory
[command]
mkdir -p ./testdir

[output]


########## mv local file
[command]
mv ./test-dl ./testdir/.

[output]


########## upload local directory
[command]
aws s3 cp --recursive ./testdir s3://3bihbmwqlzmh7evd/testdir

[output]
upload: testdir/test-dl to s3://3bihbmwqlzmh7evd/testdir/test-dl


########## check uploaded s3 directory
[command]
aws s3 ls --recursive s3://3bihbmwqlzmh7evd

[output]
2018-11-04 03:45:50          0 testdir/test-dl


########## remove local directory
[command]
rm -rf ./testdir

[output]


########## download s3 directory
[command]
aws s3 cp --recursive s3://3bihbmwqlzmh7evd/testdir ./testdir

[output]
download: s3://3bihbmwqlzmh7evd/testdir/test-dl to testdir/test-dl


########## check downloaded local directory
[command]
find ./testdir

[output]
./testdir
./testdir/test-dl


########## make local file
[command]
touch ./testdir/./test-2

[output]


########## sync local direcotry to s3
[command]
aws s3 sync ./testdir s3://3bihbmwqlzmh7evd/testdir

[output]
upload: testdir/test-2 to s3://3bihbmwqlzmh7evd/testdir/test-2


########## remove local file
[command]
rm ./testdir/./test-2

[output]


########## sync local direcotry to s3 with --delete
[command]
aws s3 sync --delete ./testdir s3://3bihbmwqlzmh7evd/testdir

[output]
delete: s3://3bihbmwqlzmh7evd/testdir/test-2


########## make local file
[command]
touch ./testdir/./test-3 ./testdir/./test-4

[output]


########## sync local direcotry to s3 with --exclude
[command]
aws s3 sync --exclude 'test-3' ./testdir s3://3bihbmwqlzmh7evd/testdir

[output]
upload: testdir/test-4 to s3://3bihbmwqlzmh7evd/testdir/test-4


########## remove s3 bucket
[command]
aws s3 rb --force s3://3bihbmwqlzmh7evd

[output]
delete: s3://3bihbmwqlzmh7evd/testdir/test-4
delete: s3://3bihbmwqlzmh7evd/testdir/test-dl
remove_bucket: 3bihbmwqlzmh7evd