s3cmd for ceph

# get python3 and pip3 installed on your linux or macbook

# then install s3cmd
pip3 install s3cmd

# next overwrite your ~/.s3cfg file and point at your ceph bucket

cat << 'EOF' > ~/.s3cfg
[default]
access_key = <REPLACE WITH YOUR ACCESS KEY>
secret_key = <REPLACE WITH YOUR SECRET>
host_base = <REPLACE WITH YOUR HOST>
host_bucket = <REPLACE WITH YOUR HOST>
access_token =
add_encoding_exts =
add_headers =
ca_certs_file =
cache_file =
check_ssl_certificate = True
check_ssl_hostname = True
cloudfront_host = cloudfront.amazonaws.com
default_mime_type = binary/octet-stream
delay_updates = False
delete_after = False
delete_after_fetch = False
delete_removed = False
dry_run = False
enable_multipart = True
encoding = UTF-8
encrypt = False
expiry_date =
expiry_days =
expiry_prefix =
follow_symlinks = False
force = False
get_continue = False
gpg_command = /bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase =
guess_mime_type = True
human_readable_sizes = False
invalidate_default_index_on_cf = False
invalidate_default_index_root_on_cf = True
invalidate_on_cf = False
kms_key =
limit = -1
limitrate = 0
list_md5 = False
log_target_prefix =
long_listing = False
max_delete = -1
mime_type =
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
preserve_attrs = True
progress_meter = True
proxy_host =
proxy_port = 0
put_continue = False
recursive = False
recv_chunk = 65536
reduced_redundancy = False
requester_pays = False
restore_days = 1
restore_priority = Standard
send_chunk = 65536
server_side_encryption = False
signature_v2 = False
signurl_use_https = False
simpledb_host = sdb.amazonaws.com
skip_existing = False
socket_timeout = 300
stats = False
stop_on_error = False
storage_class =
urlencoding_mode = normal
use_http_expect = False
use_https = True
use_mime_magic = True
verbosity = WARNING
website_error =
website_index = index.html
EOF

# now search all your buckets for something
export job_id=someUniqueTextToSearchFor
for i in $(s3cmd ls | awk '{print $3}'); do s3cmd ls "$i" | grep $job_id; done

# pro tip, put this in your ~/.bash_profile:
alias s3jobid='for i in $(s3cmd ls | awk '"'"'{print $3}'"'"'); do s3cmd ls "$i" | grep $job_id; done'

# and then you can search like this:
export job_id=foobar; s3jobid

Leave a Comment