| /trunk/src/gsutil r12 | /trunk/src/gsutil r13 | ||
| 1 | #!/usr/bin/env python | 1 | #!/usr/bin/env python |
|---|---|---|---|
| 2 | # | 2 | # |
| 3 | # Copyright 2010 Google Inc. | 3 | # Copyright 2010 Google Inc. |
| 4 | # | 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. | 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at | 7 | # You may obtain a copy of the License at |
| 8 | # | 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # | 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software | 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, | 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. | 15 | # limitations under the License. |
| 16 | 16 | ||
| 17 | """Google Storage command line tool.""" | 17 | """Google Storage command line tool.""" |
| 18 | 18 | ||
| 19 | import datetime | 19 | import datetime |
| 20 | import fnmatch | 20 | import fnmatch |
| 21 | import getopt | 21 | import getopt |
| 22 | import glob | 22 | import glob |
| 23 | import mimetypes | ||
| 23 | import os | 24 | import os |
| 24 | import platform | 25 | import platform |
| 25 | import re | 26 | import re |
| 26 | import shutil | 27 | import shutil |
| 27 | import signal | 28 | import signal |
| 28 | import stat | 29 | import stat |
| 29 | import sys | 30 | import sys |
| 30 | import tarfile | 31 | import tarfile |
| 31 | import tempfile | 32 | import tempfile |
| 32 | import xml.dom.minidom | 33 | import xml.dom.minidom |
| 33 | 34 | ||
| 34 | 35 | ||
| 35 | def OutputAndExit(message): | 36 | def OutputAndExit(message): |
| 36 | sys.stderr.write('%s\n' % message) | 37 | sys.stderr.write('%s\n' % message) |
| 37 | sys.exit(1) | 38 | sys.exit(1) |
| 38 | 39 | ||
| 39 | 40 | ||
| 40 | # Before importing boto, find where gsutil is installed and include its | 41 | # Before importing boto, find where gsutil is installed and include its |
| 41 | # boto sub-directory at the start of the PYTHONPATH, to ensure the versions of | 42 | # boto sub-directory at the start of the PYTHONPATH, to ensure the versions of |
| 42 | # gsutil and boto stay in sync after software updates. This also allows gsutil | 43 | # gsutil and boto stay in sync after software updates. This also allows gsutil |
| 43 | # to be used without explicitly adding it to the PYTHONPATH. | 44 | # to be used without explicitly adding it to the PYTHONPATH. |
| 44 | # We use realpath() below to unwind symlinks if any were used in the gsutil | 45 | # We use realpath() below to unwind symlinks if any were used in the gsutil |
| 45 | # installation. | 46 | # installation. |
| 46 | gsutil_bin_dir = os.path.realpath(os.path.dirname(sys.argv[0])) | 47 | gsutil_bin_dir = os.path.realpath(os.path.dirname(sys.argv[0])) |
| 47 | if not gsutil_bin_dir: | 48 | if not gsutil_bin_dir: |
| 48 | OutputAndExit('Unable to determine where gsutil is installed. Sorry, ' | 49 | OutputAndExit('Unable to determine where gsutil is installed. Sorry, ' |
| 49 | 'cannot run correctly without this.\n') | 50 | 'cannot run correctly without this.\n') |
| 50 | boto_lib_dir = gsutil_bin_dir + os.sep + 'boto' | 51 | boto_lib_dir = gsutil_bin_dir + os.sep + 'boto' |
| 51 | if not os.path.isdir(boto_lib_dir): | 52 | if not os.path.isdir(boto_lib_dir): |
| 52 | OutputAndExit('There is no boto library under the gsutil install directory ' | 53 | OutputAndExit('There is no boto library under the gsutil install directory ' |
| 53 | '(%s).\nThe gsutil command cannot work properly when installed ' | 54 | '(%s).\nThe gsutil command cannot work properly when installed ' |
| 54 | 'this way.\nPlease re-install gsutil per the installation ' | 55 | 'this way.\nPlease re-install gsutil per the installation ' |
| 55 | 'instructions.' % gsutil_bin_dir) | 56 | 'instructions.' % gsutil_bin_dir) |
| 56 | if 'PYTHONPATH' not in os.environ: | 57 | if 'PYTHONPATH' not in os.environ: |
| 57 | os.environ['PYTHONPATH'] = boto_lib_dir | 58 | os.environ['PYTHONPATH'] = boto_lib_dir |
| 58 | else: | 59 | else: |
| 59 | os.environ['PYTHONPATH'] = boto_lib_dir + ':' + os.getenv('PYTHONPATH') | 60 | os.environ['PYTHONPATH'] = boto_lib_dir + ':' + os.getenv('PYTHONPATH') |
| 60 | 61 | ||
| 61 | import boto | 62 | import boto |
| 63 | boto.UserAgent = boto.UserAgent + '/gsutil' | ||
| 62 | from boto import handler | 64 | from boto import handler |
| 63 | from boto.exception import BotoClientError | 65 | from boto.exception import BotoClientError |
| 66 | from boto.exception import InvalidAclError | ||
| 64 | from boto.exception import InvalidUriError | 67 | from boto.exception import InvalidUriError |
| 65 | from boto.exception import S3ResponseError | 68 | from boto.exception import S3ResponseError |
| 66 | from boto.s3.acl import CannedACLStrings | ||
| 67 | from boto.s3.acl import Policy | ||
| 68 | 69 | ||
| 69 | usage_string = """ | 70 | usage_string = """ |
| 70 | SYNOPSIS | 71 | SYNOPSIS |
| 71 | gsutil [-d] command args | 72 | gsutil [-d] [-h header]... command args |
| 72 | 73 | ||
| 73 | -d option will show HTTP protocol detail. | 74 | -d option shows HTTP protocol detail. |
| 75 | |||
| 76 | -h option allows you to specify additional HTTP headers, for example: | ||
| 77 | gsutil -h "Cache-Control:public,max-age=3600" -h "Content-Type:gzip" cp * gs://bucket | ||
| 74 | 78 | ||
| 75 | Commands: | 79 | Commands: |
| 76 | Concatenate object content to stdout: | 80 | Concatenate object content to stdout: |
| 77 | cat [-h] uri1 [uri2...] | 81 | cat [-h] uri... |
| 82 | -h Prints short header for each object. | ||
| 78 | Copy objects: | 83 | Copy objects: |
| 79 | cp src_uri dst_uri | 84 | cp [-a canned_acl] [-t] src_uri dst_uri |
| 80 | - or - | 85 | - or - |
| 81 | cp uri1 [uri2 ...] dst_uri | 86 | cp [-a canned_acl] [-t] uri... dst_uri |
| 87 | -a Sets named canned_acl when uploaded objects created (list below). | ||
| 88 | -t Sets MIME type based on file extension. | ||
| 89 | Get ACL XML for a bucket or object (save and edit for "setacl" command): | ||
| 90 | getacl uri | ||
| 82 | List buckets or objects: | 91 | List buckets or objects: |
| 83 | ls [-l] uri1 [uri2...] | 92 | ls [-l] uri... |
| 93 | -l Prints long listing. | ||
| 84 | Make buckets: | 94 | Make buckets: |
| 85 | mb uri1 [uri2...] | 95 | mb uri... |
| 86 | Move/rename objects: | 96 | Move/rename objects: |
| 87 | mv src_uri dst_uri | 97 | mv src_uri dst_uri |
| 88 | - or - | 98 | - or - |
| 89 | mv uri1 [uri2 ...] dst_uri | 99 | mv uri... dst_uri |
| 90 | Remove buckets: | 100 | Remove buckets: |
| 91 | rb uri1 [uri2...] | 101 | rb uri... |
| 92 | Remove objects: | 102 | Remove objects: |
| 93 | rm uri1 [uri2...] | 103 | rm uri... |
| 94 | Set ACL on buckets and/or objects: | 104 | Set ACL on buckets and/or objects: |
| 95 | setacl <canned_acl_name> uri1 [uri2...] | 105 | setacl file-or-canned_acl_name uri... |
| 96 | canned_acl_name can be one of: "public-read", "public-read-write", | ||
| 97 | "authenticated-read", "bucket-owner-read", "bucket-owner-full-control" | ||
| 98 | 106 | ||
| 99 | Omitting URI scheme defaults to "file". For example, "dir/file.txt" is | 107 | Omitting URI scheme defaults to "file". For example, "dir/file.txt" is |
| 100 | equivalent to "file://dir/file.txt" | 108 | equivalent to "file://dir/file.txt" |
| 101 | 109 | ||
| 102 | URIs support object name wildcards, for example: | 110 | URIs support object name wildcards, for example: |
| 103 | gsutil cp gs://mybucket/[a-f]*.doc localdir | 111 | gsutil cp gs://mybucket/[a-f]*.doc localdir |
| 104 | 112 | ||
| 105 | Source directory or bucket names are implicitly wildcarded, so | 113 | Source directory or bucket names are implicitly wildcarded, so |
| 106 | gsutil cp localdir gs://mybucket | 114 | gsutil cp localdir gs://mybucket |
| 107 | will recursively copy localdir. | 115 | will recursively copy localdir. |
| 116 | |||
| 117 | canned_acl_name can be one of: "public-read", "public-read-write", | ||
| 118 | "authenticated-read", "bucket-owner-read", "bucket-owner-full-control" | ||
| 108 | """ | 119 | """ |
| 109 | 120 | ||
| 110 | 121 | ||
| 111 | def OutputUsageAndExit(): | 122 | def OutputUsageAndExit(): |
| 112 | sys.stderr.write(usage_string) | 123 | sys.stderr.write(usage_string) |
| 113 | sys.exit(0) | 124 | sys.exit(0) |
| 114 | 125 | ||
| 115 | 126 | ||
| 116 | def StorageUri(uri_str, disallow_object_name=False, debug=False): | 127 | def StorageUri(uri_str, disallow_object_name=False, debug=False): |
| 117 | """Instantiate boto.StorageUri with given debug flag with validity checks. | 128 | """Instantiate boto.StorageUri with given debug flag with validity checks. |
| 118 | 129 | ||
| 119 | Args: | 130 | Args: |
| 120 | uri_str: URI naming bucket + optional object. | 131 | uri_str: StorageUri naming bucket + optional object. |
| 121 | disallow_object_name: Set to true to only allow object name-less buckets. | 132 | disallow_object_name: Set to true to only allow object name-less buckets. |
| 122 | debug: Whether to enable debugging on StorageUri method calls. | 133 | debug: Whether to enable debugging on StorageUri method calls. |
| 123 | 134 | ||
| 124 | Returns: | 135 | Returns: |
| 125 | boto.StorageUri for given uri_str. | 136 | boto.StorageUri for given uri_str. |
| 126 | 137 | ||
| 127 | Raises: | 138 | Raises: |
| 128 | InvalidUriError: if uri_str not valid. | 139 | InvalidUriError: if uri_str not valid. |
| 129 | """ | 140 | """ |
| 130 | 141 | ||
| 131 | uri = boto.storage_uri(uri_str, 'file', debug) | 142 | uri = boto.storage_uri(uri_str, 'file', debug) |
| 132 | if disallow_object_name and uri.object_name: | 143 | if disallow_object_name and uri.object_name: |
| 133 | raise InvalidUriError('Command requires a URI with no object name') | 144 | raise InvalidUriError('Command requires a URI with no object name') |
| 134 | if uri.bucket_name and re.search('[*?\[\]]', uri.bucket_name): | 145 | if uri.bucket_name and re.search('[*?\[\]]', uri.bucket_name): |
| 135 | raise InvalidUriError('Bucket names cannot contain wildcards') | 146 | raise InvalidUriError('Bucket names cannot contain wildcards') |
| 136 | return uri | 147 | return uri |
| 137 | 148 | ||
| 138 | 149 | ||
| 139 | def ExpandStorageUriGlob(bucket_obj_glob, debug): | 150 | def ExpandStorageUriGlob(bucket_obj_glob, headers=None, debug=False): |
| 140 | """Expands globbing (if any) in bucket_obj_glob. | 151 | """Expands globbing (if any) in bucket_obj_glob. |
| 141 | 152 | ||
| 142 | Args: | 153 | Args: |
| 143 | bucket_obj_glob: bucket+object name, possibly including glob chars. | 154 | bucket_obj_glob: bucket+object name, possibly including glob chars. |
| 155 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 144 | debug: flag indicating whether to include debug output | 156 | debug: flag indicating whether to include debug output |
| 145 | 157 | ||
| 146 | Returns: | 158 | Returns: |
| 147 | list of uri strings, after expanding any globbing. | 159 | list of boto.StorageUri, after expanding any globbing. |
| 148 | 160 | ||
| 149 | For example, ExpandStorageUriGlob('gs://mybucket/*.txt', debug) might return | 161 | For example, ExpandStorageUriGlob('gs://mybucket/*.txt', headers, debug) |
| 150 | ['gs://mybucket/obj1.txt', 'gs://mybucket/obj2.txt'] | 162 | might return [StorageUri('gs://mybucket/obj1.txt'), |
| 163 | StorageUri('gs://mybucket/obj2.txt')] | ||
| 151 | """ | 164 | """ |
| 152 | 165 | ||
| 153 | uri = StorageUri(bucket_obj_glob, False, debug) | 166 | uri = StorageUri(bucket_obj_glob, False, debug) |
| 154 | key_glob = uri.object_name | 167 | key_glob = uri.object_name |
| 155 | result = [] | 168 | result = [] |
| 156 | # Avoid server round trip if input contains no glob chars. | 169 | # Avoid server round trip if input contains no glob chars. |
| 157 | if not re.search('[*?\[\]]', key_glob): | 170 | if not re.search('[*?\[\]]', key_glob): |
| 158 | result.append(bucket_obj_glob) | 171 | result.append(uri) |
| 159 | elif uri.is_file_uri(): | 172 | elif uri.is_file_uri(): |
| 160 | # FileStorageUri objects don't provide a way to return a list of all | 173 | # FileStorageUri objects don't provide a way to return a list of all |
| 161 | # files in the 'bucket', so do our own wildcard expansion for that case. | 174 | # files in the 'bucket', so do our own wildcard expansion for that case. |
| 162 | filenames = glob.glob(uri.object_name) | 175 | filenames = glob.glob(uri.object_name) |
| 163 | for filename in filenames: | 176 | for filename in filenames: |
| 164 | expanded_uri = uri.clone_replace_name(filename) | 177 | expanded_uri = uri.clone_replace_name(filename) |
| 165 | result.append(expanded_uri.uri) | 178 | result.append(expanded_uri) |
| 166 | else: | 179 | else: |
| 167 | # BucketStorageUri with wildcarding. | 180 | # BucketStorageUri with wildcarding. |
| 168 | objs = uri.get_bucket() | 181 | objs = uri.get_bucket(False, headers) |
| 169 | for obj in objs: | 182 | for obj in objs: |
| 170 | if fnmatch.fnmatch(obj.name, key_glob): | 183 | if fnmatch.fnmatch(obj.name, key_glob): |
| 171 | # Replace wildcard name in URI with specific matched obj.name | 184 | # Replace wildcard name in URI with specific matched obj.name |
| 172 | expanded_uri = uri.clone_replace_name(obj.name) | 185 | expanded_uri = uri.clone_replace_name(obj.name) |
| 173 | result.append(expanded_uri.__str__()) | 186 | result.append(expanded_uri) |
| 174 | return result | 187 | return result |
| 175 | 188 | ||
| 176 | 189 | ||
| 177 | def ExpandWildcardsAndContainers(uri_strs, debug): | 190 | def ExpandWildcardsAndContainers(uri_strs, headers=None, debug=False): |
| 178 | """Expands any URI globbing, object-less bucket names, or directory names. | 191 | """Expands any URI globbing, object-less bucket names, or directory names. |
| 179 | 192 | ||
| 180 | Args: | 193 | Args: |
| 181 | uri_strs: URI strings needing expansion | 194 | uri_strs: URI strings needing expansion |
| 195 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 182 | debug: flag indicating whether to include debug output | 196 | debug: flag indicating whether to include debug output |
| 183 | 197 | ||
| 184 | Returns: | 198 | Returns: |
| 185 | list of uri strings, after expanding any globbing and recursively | 199 | list of boto.StorageUri, after expanding any globbing and recursively |
| 186 | walking directories. | 200 | walking directories. |
| 187 | """ | 201 | """ |
| 188 | 202 | ||
| 189 | exp_uri_strs = [] | 203 | result = [] |
| 190 | for uri_str in uri_strs: | 204 | for uri_str in uri_strs: |
| 191 | for exp_uri_str in ExpandStorageUriGlob(uri_str, debug): | 205 | for exp_uri in ExpandStorageUriGlob(uri_str, headers, debug): |
| 192 | exp_uri = StorageUri(exp_uri_str, False, debug) | ||
| 193 | if exp_uri.is_file_uri() and exp_uri.names_container(): | 206 | if exp_uri.is_file_uri() and exp_uri.names_container(): |
| 194 | # exp_uri_str is a file:// URI that names a directory, so include | 207 | # exp_uri is a file:// URI that names a directory, so include |
| 195 | # all its nested files. | 208 | # all its nested files. |
| 196 | for root, unused_dirs, files in os.walk(exp_uri.object_name): | 209 | for root, unused_dirs, files in os.walk(exp_uri.object_name): |
| 197 | for name in files: | 210 | for name in files: |
| 198 | exp_uri_strs.append(os.path.join(root, name)) | 211 | result.append(StorageUri(os.path.join(root, name))) |
| 199 | elif exp_uri.is_cloud_uri() and exp_uri.names_container(): | 212 | elif exp_uri.is_cloud_uri() and exp_uri.names_container(): |
| 200 | # exp_uri_str is an object-less bucket URI, so include | 213 | # exp_uri is an object-less bucket URI, so include |
| 201 | # all its nested objects. | 214 | # all its nested objects. |
| 202 | bucket_wildcard = exp_uri.clone_replace_name('*').uri | 215 | bucket_wildcard = exp_uri.clone_replace_name('*').uri |
| 203 | obj_list = ExpandStorageUriGlob(bucket_wildcard, debug) | 216 | uris = ExpandStorageUriGlob(bucket_wildcard, headers, debug) |
| 204 | for exp_bucket_uri_str in obj_list: | 217 | for uri in uris: |
| 205 | exp_uri_strs.append(exp_bucket_uri_str) | 218 | result.append(uri) |
| 206 | else: | 219 | else: |
| 207 | exp_uri_strs.append(exp_uri_str) | 220 | result.append(exp_uri) |
| 208 | return exp_uri_strs | 221 | return result |
| 209 | 222 | ||
| 210 | 223 | ||
| 211 | def InsistUriNamesContainer(command, uri): | 224 | def InsistUriNamesContainer(command, uri): |
| 212 | """Prints error and exists if URI doesn't name a directory or bucket. | 225 | """Prints error and exists if URI doesn't name a directory or bucket. |
| 213 | 226 | ||
| 214 | Args: | 227 | Args: |
| 215 | command: command being run | 228 | command: command being run |
| 216 | uri: StorageUri to check | 229 | uri: StorageUri to check |
| 217 | """ | 230 | """ |
| 218 | 231 | ||
| 219 | if uri.names_singleton(): | 232 | if uri.names_singleton(): |
| 220 | OutputAndExit('Destination URI must name a bucket or directory ' | 233 | OutputAndExit('destination StorageUri must name a bucket or directory ' |
| 221 | 'for the multiple source\nform of the "%s" command.' % | 234 | 'for the multiple source\nform of the "%s" command.' % |
| 222 | command) | 235 | command) |
| 223 | 236 | ||
| 224 | def CatCommand(args, sub_opts, debug): | 237 | |
| 238 | def CatCommand(args, sub_opts, headers=None, debug=False): | ||
| 225 | """Implementation of cat command. | 239 | """Implementation of cat command. |
| 226 | 240 | ||
| 227 | Args: | 241 | Args: |
| 228 | args: command-line arguments | 242 | args: command-line arguments |
| 229 | sub_opts: command-specific options from getopt. | 243 | sub_opts: command-specific options from getopt. |
| 244 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 230 | debug: flag indicating whether to include debug output | 245 | debug: flag indicating whether to include debug output |
| 231 | """ | 246 | """ |
| 232 | 247 | ||
| 233 | show_header = False | 248 | show_header = False |
| 234 | for o, unused_a in sub_opts: | 249 | for o, unused_a in sub_opts: |
| 235 | if o == '-h': | 250 | if o == '-h': |
| 236 | show_header = True | 251 | show_header = True |
| 237 | 252 | ||
| 238 | # Expand object name globs, if any. | 253 | # Expand object name globs, if any. |
| 239 | exp_args = [] | 254 | exp_uris = [] |
| 240 | for uri_str in args: | 255 | for uri_str in args: |
| 241 | for exp_uri_str in ExpandStorageUriGlob(uri_str, debug): | 256 | for exp_uri in ExpandStorageUriGlob(uri_str, headers, debug): |
| 242 | exp_args.append(exp_uri_str) | 257 | exp_uris.append(exp_uri) |
| 243 | if not exp_args: | 258 | if not exp_uris: |
| 244 | OutputAndExit('No objects matched.') | 259 | OutputAndExit('No objects matched.') |
| 245 | printed_one = False | 260 | printed_one = False |
| 246 | for uri_str in exp_args: | 261 | for uri in exp_uris: |
| 247 | uri = StorageUri(uri_str, False, debug) | ||
| 248 | if not uri.object_name: | 262 | if not uri.object_name: |
| 249 | OutputAndExit('"cat" command must specify objects.') | 263 | OutputAndExit('"cat" command must specify objects.') |
| 250 | if show_header: | 264 | if show_header: |
| 251 | if printed_one: | 265 | if printed_one: |
| 252 | 266 | ||
| 253 | print('==> %s <==' % uri_str) | 267 | print '==> %s <==' % uri.__str__() |
| 254 | printed_one = True | 268 | printed_one = True |
| 255 | sys.stdout.write(uri.get_contents_as_string()) | 269 | sys.stdout.write(uri.get_contents_as_string(False, headers)) |
| 256 | 270 | ||
| 257 | def SetAclCommand(args, unused_sub_opts, debug): | 271 | |
| 272 | def SetAclCommand(args, unused_sub_opts, headers=None, debug=False): | ||
| 258 | """Implementation of setacl command. | 273 | """Implementation of setacl command. |
| 259 | 274 | ||
| 260 | Args: | 275 | Args: |
| 261 | args: command-line arguments | 276 | args: command-line arguments |
| 262 | unused_sub_opts: command-specific options from getopt. | 277 | unused_sub_opts: command-specific options from getopt. |
| 278 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 263 | debug: flag indicating whether to include debug output | 279 | debug: flag indicating whether to include debug output |
| 264 | """ | 280 | """ |
| 265 | 281 | ||
| 266 | acl_arg = args[0] | 282 | acl_arg = args[0] |
| 267 | # Expand object name globs, if any. | 283 | # Expand object name globs, if any. |
| 268 | exp_args = [] | 284 | exp_uris = [] |
| 269 | for uri_str in args[1:]: | 285 | for uri_str in args[1:]: |
| 270 | for exp_uri_str in ExpandStorageUriGlob(uri_str, debug): | 286 | for exp_uri in ExpandStorageUriGlob(uri_str, headers, debug): |
| 271 | exp_args.append(exp_uri_str) | 287 | exp_uris.append(exp_uri) |
| 272 | 288 | ||
| 273 | # Determine whether acl_arg names a file vs. a canned ACL name. | 289 | # Disallow setacl command spanning multiple providers because |
| 274 | # Note: passing an XML file is currently experimental. | 290 | # there are differences in the ACL models. |
| 291 | provider = None | ||
| 292 | for uri in exp_uris: | ||
| 293 | if not provider: | ||
| 294 | provider = uri.provider | ||
| 295 | elif uri.provider != provider: | ||
| 296 | OutputAndExit('"setacl" command spanning multiple providers not allowed.') | ||
| 297 | |||
| 298 | # Get ACL object from connection for the first URI, for interpreting the | ||
| 299 | # ACL. This won't fail because the main startup code insists on 1 arg | ||
| 300 | # for this command. | ||
| 301 | storage_uri = exp_uris[0] | ||
| 302 | acl_class = storage_uri.acl_class() | ||
| 303 | canned_acls = storage_uri.canned_acls() | ||
| 304 | |||
| 305 | # Determine whether acl_arg names a file containing XML ACL text vs. the | ||
| 306 | # string name of a canned ACL. | ||
| 275 | if os.path.isfile(acl_arg): | 307 | if os.path.isfile(acl_arg): |
| 276 | acl_file = open(acl_arg, 'r') | 308 | acl_file = open(acl_arg, 'r') |
| 277 | acl_txt = acl_file.read() | 309 | acl_txt = acl_file.read() |
| 278 | acl_file.close() | 310 | acl_file.close() |
| 279 | # We need a bucket object for parsing the XML into a Policy, so just | 311 | acl_obj = acl_class() |
| 280 | # use the bucket object from the first URI. This won't fail because the | 312 | h = handler.XmlHandler(acl_obj, storage_uri.get_bucket()) |
| 281 | # main startup code insists on 1 arg for this command. | ||
| 282 | bucket = StorageUri(exp_args[0], False, debug).get_bucket() | ||
| 283 | policy = Policy(bucket) | ||
| 284 | h = handler.XmlHandler(policy, bucket) | ||
| 285 | xml.sax.parseString(acl_txt, h) | 313 | xml.sax.parseString(acl_txt, h) |
| 286 | acl_arg = policy | 314 | acl_arg = acl_obj |
| 287 | else: | 315 | else: |
| 288 | # No file exists, so expect a canned ACL string. | 316 | # No file exists, so expect a canned ACL string. |
| 289 | if acl_arg not in CannedACLStrings: | 317 | if acl_arg not in canned_acls: |
| 290 | OutputAndExit('Invalid canned ACL "%s".' % acl_arg) | 318 | OutputAndExit('Invalid canned ACL "%s".' % acl_arg) |
| 291 | 319 | ||
| 292 | # Now iterate over URI args and set the ACL on each. | 320 | # Now iterate over URIs and set the ACL on each. |
| 293 | for uri_str in exp_args: | 321 | for uri in exp_uris: |
| 294 | uri = StorageUri(uri_str, False, debug) | 322 | if len(exp_uris) > 1: |
| 295 | if len(exp_args) > 1: | ||
| 296 | # Progress indicator | 323 | # Progress indicator |
| 297 | print 'Setting ACL on %s...' % uri | 324 | print 'Setting ACL on %s...' % uri |
| 298 | uri.set_acl(acl_arg, uri.object_name) | 325 | uri.set_acl(acl_arg, uri.object_name, False, headers) |
| 299 | 326 | ||
| 300 | 327 | ||
| 301 | def ExplainIfSudoNeeded(tf, dirs_to_remove): | 328 | def ExplainIfSudoNeeded(tf, dirs_to_remove): |
| 302 | """Explains what to do if sudo needed to update gsutil software. | 329 | """Explains what to do if sudo needed to update gsutil software. |
| 303 | 330 | ||
| 304 | Happens if gsutil was previously installed by a different user (typically if | 331 | Happens if gsutil was previously installed by a different user (typically if |
| 305 | someone originally installed in a shared file system location, using sudo). | 332 | someone originally installed in a shared file system location, using sudo). |
| 306 | 333 | ||
| 307 | Args: | 334 | Args: |
| 308 | tf: opened TarFile. | 335 | tf: opened TarFile. |
| 309 | dirs_to_remove: list of directories to remove. | 336 | dirs_to_remove: list of directories to remove. |
| 310 | """ | 337 | """ |
| 311 | 338 | ||
| 312 | system = platform.system() | 339 | system = platform.system() |
| 313 | # If running under Windows we don't need (or have) sudo. | 340 | # If running under Windows we don't need (or have) sudo. |
| 314 | if system.lower().startswith('windows'): | 341 | if system.lower().startswith('windows'): |
| 315 | return | 342 | return |
| 316 | 343 | ||
| 317 | user_id = os.getuid() | 344 | user_id = os.getuid() |
| 318 | if (os.stat(gsutil_bin_dir).st_uid == user_id and | 345 | if (os.stat(gsutil_bin_dir).st_uid == user_id and |
| 319 | os.stat(boto_lib_dir).st_uid == user_id): | 346 | os.stat(boto_lib_dir).st_uid == user_id): |
| 320 | return | 347 | return |
| 321 | 348 | ||
| 322 | # Won't fail - this command runs after main startup code that insists on | 349 | # Won't fail - this command runs after main startup code that insists on |
| 323 | # having a config file. | 350 | # having a config file. |
| 324 | config_file = GetBotoConfigFileList()[0] | 351 | config_file = GetBotoConfigFileList()[0] |
| 325 | CleanUpUpdateCommand(tf, dirs_to_remove) | 352 | CleanUpUpdateCommand(tf, dirs_to_remove) |
| 326 | OutputAndExit(('Since it was installed by a different user previously, ' | 353 | OutputAndExit(('Since it was installed by a different user previously, ' |
| 327 | 'you will need to update using the following commands.\n' | 354 | 'you will need to update using the following commands.\n' |
| 328 | 'You will be prompted for your password, and the install ' | 355 | 'You will be prompted for your password, and the install ' |
| 329 | 'will run as "root". If you\'re unsure what this means please ' | 356 | 'will run as "root". If you\'re unsure what this means please ' |
| 330 | 'ask your system administrator for help:\n' | 357 | 'ask your system administrator for help:\n' |
| 331 | '\tchmod 644 %s\n' | 358 | '\tchmod 644 %s\n' |
| 332 | '\tsudo env BOTO_CONFIG=%s gsutil update\n' | 359 | '\tsudo env BOTO_CONFIG=%s gsutil update\n' |
| 333 | '\tchmod 600 %s') % (config_file, config_file, config_file)) | 360 | '\tchmod 600 %s') % (config_file, config_file, config_file)) |
| 334 | 361 | ||
| 335 | 362 | ||
| 336 | def CleanUpUpdateCommand(tf, dirs_to_remove): | 363 | def CleanUpUpdateCommand(tf, dirs_to_remove): |
| 337 | """Cleans up temp files etc. from running update command. | 364 | """Cleans up temp files etc. from running update command. |
| 338 | 365 | ||
| 339 | Args: | 366 | Args: |
| 340 | tf: opened TarFile. | 367 | tf: opened TarFile. |
| 341 | dirs_to_remove: list of directories to remove. | 368 | dirs_to_remove: list of directories to remove. |
| 342 | 369 | ||
| 343 | """ | 370 | """ |
| 344 | tf.close() | 371 | tf.close() |
| 345 | for directory in dirs_to_remove: | 372 | for directory in dirs_to_remove: |
| 346 | shutil.rmtree(directory) | 373 | shutil.rmtree(directory) |
| 347 | 374 | ||
| 348 | 375 | ||
| 349 | def LoadVersionString(): | 376 | def LoadVersionString(): |
| 350 | """Loads version string for currently installed gsutil command. | 377 | """Loads version string for currently installed gsutil command. |
| 351 | 378 | ||
| 352 | Returns: | 379 | Returns: |
| 353 | Version string. | 380 | Version string. |
| 354 | """ | 381 | """ |
| 355 | 382 | ||
| 356 | ver_file_path = gsutil_bin_dir + os.sep + 'VERSION' | 383 | ver_file_path = gsutil_bin_dir + os.sep + 'VERSION' |
| 357 | if not os.path.isfile(ver_file_path): | 384 | if not os.path.isfile(ver_file_path): |
| 358 | OutputAndExit('%s not found. Did you install the\ncomplete gsutil software ' | 385 | OutputAndExit('%s not found. Did you install the\ncomplete gsutil software ' |
| 359 | 'after the gsutil "update" command was implemented?' % | 386 | 'after the gsutil "update" command was implemented?' % |
| 360 | ver_file_path) | 387 | ver_file_path) |
| 361 | ver_file = open(ver_file_path, 'r') | 388 | ver_file = open(ver_file_path, 'r') |
| 362 | installed_version_string = ver_file.read().rstrip('\n') | 389 | installed_version_string = ver_file.read().rstrip('\n') |
| 363 | ver_file.close() | 390 | ver_file.close() |
| 364 | return installed_version_string | 391 | return installed_version_string |
| 365 | 392 | ||
| 366 | 393 | ||
| 367 | def UpdateCommand(unused_args, sub_opts, debug): | 394 | def UpdateCommand(unused_args, sub_opts, headers=None, debug=False): |
| 368 | """Implementation of experimental update command. | 395 | """Implementation of experimental update command. |
| 369 | 396 | ||
| 370 | Args: | 397 | Args: |
| 371 | unused_args: command-line arguments | 398 | unused_args: command-line arguments |
| 372 | sub_opts: command-specific options from getopt. | 399 | sub_opts: command-specific options from getopt. |
| 400 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 373 | debug: flag indicating whether to include debug output | 401 | debug: flag indicating whether to include debug output |
| 374 | """ | 402 | """ |
| 375 | 403 | ||
| 376 | installed_version_string = LoadVersionString() | 404 | installed_version_string = LoadVersionString() |
| 377 | 405 | ||
| 378 | dirs_to_remove = [] | 406 | dirs_to_remove = [] |
| 379 | # Retrieve gsutil tarball and check if it's newer than installed code. | 407 | # Retrieve gsutil tarball and check if it's newer than installed code. |
| 380 | # TODO: Expose a way to set metadata on objects via gsutil, then store | 408 | # TODO: Store this version info as metadata on the tarball object and |
| 381 | # this version info there and change this command's implementation to check | 409 | # change this command's implementation to check that metadata instead of |
| 382 | # that metadata instead of downloading the tarball to check the version info. | 410 | # downloading the tarball to check the version info. |
| 383 | tmp_dir = tempfile.mkdtemp() | 411 | tmp_dir = tempfile.mkdtemp() |
| 384 | dirs_to_remove.append(tmp_dir) | 412 | dirs_to_remove.append(tmp_dir) |
| 385 | os.chdir(tmp_dir) | 413 | os.chdir(tmp_dir) |
| 386 | print 'Checking for software update...' | 414 | print 'Checking for software update...' |
| 387 | CopyObjsCommand(['gs://pub/gsutil.tar.gz', 'file://gsutil.tar.gz'], [], debug) | 415 | CopyObjsCommand(['gs://pub/gsutil.tar.gz', 'file://gsutil.tar.gz'], [], |
| 416 | headers, debug) | ||
| 388 | tf = tarfile.open('gsutil.tar.gz') | 417 | tf = tarfile.open('gsutil.tar.gz') |
| 389 | tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions. | 418 | tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions. |
| 390 | tf.extract('./gsutil/VERSION') | 419 | tf.extract('./gsutil/VERSION') |
| 391 | ver_file = open('gsutil/VERSION', 'r') | 420 | ver_file = open('gsutil/VERSION', 'r') |
| 392 | latest_version_string = ver_file.read().rstrip('\n') | 421 | latest_version_string = ver_file.read().rstrip('\n') |
| 393 | ver_file.close() | 422 | ver_file.close() |
| 394 | 423 | ||
| 395 | # The force_update option works around a problem with the way the | 424 | # The force_update option works around a problem with the way the |
| 396 | # first gsutil "update" command exploded the gsutil and boto directories, | 425 | # first gsutil "update" command exploded the gsutil and boto directories, |
| 397 | # which didn't correctly install boto. People running that older code can | 426 | # which didn't correctly install boto. People running that older code can |
| 398 | # run "gsutil update" (to update to the newer gsutil update code) followed by | 427 | # run "gsutil update" (to update to the newer gsutil update code) followed by |
| 399 | # "gsutil update -f" (which will then update the boto code, even though the | 428 | # "gsutil update -f" (which will then update the boto code, even though the |
| 400 | # VERSION is already the latest version). | 429 | # VERSION is already the latest version). |
| 401 | force_update = False | 430 | force_update = False |
| 402 | for o, unused_a in sub_opts: | 431 | for o, unused_a in sub_opts: |
| 403 | if o == '-f': | 432 | if o == '-f': |
| 404 | force_update = True | 433 | force_update = True |
| 405 | if not force_update and installed_version_string == latest_version_string: | 434 | if not force_update and installed_version_string == latest_version_string: |
| 406 | CleanUpUpdateCommand(tf, dirs_to_remove) | 435 | CleanUpUpdateCommand(tf, dirs_to_remove) |
| 407 | OutputAndExit('You have the latest version of gsutil installed.') | 436 | OutputAndExit('You have the latest version of gsutil installed.') |
| 408 | 437 | ||
| 409 | print(('This command will update to the "%s" version of\ngsutil at %s') % | 438 | print(('This command will update to the "%s" version of\ngsutil at %s') % |
| 410 | (latest_version_string, gsutil_bin_dir)) | 439 | (latest_version_string, gsutil_bin_dir)) |
| 411 | ExplainIfSudoNeeded(tf, tmp_dir) | 440 | ExplainIfSudoNeeded(tf, tmp_dir) |
| 412 | 441 | ||
| 413 | answer = raw_input('Proceed (Note: experimental command)? [y/N] ') | 442 | answer = raw_input('Proceed (Note: experimental command)? [y/N] ') |
| 414 | if not answer or answer.lower()[0] != 'y': | 443 | if not answer or answer.lower()[0] != 'y': |
| 415 | CleanUpUpdateCommand(tf, dirs_to_remove) | 444 | CleanUpUpdateCommand(tf, dirs_to_remove) |
| 416 | OutputAndExit('Not running update.') | 445 | OutputAndExit('Not running update.') |
| 417 | 446 | ||
| 418 | # Ignore keyboard interrupts during the update to reduce the chance someone | 447 | # Ignore keyboard interrupts during the update to reduce the chance someone |
| 419 | # hitting ^C leaves gsutil in a broken state. | 448 | # hitting ^C leaves gsutil in a broken state. |
| 420 | signal.signal(signal.SIGINT, signal.SIG_IGN) | 449 | signal.signal(signal.SIGINT, signal.SIG_IGN) |
| 421 | 450 | ||
| 422 | # gsutil_bin_dir lists the path where the code should end up (like | 451 | # gsutil_bin_dir lists the path where the code should end up (like |
| 423 | # /usr/local/gsutil), which is one level down from the relative path in the | 452 | # /usr/local/gsutil), which is one level down from the relative path in the |
| 424 | # tarball (since the latter creates files in ./gsutil). So, we need to | 453 | # tarball (since the latter creates files in ./gsutil). So, we need to |
| 425 | # extract at the parent directory level. | 454 | # extract at the parent directory level. |
| 426 | gsutil_bin_parent_dir = os.path.dirname(gsutil_bin_dir) | 455 | gsutil_bin_parent_dir = os.path.dirname(gsutil_bin_dir) |
| 427 | 456 | ||
| 428 | # Extract tarball to a temporary directory in a sibling to gsutil_bin_dir. | 457 | # Extract tarball to a temporary directory in a sibling to gsutil_bin_dir. |
| 429 | old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) | 458 | old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) |
| 430 | new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) | 459 | new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) |
| 431 | dirs_to_remove.append(old_dir) | 460 | dirs_to_remove.append(old_dir) |
| 432 | dirs_to_remove.append(new_dir) | 461 | dirs_to_remove.append(new_dir) |
| 433 | try: | 462 | try: |
| 434 | tf.extractall(path=new_dir) | 463 | tf.extractall(path=new_dir) |
| 435 | except Exception, e: | 464 | except Exception, e: |
| 436 | CleanUpUpdateCommand(tf, dirs_to_remove) | 465 | CleanUpUpdateCommand(tf, dirs_to_remove) |
| 437 | OutputAndExit('Update failed: %s.' % e) | 466 | OutputAndExit('Update failed: %s.' % e) |
| 438 | 467 | ||
| 439 | # Move old installation aside and new into place. | 468 | # Move old installation aside and new into place. |
| 440 | os.rename(gsutil_bin_dir, old_dir + os.sep + 'old') | 469 | os.rename(gsutil_bin_dir, old_dir + os.sep + 'old') |
| 441 | os.rename(new_dir + os.sep + 'gsutil', gsutil_bin_dir) | 470 | os.rename(new_dir + os.sep + 'gsutil', gsutil_bin_dir) |
| 442 | CleanUpUpdateCommand(tf, dirs_to_remove) | 471 | CleanUpUpdateCommand(tf, dirs_to_remove) |
| 443 | signal.signal(signal.SIGINT, signal.SIG_DFL) | 472 | signal.signal(signal.SIGINT, signal.SIG_DFL) |
| 444 | print 'Update complete.' | 473 | print 'Update complete.' |
| 445 | 474 | ||
| 446 | 475 | ||
| 447 | def CheckForDirFileConflict(src_uri, dst_path): | 476 | def CheckForDirFileConflict(src_uri, dst_path): |
| 448 | """Checks whether copying src_uri into dst_path is not possible. | 477 | """Checks whether copying src_uri into dst_path is not possible. |
| 449 | 478 | ||
| 450 | This happens if a directory exists in local file system where a file needs | 479 | This happens if a directory exists in local file system where a file needs |
| 451 | to go or vice versa. In that case we print an error message and exits. | 480 | to go or vice versa. In that case we print an error message and exits. |
| 452 | Example: if the file "./x" exists and you try to do: | 481 | Example: if the file "./x" exists and you try to do: |
| 453 | gsutil cp gs://mybucket/x/y . | 482 | gsutil cp gs://mybucket/x/y . |
| 454 | the request can't succeed because it requires a directory where | 483 | the request can't succeed because it requires a directory where |
| 455 | the file x exists. | 484 | the file x exists. |
| 456 | 485 | ||
| 457 | Args: | 486 | Args: |
| 458 | src_uri: source URI of copy | 487 | src_uri: source StorageUri of copy |
| 459 | dst_path: destination path. | 488 | dst_path: destination path. |
| 460 | """ | 489 | """ |
| 461 | 490 | ||
| 462 | final_dir = os.path.dirname(dst_path) | 491 | final_dir = os.path.dirname(dst_path) |
| 463 | if os.path.isfile(final_dir): | 492 | if os.path.isfile(final_dir): |
| 464 | OutputAndExit('Cannot retrieve %s because it a file exists where a ' | 493 | OutputAndExit('Cannot retrieve %s because it a file exists where a ' |
| 465 | 'directory needs to be created (%s).' % (src_uri, final_dir)) | 494 | 'directory needs to be created (%s).' % (src_uri, final_dir)) |
| 466 | if os.path.isdir(dst_path): | 495 | if os.path.isdir(dst_path): |
| 467 | OutputAndExit('Cannot retrieve %s because a directory exists ' | 496 | OutputAndExit('Cannot retrieve %s because a directory exists ' |
| 468 | '(%s) where the file needs to be created.' % | 497 | '(%s) where the file needs to be created.' % |
| 469 | (src_uri, dst_path)) | 498 | (src_uri, dst_path)) |
| 470 | 499 | ||
| 471 | 500 | ||
| 472 | def ReportNoMatchesAndExit(uri): | 501 | def ReportNoMatchesAndExit(uri): |
| 473 | """Reports no URI wildcard matches and exits. | 502 | """Reports no URI wildcard matches and exits. |
| 474 | 503 | ||
| 475 | Args: | 504 | Args: |
| 476 | uri: the URI that didn't match. | 505 | uri: the StorageUri that didn't match. |
| 477 | """ | 506 | """ |
| 478 | if uri.is_file_uri(): | 507 | if uri.is_file_uri(): |
| 479 | OutputAndExit('"%s" matches no files.' % uri.uri) | 508 | OutputAndExit('"%s" matches no files.' % uri.uri) |
| 480 | else: | 509 | else: |
| 481 | OutputAndExit('"%s" matches no objects.' % uri.uri) | 510 | OutputAndExit('"%s" matches no objects.' % uri.uri) |
| 482 | 511 | ||
| 483 | 512 | ||
| 484 | def GetAclCommand(args, unused_sub_opts, debug): | 513 | def GetAclCommand(args, unused_sub_opts, headers=None, debug=False): |
| 485 | """Implementation of experimental getacl command. | 514 | """Implementation of getacl command. |
| 486 | 515 | ||
| 487 | Args: | 516 | Args: |
| 488 | args: command-line arguments | 517 | args: command-line arguments |
| 489 | unused_sub_opts: command-specific options from getopt. | 518 | unused_sub_opts: command-specific options from getopt. |
| 519 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 490 | debug: flag indicating whether to include debug output | 520 | debug: flag indicating whether to include debug output |
| 491 | """ | 521 | """ |
| 492 | 522 | ||
| 493 | # Wildcarding is allowed but must resolve to just one object. | 523 | # Wildcarding is allowed but must resolve to just one object. |
| 494 | uris = ExpandStorageUriGlob(args[0], debug) | 524 | uris = ExpandStorageUriGlob(args[0], headers, debug) |
| 495 | if len(uris) != 1: | 525 | if len(uris) != 1: |
| 496 | OutputAndExit('Wildcards must resolve to exactly one object for "getacl" ' | 526 | OutputAndExit('Wildcards must resolve to exactly one object for "getacl" ' |
| 497 | 'command.') | 527 | 'command.') |
| 498 | uri = StorageUri(uris[0], False, debug) | 528 | uri = uris[0] |
| 499 | if not uri.bucket_name: | 529 | if not uri.bucket_name: |
| 500 | OutputAndExit('"getacl" command must specify a bucket or object.') | 530 | OutputAndExit('"getacl" command must specify a bucket or object.') |
| 501 | acl = uri.get_acl() | 531 | acl = uri.get_acl(False, headers) |
| 502 | # Pretty-print the XML to make it more easily human editable. | 532 | # Pretty-print the XML to make it more easily human editable. |
| 503 | parsed_xml = xml.dom.minidom.parseString(acl.to_xml()) | 533 | parsed_xml = xml.dom.minidom.parseString(acl.to_xml()) |
| 504 | print parsed_xml.toprettyxml(indent=' ') | 534 | print parsed_xml.toprettyxml(indent=' ') |
| 505 | 535 | ||
| 506 | def PerformCopy(src_uri, dst_uri): | 536 | |
| 537 | def PerformCopy(src_uri, dst_uri, sub_opts, headers): | ||
| 507 | """Helper method for CopyObjsCommand. | 538 | """Helper method for CopyObjsCommand. |
| 508 | 539 | ||
| 509 | Args: | 540 | Args: |
| 510 | src_uri source URI for copy. | 541 | src_uri: source StorageUri for copy. |
| 511 | dst_uri: destination URI for copy. | 542 | dst_uri: destination StorageUri for copy. |
| 543 | sub_opts: command-specific options from getopt. | ||
| 544 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 512 | """ | 545 | """ |
| 513 | 546 | ||
| 514 | src_key = src_uri.get_key() | 547 | # Make a copy of the input headers each time so we can set a different |
| 548 | # MIME type for each object. | ||
| 549 | metadata = headers.copy() | ||
| 550 | canned_acl = None | ||
| 551 | for o, a in sub_opts: | ||
| 552 | if o == "-a": | ||
| 553 | canned_acls = dst_uri.canned_acls() | ||
| 554 | if a not in canned_acls: | ||
| 555 | OutputAndExit('Invalid canned ACL "%s".' % a) | ||
| 556 | canned_acl = a | ||
| 557 | elif o == "-t": | ||
| 558 | mimetype_tuple = mimetypes.guess_type(src_uri.object_name) | ||
| 559 | mime_type = mimetype_tuple[0] | ||
| 560 | content_encoding = mimetype_tuple[1] | ||
| 561 | if mime_type: | ||
| 562 | metadata['Content-Type'] = mime_type | ||
| 563 | print '\t[Setting Content-Type=%s]' % mime_type | ||
| 564 | else: | ||
| 565 | print '\t[Unknown content type -> using application/octet stream]' | ||
| 566 | if content_encoding: | ||
| 567 | metadata['Content-Encoding'] = content_encoding | ||
| 568 | |||
| 569 | src_key = src_uri.get_key(False, headers) | ||
| 515 | if not src_key: | 570 | if not src_key: |
| 516 | OutputAndExit('"%s" does not exist.' % src_uri) | 571 | OutputAndExit('"%s" does not exist.' % src_uri) |
| 517 | 572 | ||
| 518 | # Separately handle cases to avoid extra file and network copying of | 573 | # Separately handle cases to avoid extra file and network copying of |
| 519 | # potentially very large files/objects. | 574 | # potentially very large files/objects. |
| 520 | 575 | ||
| 521 | if (src_uri.is_cloud_uri() and dst_uri.is_cloud_uri() and | 576 | if (src_uri.is_cloud_uri() and dst_uri.is_cloud_uri() and |
| 522 | src_uri.provider == dst_uri.provider): | 577 | src_uri.provider == dst_uri.provider): |
| 523 | # Object -> object, within same provider (uses x-<provider>-copy-source | 578 | # Object -> object, within same provider (uses x-<provider>-copy-source |
| 524 | # metadata HTTP header to request copying at the server): | 579 | # metadata HTTP header to request copying at the server). (Note: boto |
| 525 | src_bucket = src_uri.get_bucket() | 580 | # does not currently provide a way to pass canned_acl when copying from |
| 526 | dst_bucket = dst_uri.get_bucket() | 581 | # object-to-object through x-<provider>-copy-source): |
| 582 | src_bucket = src_uri.get_bucket(False, headers) | ||
| 583 | dst_bucket = dst_uri.get_bucket(False, headers) | ||
| 527 | dst_bucket.copy_key(dst_uri.object_name, src_bucket.name, | 584 | dst_bucket.copy_key(dst_uri.object_name, src_bucket.name, |
| 528 | src_uri.object_name) | 585 | src_uri.object_name, metadata) |
| 529 | return | 586 | return |
| 530 | 587 | ||
| 531 | dst_key = dst_uri.new_key() | 588 | dst_key = dst_uri.new_key(False, headers) |
| 532 | if src_uri.is_file_uri() and dst_uri.is_cloud_uri(): | 589 | if src_uri.is_file_uri() and dst_uri.is_cloud_uri(): |
| 533 | # File -> object: | 590 | # File -> object: |
| 534 | dst_key.set_contents_from_file(src_key.fp) | 591 | fname_parts = src_uri.object_name.split('.') |
| 592 | dst_key.set_contents_from_file(src_key.fp, metadata, policy=canned_acl) | ||
| 535 | elif src_uri.is_cloud_uri() and dst_uri.is_file_uri(): | 593 | elif src_uri.is_cloud_uri() and dst_uri.is_file_uri(): |
| 536 | # Object -> file: | 594 | # Object -> file: |
| 537 | src_key.get_file(dst_key.fp) | 595 | src_key.get_file(dst_key.fp, headers) |
| 538 | elif src_uri.is_file_uri() and dst_uri.is_file_uri(): | 596 | elif src_uri.is_file_uri() and dst_uri.is_file_uri(): |
| 539 | # File -> file: | 597 | # File -> file: |
| 540 | dst_key.set_contents_from_file(src_key.fp) | 598 | dst_key.set_contents_from_file(src_key.fp, metadata) |
| 541 | else: | 599 | else: |
| 542 | # We implement cross-provider object copy through a local temp file: | 600 | # We implement cross-provider object copy through a local temp file: |
| 543 | tmp = tempfile.TemporaryFile() | 601 | tmp = tempfile.TemporaryFile() |
| 544 | src_key.get_file(tmp) | 602 | src_key.get_file(tmp, headers) |
| 545 | tmp.seek(0) | 603 | tmp.seek(0) |
| 546 | dst_key.set_contents_from_file(tmp) | 604 | dst_key.set_contents_from_file(tmp, metadata) |
| 547 | 605 | ||
| 548 | 606 | ||
| 549 | def CopyObjsCommand(args, unused_sub_opts, debug): | 607 | def CopyObjsCommand(args, sub_opts, headers=None, debug=False): |
| 550 | """Implementation of cp command. | 608 | """Implementation of cp command. |
| 551 | 609 | ||
| 552 | Args: | 610 | Args: |
| 553 | args: command-line arguments | 611 | args: command-line arguments |
| 554 | unused_sub_opts: command-specific options from getopt. | 612 | sub_opts: command-specific options from getopt. |
| 613 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 555 | debug: flag indicating whether to include debug output | 614 | debug: flag indicating whether to include debug output |
| 556 | """ | 615 | """ |
| 557 | 616 | ||
| 558 | src_uri_strs = args[0:len(args)-1] | 617 | src_uri_strs = args[0:len(args)-1] |
| 559 | dst_uri = StorageUri(args[-1], False, debug) | 618 | dst_uri = StorageUri(args[-1], False, debug) |
| 560 | multi_obj_copy = True | 619 | multi_obj_copy = True |
| 561 | 620 | ||
| 562 | # Expand wildcards and containers in source URIs. | 621 | # Expand wildcards and containers in source StorageUris. |
| 563 | exp_src_uri_strs = ExpandWildcardsAndContainers(src_uri_strs, debug) | 622 | exp_src_uris = ExpandWildcardsAndContainers(src_uri_strs, headers, debug) |
| 564 | 623 | ||
| 565 | # Abort if wildcarding produced no matches. | 624 | # Abort if wildcarding produced no matches. |
| 566 | if not exp_src_uri_strs: | 625 | if not exp_src_uris: |
| 567 | ReportNoMatches(StorageUri(src_uri_strs[0])) | 626 | ReportNoMatchesAndExit(StorageUri(src_uri_strs[0])) |
| 568 | 627 | ||
| 569 | # If there is 1 source arg after expansion, with src_uri naming an | 628 | # If there is 1 source arg after expansion, with src_uri naming an |
| 570 | # object-less bucket and dst_uri naming a directory, handle two cases to | 629 | # object-less bucket and dst_uri naming a directory, handle two cases to |
| 571 | # make copy command work like UNIX "cp -r" works: | 630 | # make copy command work like UNIX "cp -r" works: |
| 572 | # a) if no directory exists for dst_uri copy objects to a new directory | 631 | # a) if no directory exists for dst_uri copy objects to a new directory |
| 573 | # with the dst_uri name, e.g., "bucket/a" -> "dir/a" | 632 | # with the dst_uri name, e.g., "bucket/a" -> "dir/a" |
| 574 | # b) if a directory exists for dst_uri copy objects to a new directory | 633 | # b) if a directory exists for dst_uri copy objects to a new directory |
| 575 | # under that directory, e.g., "bucket/a" -> "dir/bucket/a" | 634 | # under that directory, e.g., "bucket/a" -> "dir/bucket/a" |
| 576 | if len(exp_src_uri_strs) == 1: | 635 | if len(exp_src_uris) == 1: |
| 577 | src_uri_to_check = StorageUri(exp_src_uri_strs[0]) | 636 | src_uri_to_check = exp_src_uris[0] |
| 578 | if src_uri_to_check.names_container(): | 637 | if src_uri_to_check.names_container(): |
| 579 | if dst_uri.names_container() and os.path.exists(dst_uri.object_name): | 638 | if dst_uri.names_container() and os.path.exists(dst_uri.object_name): |
| 580 | dst_uri = dst_uri.clone_replace_name(dst_uri.object_name + os.sep + | 639 | dst_uri = dst_uri.clone_replace_name(dst_uri.object_name + os.sep + |
| 581 | src_uri_to_check.bucket_name) | 640 | src_uri_to_check.bucket_name) |
| 582 | else: | 641 | else: |
| 583 | multi_obj_copy = False | 642 | multi_obj_copy = False |
| 584 | 643 | ||
| 585 | if (multi_obj_copy and dst_uri.is_file_uri() | 644 | if (multi_obj_copy and dst_uri.is_file_uri() |
| 586 | and not os.path.exists(dst_uri.object_name)): | 645 | and not os.path.exists(dst_uri.object_name)): |
| 587 | os.makedirs(dst_uri.object_name) | 646 | os.makedirs(dst_uri.object_name) |
| 588 | 647 | ||
| 589 | if multi_obj_copy: | 648 | if multi_obj_copy: |
| 590 | InsistUriNamesContainer('cp', dst_uri) | 649 | InsistUriNamesContainer('cp', dst_uri) |
| 591 | 650 | ||
| 592 | # Abort if any source overlaps with a dest. | 651 | # Abort if any source overlaps with a dest. |
| 593 | for src_uri_str in exp_src_uri_strs: | 652 | for src_uri in exp_src_uris: |
| 594 | src_uri = StorageUri(src_uri_str, False, debug) | ||
| 595 | if (src_uri.equals(dst_uri) or | 653 | if (src_uri.equals(dst_uri) or |
| 596 | # Example case: gsutil cp gs://mybucket/a/bb mybucket | 654 | # Example case: gsutil cp gs://mybucket/a/bb mybucket |
| 597 | (dst_uri.is_cloud_uri() and src_uri.uri.find(dst_uri.uri) != -1)): | 655 | (dst_uri.is_cloud_uri() and src_uri.uri.find(dst_uri.uri) != -1)): |
| 598 | OutputAndExit('Overlapping source and dest URIs not allowed.') | 656 | OutputAndExit('Overlapping source and dest URIs not allowed.') |
| 599 | 657 | ||
| 600 | # Now iterate over expanded src URIs, and perform copy operations. | 658 | # Now iterate over expanded src URIs, and perform copy operations. |
| 601 | for src_uri_str in exp_src_uri_strs: | 659 | for src_uri in exp_src_uris: |
| 602 | src_uri = StorageUri(src_uri_str, False, debug) | 660 | if len(exp_src_uris) > 1: |
| 603 | if len(exp_src_uri_strs) > 1: | ||
| 604 | # Progress indicator | 661 | # Progress indicator |
| 605 | print 'Copying %s...' % src_uri | 662 | print 'Copying %s...' % src_uri |
| 606 | if dst_uri.names_container(): | 663 | if dst_uri.names_container(): |
| 607 | if dst_uri.is_file_uri(): | 664 | if dst_uri.is_file_uri(): |
| 608 | # dest names a directory, so append src obj name to dst obj name | 665 | # dest names a directory, so append src obj name to dst obj name |
| 609 | dst_key_name = dst_uri.object_name + os.sep + src_uri.object_name | 666 | dst_key_name = dst_uri.object_name + os.sep + src_uri.object_name |
| 610 | CheckForDirFileConflict(src_uri, dst_key_name) | 667 | CheckForDirFileConflict(src_uri, dst_key_name) |
| 611 | else: | 668 | else: |
| 612 | # dest names a bucket: use src obj name for dst obj name. | 669 | # dest names a bucket: use src obj name for dst obj name. |
| 613 | dst_key_name = src_uri.object_name | 670 | dst_key_name = src_uri.object_name |
| 614 | else: | 671 | else: |
| 615 | # dest is an object or file: use dst obj name | 672 | # dest is an object or file: use dst obj name |
| 616 | dst_key_name = dst_uri.object_name | 673 | dst_key_name = dst_uri.object_name |
| 617 | new_dst_uri = dst_uri.clone_replace_name(dst_key_name) | 674 | new_dst_uri = dst_uri.clone_replace_name(dst_key_name) |
| 618 | PerformCopy(src_uri, new_dst_uri) | 675 | PerformCopy(src_uri, new_dst_uri, sub_opts, headers) |
| 619 | 676 | ||
| 620 | 677 | ||
| 621 | def PrintAcl(uri, indent): | 678 | def HelpCommand(unused_args, unused_sub_opts, unused_headers=None, |
| 622 | """Print human-readable info in ACL for given URI. | 679 | unused_debug=None): |
| 623 | |||
| 624 | Args: | ||
| 625 | uri: object- or bucket- granularity StorageUri. | ||
| 626 | indent: string to indent each output line. | ||
| 627 | """ | ||
| 628 | acl = uri.get_acl() | ||
| 629 | for grant in acl.acl.grants: | ||
| 630 | if grant.type == 'AmazonCustomerByEmail': | ||
| 631 | grantee = grant.email_address | ||
| 632 | if grant.type == 'CanonicalUser': | ||
| 633 | grantee = grant.id | ||
| 634 | elif grant.type == 'Group': | ||
| 635 | # Extract the group name from the URI (e.g., | ||
| 636 | # "http://acs.amazonaws.com/groups/global/AuthenticatedUsers") | ||
| 637 | grantee = re.sub('.*/', '', grant.uri) | ||
| 638 | else: | ||
| 639 | OutputAndExit('Unexpected grant type "%s".' % grant.type) | ||
| 640 | print '%s%s: %s' % (indent, grantee, grant.permission) | ||
| 641 | |||
| 642 | |||
| 643 | def HelpCommand(unused_args, unused_sub_opts, unused_debug): | ||
| 644 | """Implementation of help command. | 680 | """Implementation of help command. |
| 645 | 681 | ||
| 646 | Args: | 682 | Args: |
| 647 | unused_args: command-line arguments | 683 | unused_args: command-line arguments |
| 648 | unused_sub_opts: command-specific options from getopt. | 684 | unused_sub_opts: command-specific options from getopt. |
| 685 | unused_headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 649 | unused_debug: flag indicating whether to include debug output | 686 | unused_debug: flag indicating whether to include debug output |
| 650 | """ | 687 | """ |
| 651 | 688 | ||
| 652 | OutputUsageAndExit() | 689 | OutputUsageAndExit() |
| 653 | 690 | ||
| 654 | 691 | ||
| 655 | def PrintObjectMetaData(uri): | 692 | def PrintObjectMetaData(uri, headers): |
| 656 | """Print object metadata. | 693 | """Print object metadata. |
| 657 | 694 | ||
| 658 | Args: | 695 | Args: |
| 659 | uri: object-granularity StorageUri | 696 | uri: object-granularity StorageUri |
| 697 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 660 | """ | 698 | """ |
| 661 | print '%s:' % uri | 699 | print '%s:' % uri |
| 662 | key = uri.get_key() | 700 | key = uri.get_key(False, headers) |
| 663 | key.open_read() | 701 | key.open_read() |
| 664 | print '\tObject size:\t%s' % key.size | 702 | print '\tObject size:\t%s' % key.size |
| 665 | print '\tLast mod:\t%s' % key.last_modified | 703 | print '\tLast mod:\t%s' % key.last_modified |
| 704 | if key.cache_control: | ||
| 705 | print '\tCache control:\t%s' % key.cache_control | ||
| 666 | print '\tMIME type:\t%s' % key.content_type | 706 | print '\tMIME type:\t%s' % key.content_type |
| 707 | if key.content_encoding: | ||
| 708 | print '\tContent-Encoding:\t%s' % key.content_encoding | ||
| 667 | print '\tMD5:\t%s' % key.etag.strip('"\'') | 709 | print '\tMD5:\t%s' % key.etag.strip('"\'') |
| 668 | print '\tACL:' % uri | 710 | print '\tACL:\t%s' % uri.get_acl(False, headers) |
| 669 | PrintAcl(uri, '\t\t') | ||
| 670 | 711 | ||
| 671 | 712 | ||
| 672 | def ListCommand(args, sub_opts, debug): | 713 | def ListCommand(args, sub_opts, headers=None, debug=False): |
| 673 | """Implementation of ls command. | 714 | """Implementation of ls command. |
| 674 | 715 | ||
| 675 | Args: | 716 | Args: |
| 676 | args: command-line arguments | 717 | args: command-line arguments |
| 677 | sub_opts: command-specific options from getopt. | 718 | sub_opts: command-specific options from getopt. |
| 719 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 678 | debug: flag indicating whether to include debug output | 720 | debug: flag indicating whether to include debug output |
| 679 | """ | 721 | """ |
| 680 | 722 | ||
| 681 | long_listing = False | 723 | long_listing = False |
| 682 | for o, unused_a in sub_opts: | 724 | for o, unused_a in sub_opts: |
| 683 | if o == '-l': | 725 | if o == '-l': |
| 684 | long_listing = True | 726 | long_listing = True |
| 685 | if not args: | 727 | if not args: |
| 686 | # default to listing all gs buckets | 728 | # default to listing all gs buckets |
| 687 | args = ['gs://'] | 729 | args = ['gs://'] |
| 688 | 730 | ||
| 689 | # First expand all URIs into URIs in exp_args, such that: | 731 | # First expand all URIs into URIs in exp_args, such that: |
| 690 | # a) provider-only URIs ('gs://') are left as-is. | 732 | # a) provider-only URIs ('gs://') are left as-is. |
| 691 | # b) bucket-only URIs ('gs://bucket') with -l option are left as bucket-only | 733 | # b) bucket-only URIs ('gs://bucket') with -l option are left as bucket-only |
| 692 | # URIs. | 734 | # URIs. |
| 693 | # c) bucket-only URIs ('gs://bucket') without -l option are replaced by | 735 | # c) bucket-only URIs ('gs://bucket') without -l option are replaced by |
| 694 | # the list of all the objects in the bucket. | 736 | # the list of all the objects in the bucket. |
| 695 | # d) complete URIs ('gs://bucket/obj') are replaced by the list of all of | 737 | # d) complete URIs ('gs://bucket/obj') are replaced by the list of all of |
| 696 | # the matching objects names (handling URIs that contain wildcards, | 738 | # the matching objects names (handling URIs that contain wildcards, |
| 697 | # as well as wildcard-less URIs). | 739 | # as well as wildcard-less URIs). |
| 698 | exp_args = [] | 740 | exp_args = [] |
| 699 | for uri_str in args: | 741 | for uri_str in args: |
| 700 | uri = StorageUri(uri_str, False, debug) | 742 | uri = StorageUri(uri_str, False, debug) |
| 701 | if not uri.bucket_name: | 743 | if not uri.bucket_name: |
| 702 | # case a: provider-only URI | 744 | # case a: provider-only URI |
| 703 | exp_args.append(uri_str) | 745 | exp_args.append(uri_str) |
| 704 | elif not uri.object_name: | 746 | elif not uri.object_name: |
| 705 | if long_listing: | 747 | if long_listing: |
| 706 | # case b: bucket-only URI with -l option | 748 | # case b: bucket-only URI with -l option |
| 707 | exp_args.append(uri.__str__()) | 749 | exp_args.append(uri.__str__()) |
| 708 | else: | 750 | else: |
| 709 | # case c: bucket-only URI without -l option | 751 | # case c: bucket-only URI without -l option |
| 710 | bucket = uri.get_bucket() | 752 | bucket = uri.get_bucket(False, headers) |
| 711 | for obj in bucket: | 753 | for obj in bucket: |
| 712 | exp_args.append(uri.clone_replace_name(obj.name).uri) | 754 | exp_args.append(uri.clone_replace_name(obj.name).uri) |
| 713 | else: | 755 | else: |
| 714 | # case d: complete URI | 756 | # case d: complete URI |
| 715 | regex = fnmatch.translate(uri.object_name) | 757 | regex = fnmatch.translate(uri.object_name) |
| 716 | bucket = uri.get_bucket() | 758 | bucket = uri.get_bucket(False, headers) |
| 717 | for obj in bucket: | 759 | for obj in bucket: |
| 718 | if re.match(regex, obj.name): | 760 | if re.match(regex, obj.name): |
| 719 | exp_args.append(uri.clone_replace_name(obj.name).uri) | 761 | exp_args.append(uri.clone_replace_name(obj.name).uri) |
| 720 | 762 | ||
| 721 | # Handle the case of a complete URI that didn't match anything | 763 | # Handle the case of a complete URI that didn't match anything |
| 722 | if uri.names_singleton() and not exp_args: | 764 | if uri.names_singleton() and not exp_args: |
| 723 | ReportNoMatchesAndExit(uri) | 765 | ReportNoMatchesAndExit(uri) |
| 724 | 766 | ||
| 725 | # Now iterate over all URIs in exp_args and print requested info for each | 767 | # Now iterate over all URIs in exp_args and print requested info for each |
| 726 | for uri_str in exp_args: | 768 | for uri_str in exp_args: |
| 727 | uri = StorageUri(uri_str, False, debug) | 769 | uri = StorageUri(uri_str, False, debug) |
| 728 | if not uri.bucket_name: | 770 | if not uri.bucket_name: |
| 729 | if long_listing: | 771 | if long_listing: |
| 730 | # Provider long listing: print metadata for all buckets | 772 | # Provider long listing: print metadata for all buckets |
| 731 | buckets = uri.get_all_buckets() | 773 | buckets = uri.get_all_buckets() |
| 732 | for bucket in buckets: | 774 | for bucket in buckets: |
| 733 | bucket_uri = StorageUri(uri.provider + '://' + bucket.name, False, | 775 | bucket_uri = StorageUri(uri.provider + '://' + bucket.name, False, |
| 734 | debug) | 776 | debug) |
| 735 | print '%s:\n\tACL:' % bucket_uri | 777 | print '%s:\n\tACL:\t%s' % (bucket_uri, |
| 736 | PrintAcl(bucket_uri, '\t\t') | 778 | bucket_uri.get_acl(False, headers)) |
| 737 | else: | 779 | else: |
| 738 | # Provider short listing: list all buckets | 780 | # Provider short listing: list all buckets |
| 739 | buckets = uri.get_all_buckets() | 781 | buckets = uri.get_all_buckets() |
| 740 | for bucket in buckets: | 782 | for bucket in buckets: |
| 741 | bucket_uri = StorageUri(uri.provider + '://' + bucket.name, False, | 783 | bucket_uri = StorageUri(uri.provider + '://' + bucket.name, False, |
| 742 | debug) | 784 | debug) |
| 743 | print bucket_uri | 785 | print bucket_uri |
| 744 | elif not uri.object_name: | 786 | elif not uri.object_name: |
| 745 | if long_listing: | 787 | if long_listing: |
| 746 | # Bucket long listing | 788 | # Bucket long listing |
| 747 | print '%s:\n\tACL:' % uri | 789 | print '%s:\n\tACL:' % uri |
| 748 | PrintAcl(uri, '\t\t') | 790 | print '\t\t%s' % uri.get_acl(False, headers) |
| 749 | else: | 791 | else: |
| 750 | # Bucket short listing | 792 | # Bucket short listing |
| 751 | bucket = uri.get_bucket() | 793 | bucket = uri.get_bucket(False, headers) |
| 752 | for obj in bucket: | 794 | for obj in bucket: |
| 753 | print '%s://%s/%s' % (uri.provider, uri.bucket_name, obj.name) | 795 | print '%s://%s/%s' % (uri.provider, uri.bucket_name, obj.name) |
| 754 | else: | 796 | else: |
| 755 | if long_listing: | 797 | if long_listing: |
| 756 | # Object long listing | 798 | # Object long listing |
| 757 | PrintObjectMetaData(uri) | 799 | PrintObjectMetaData(uri, headers) |
| 758 | else: | 800 | else: |
| 759 | # Object short listing | 801 | # Object short listing |
| 760 | print uri | 802 | print uri |
| 761 | 803 | ||
| 762 | 804 | ||
| 763 | def MakeBucketsCommand(args, unused_sub_opts, debug): | 805 | def MakeBucketsCommand(args, unused_sub_opts, headers=None, debug=False): |
| 764 | """Implementation of mb command. | 806 | """Implementation of mb command. |
| 765 | 807 | ||
| 766 | Args: | 808 | Args: |
| 767 | args: command-line arguments | 809 | args: command-line arguments |
| 768 | unused_sub_opts: command-specific options from getopt. | 810 | unused_sub_opts: command-specific options from getopt. |
| 811 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 769 | debug: flag indicating whether to include debug output | 812 | debug: flag indicating whether to include debug output |
| 770 | """ | 813 | """ |
| 771 | 814 | ||
| 772 | for bucket_uri_str in args: | 815 | for bucket_uri_str in args: |
| 773 | bucket_uri = StorageUri(bucket_uri_str, False, debug) | 816 | bucket_uri = StorageUri(bucket_uri_str, False, debug) |
| 774 | if len(args) > 1: | 817 | if len(args) > 1: |
| 775 | # Progress indicator | 818 | # Progress indicator |
| 776 | print 'Creating %s...' % bucket_uri | 819 | print 'Creating %s...' % bucket_uri |
| 777 | bucket_uri.create_bucket() | 820 | bucket_uri.create_bucket(headers) |
| 778 | 821 | ||
| 779 | 822 | ||
| 780 | def MoveObjsCommand(args, sub_opts, debug): | 823 | def MoveObjsCommand(args, sub_opts, headers=None, debug=False): |
| 781 | """Implementation of mv command. | 824 | """Implementation of mv command. |
| 782 | 825 | ||
| 783 | Note that there is no atomic rename operation - this command is simply | 826 | Note that there is no atomic rename operation - this command is simply |
| 784 | a shorthand for 'cp' followed by 'rm'. | 827 | a shorthand for 'cp' followed by 'rm'. |
| 785 | 828 | ||
| 786 | Args: | 829 | Args: |
| 787 | args: command-line arguments | 830 | args: command-line arguments |
| 788 | sub_opts: command-specific options from getopt. | 831 | sub_opts: command-specific options from getopt. |
| 832 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 789 | debug: flag indicating whether to include debug output | 833 | debug: flag indicating whether to include debug output |
| 790 | """ | 834 | """ |
| 791 | 835 | ||
| 792 | # Refuse to delete a bucket or directory src URI (force users to explicitly | 836 | # Refuse to delete a bucket or directory src URI (force users to explicitly |
| 793 | # do that as a separate operation). | 837 | # do that as a separate operation). |
| 794 | src_uri_to_check = StorageUri(args[0]) | 838 | src_uri_to_check = StorageUri(args[0]) |
| 795 | if src_uri_to_check.names_container(): | 839 | if src_uri_to_check.names_container(): |
| 796 | OutputAndExit('Will not remove source buckets or directories. You should ' | 840 | OutputAndExit('Will not remove source buckets or directories. You should ' |
| 797 | 'separately copy and remove for that purpose.') | 841 | 'separately copy and remove for that purpose.') |
| 798 | 842 | ||
| 799 | if len(args) > 2: | 843 | if len(args) > 2: |
| 800 | InsistUriNamesContainer('mv', StorageUri(args[-1])) | 844 | InsistUriNamesContainer('mv', StorageUri(args[-1])) |
| 801 | 845 | ||
| 802 | CopyObjsCommand(args, sub_opts, debug) | 846 | CopyObjsCommand(args, sub_opts, headers, debug) |
| 803 | RemoveObjsCommand(args[0:1], sub_opts, debug) | 847 | RemoveObjsCommand(args[0:1], sub_opts, headers, debug) |
| 804 | 848 | ||
| 805 | 849 | ||
| 806 | def RemoveBucketsCommand(args, unused_sub_opts, debug): | 850 | def RemoveBucketsCommand(args, unused_sub_opts, headers=None, debug=False): |
| 807 | """Implementation of rb command. | 851 | """Implementation of rb command. |
| 808 | 852 | ||
| 809 | Args: | 853 | Args: |
| 810 | args: command-line arguments | 854 | args: command-line arguments |
| 811 | unused_sub_opts: command-specific options from getopt. | 855 | unused_sub_opts: command-specific options from getopt. |
| 856 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 812 | debug: flag indicating whether to include debug output | 857 | debug: flag indicating whether to include debug output |
| 813 | """ | 858 | """ |
| 814 | 859 | ||
| 815 | for bucket_uri_str in args: | 860 | for bucket_uri_str in args: |
| 816 | bucket_uri = StorageUri(bucket_uri_str, True, debug) | 861 | bucket_uri = StorageUri(bucket_uri_str, True, debug) |
| 817 | if len(args) > 1: | 862 | if len(args) > 1: |
| 818 | # Progress indicator | 863 | # Progress indicator |
| 819 | print 'Removing %s...' % bucket_uri | 864 | print 'Removing %s...' % bucket_uri |
| 820 | bucket_uri.delete_bucket() | 865 | bucket_uri.delete_bucket(headers) |
| 821 | 866 | ||
| 822 | 867 | ||
| 823 | def RemoveObjsCommand(args, unused_sub_opts, debug): | 868 | def RemoveObjsCommand(args, unused_sub_opts, headers=None, debug=False): |
| 824 | """Implementation of rm command. | 869 | """Implementation of rm command. |
| 825 | 870 | ||
| 826 | Args: | 871 | Args: |
| 827 | args: command-line arguments | 872 | args: command-line arguments |
| 828 | unused_sub_opts: command-specific options from getopt. | 873 | unused_sub_opts: command-specific options from getopt. |
| 874 | headers: dictionary containing optional HTTP headers to pass to boto. | ||
| 829 | debug: flag indicating whether to include debug output | 875 | debug: flag indicating whether to include debug output |
| 830 | """ | 876 | """ |
| 831 | 877 | ||
| 832 | # Expand object name globs, if any. | 878 | # Expand object name globs, if any. |
| 833 | exp_args = [] | 879 | exp_uris = [] |
| 834 | for uri_str in args: | 880 | for uri_str in args: |
| 835 | for exp_uri_str in ExpandStorageUriGlob(uri_str, debug): | 881 | for exp_uri in ExpandStorageUriGlob(uri_str, headers, debug): |
| 836 | exp_args.append(exp_uri_str) | 882 | exp_uris.append(exp_uri) |
| 837 | for uri_str in exp_args: | 883 | for uri in exp_uris: |
| 838 | uri = StorageUri(uri_str, False, debug) | ||
| 839 | if uri.names_container(): | 884 | if uri.names_container(): |
| 840 | OutputAndExit('"rm" command will not remove buckets. To delete this ' | 885 | OutputAndExit('"rm" command will not remove buckets. To delete this ' |
| 841 | 'bucket do:\n\tgsutil rm %s/*\n\tgsutil rb %s' % | 886 | 'bucket do:\n\tgsutil rm %s/*\n\tgsutil rb %s' % |
| 842 | (uri.uri, uri.uri)) | 887 | (uri.uri, uri.uri)) |
| 843 | if len(exp_args) > 1: | 888 | if len(exp_uris) > 1: |
| 844 | # Progress indicator | 889 | # Progress indicator |
| 845 | print 'Removing %s...' % uri | 890 | print 'Removing %s...' % uri |
| 846 | uri.delete_key() | 891 | uri.delete_key(False, headers) |
| 847 | 892 | ||
| 848 | 893 | ||
| 849 | def HaveFileUris(args): | 894 | def HaveFileUris(args): |
| 850 | """Checks whether args contains any file URIs. | 895 | """Checks whether args contains any file URIs. |
| 851 | 896 | ||
| 852 | Args: | 897 | Args: |
| 853 | args: command-line arguments | 898 | args: command-line arguments |
| 854 | 899 | ||
| 855 | Returns: | 900 | Returns: |
| 856 | True if args contains any file URIs. | 901 | True if args contains any file URIs. |
| 857 | """ | 902 | """ |
| 858 | for uri_str in args: | 903 | for uri_str in args: |
| 859 | if uri_str.lower().startswith('file://') or uri_str.find(':') == -1: | 904 | if uri_str.lower().startswith('file://') or uri_str.find(':') == -1: |
| 860 | return True | 905 | return True |
| 861 | return False | 906 | return False |
| 862 | 907 | ||
| 863 | 908 | ||
| 864 | def HaveProviderUris(args): | 909 | def HaveProviderUris(args): |
| 865 | """Checks whether args contains any provider URIs (like 'gs://'). | 910 | """Checks whether args contains any provider URIs (like 'gs://'). |
| 866 | 911 | ||
| 867 | Args: | 912 | Args: |
| 868 | args: command-line arguments | 913 | args: command-line arguments |
| 869 | 914 | ||
| 870 | Returns: | 915 | Returns: |
| 871 | True if args contains any provider URIs. | 916 | True if args contains any provider URIs. |
| 872 | """ | 917 | """ |
| 873 | for uri_str in args: | 918 | for uri_str in args: |
| 874 | if re.match('^[a-z]+://$', uri_str): | 919 | if re.match('^[a-z]+://$', uri_str): |
| 875 | return True | 920 | return True |
| 876 | return False | 921 | return False |
| 877 | 922 | ||
| 878 | 923 | ||
| 879 | def GetBotoConfigFileList(): | 924 | def GetBotoConfigFileList(): |
| 880 | """Returns list of boto config files that exist.""" | 925 | """Returns list of boto config files that exist.""" |
| 881 | 926 | ||
| 882 | config_paths = boto.pyami.config.BotoConfigLocations | 927 | config_paths = boto.pyami.config.BotoConfigLocations |
| 883 | if 'AWS_CREDENTIAL_FILE' in os.environ: | 928 | if 'AWS_CREDENTIAL_FILE' in os.environ: |
| 884 | config_paths.append(os.environ['AWS_CREDENTIAL_FILE']) | 929 | config_paths.append(os.environ['AWS_CREDENTIAL_FILE']) |
| 885 | config_files = {} | 930 | config_files = {} |
| 886 | for config_path in config_paths: | 931 | for config_path in config_paths: |
| 887 | if os.path.exists(config_path): | 932 | if os.path.exists(config_path): |
| 888 | config_files[config_path] = 1 | 933 | config_files[config_path] = 1 |
| 889 | cf_list = [] | 934 | cf_list = [] |
| 890 | for config_file in config_files: | 935 | for config_file in config_files: |
| 891 | cf_list.append(config_file) | 936 | cf_list.append(config_file) |
| 892 | return cf_list | 937 | return cf_list |
| 893 | 938 | ||
| 894 | 939 | ||
| 895 | prelude_config_content = ( | 940 | prelude_config_content = ( |
| 896 | """# This file contains credentials and other configuration information needed | 941 | """# This file contains credentials and other configuration information needed |
| 897 | # by the boto library, used by gsutil. You can edit this file (e.g., to add | 942 | # by the boto library, used by gsutil. You can edit this file (e.g., to add |
| 898 | # credentials) but be careful not to mis-edit any of the variable names (like | 943 | # credentials) but be careful not to mis-edit any of the variable names (like |
| 899 | # "gs_access_key_id") or remove important markers (like the "[Credentials]" and | 944 | # "gs_access_key_id") or remove important markers (like the "[Credentials]" and |
| 900 | # "[Boto]" section delimeters). | 945 | # "[Boto]" section delimeters). |
| 901 | # | 946 | # |
| 902 | """) | 947 | """) |
| 903 | 948 | ||
| 904 | additional_config_content = """ | 949 | additional_config_content = """ |
| 905 | 950 | ||
| 906 | [Boto] | 951 | [Boto] |
| 907 | 952 | ||
| 908 | # To use a proxy, edit and uncomment the proxy and proxy_port lines. If you | 953 | # To use a proxy, edit and uncomment the proxy and proxy_port lines. If you |
| 909 | # need a user/password with this proxy, edit and uncomment those lines as well. | 954 | # need a user/password with this proxy, edit and uncomment those lines as well. |
| 910 | #proxy = <proxy host> | 955 | #proxy = <proxy host> |
| 911 | #proxy_port = <proxy port> | 956 | #proxy_port = <proxy port> |
| 912 | #proxy_user = <your proxy user name> | 957 | #proxy_user = <your proxy user name> |
| 913 | #proxy_pass = <your proxy password> | 958 | #proxy_pass = <your proxy password> |
| 914 | 959 | ||
| 915 | # Set 'is_secure' to False to cause boto to connect using HTTP instead of the | 960 | # Set 'is_secure' to False to cause boto to connect using HTTP instead of the |
| 916 | # default HTTPS. This is useful if you want to capture/analyze traffic | 961 | # default HTTPS. This is useful if you want to capture/analyze traffic |
| 917 | # (e.g., with tcpdump). | 962 | # (e.g., with tcpdump). |
| 918 | #is_secure = False | 963 | #is_secure = False |
| 919 | 964 | ||
| 920 | # 'debug' controls the level of debug messages printed: 0 for none, 1 | 965 | # 'debug' controls the level of debug messages printed: 0 for none, 1 |
| 921 | # for basic boto debug, 2 for all boto debug plus HTTP requests/responses. | 966 | # for basic boto debug, 2 for all boto debug plus HTTP requests/responses. |
| 922 | # Note: 'gsutil -d' sets debug to 2 for that one command run. | 967 | # Note: 'gsutil -d' sets debug to 2 for that one command run. |
| 923 | #debug = <0, 1, or 2> | 968 | #debug = <0, 1, or 2> |
| 924 | 969 | ||
| 925 | # 'num_retries' controls the number of retry attempts made when errors occur. | 970 | # 'num_retries' controls the number of retry attempts made when errors occur. |
| 926 | # The default is 5. | 971 | # The default is 5. |
| 927 | #num_retries = <integer value> | 972 | #num_retries = <integer value> |
| 928 | """ | 973 | """ |
| 929 | 974 | ||
| 930 | 975 | ||
| 931 | def CreateBotoConfigFile(): | 976 | def CreateBotoConfigFile(): |
| 932 | """Creates a boto config file interactively, containing needed credentials.""" | 977 | """Creates a boto config file interactively, containing needed credentials.""" |
| 933 | 978 | ||
| 934 | if 'HOME' in os.environ: | 979 | if 'HOME' in os.environ: |
| 935 | config_path = ('%s%s.boto' % (os.environ['HOME'], os.sep)) | 980 | config_path = ('%s%s.boto' % (os.environ['HOME'], os.sep)) |
| 936 | else: | 981 | else: |
| 937 | config_path = ('.%s.boto' % os.sep) | 982 | config_path = ('.%s.boto' % os.sep) |
| 938 | sys.stderr.write('You have no boto config file. This script will create ' | 983 | sys.stderr.write('You have no boto config file. This script will create ' |
| 939 | 'one at\n%s\ncontaining your credentials, based on your ' | 984 | 'one at\n%s\ncontaining your credentials, based on your ' |
| 940 | 'responses to the following questions.\n\n' % config_path) | 985 | 'responses to the following questions.\n\n' % config_path) |
| 941 | 986 | ||
| 942 | provider_map = {'Amazon': 'aws', 'Google': 'gs'} | 987 | provider_map = {'Amazon': 'aws', 'Google': 'gs'} |
| 943 | uri_map = {'Amazon': 's3', 'Google': 'gs'} | 988 | uri_map = {'Amazon': 's3', 'Google': 'gs'} |
| 944 | key_ids = {} | 989 | key_ids = {} |
| 945 | sec_keys = {} | 990 | sec_keys = {} |
| 946 | got_creds = False | 991 | got_creds = False |
| 947 | for provider in provider_map: | 992 | for provider in provider_map: |
| 948 | if provider == 'Google': | 993 | if provider == 'Google': |
| 949 | key_ids[provider] = raw_input('What is your %s access key ID? ' % | 994 | key_ids[provider] = raw_input('What is your %s access key ID? ' % |
| 950 | provider) | 995 | provider) |
| 951 | sec_keys[provider] = raw_input('What is your %s secret access key? ' % | 996 | sec_keys[provider] = raw_input('What is your %s secret access key? ' % |
| 952 | provider) | 997 | provider) |
| 953 | got_creds = True | 998 | got_creds = True |
| 954 | if not key_ids[provider] or not sec_keys[provider]: | 999 | if not key_ids[provider] or not sec_keys[provider]: |
| 955 | OutputAndExit('Incomplete credentials provided. Please try again.') | 1000 | OutputAndExit('Incomplete credentials provided. Please try again.') |
| 956 | if not got_creds: | 1001 | if not got_creds: |
| 957 | OutputAndExit('No credentials provided. Please try again.') | 1002 | OutputAndExit('No credentials provided. Please try again.') |
| 958 | cfp = open(config_path, 'w') | 1003 | cfp = open(config_path, 'w') |
| 959 | if not cfp: | 1004 | if not cfp: |
| 960 | OutputAndExit('Unable to write "%s".' % config_path) | 1005 | OutputAndExit('Unable to write "%s".' % config_path) |
| 961 | os.chmod(config_path, stat.S_IRUSR | stat.S_IWUSR) | 1006 | os.chmod(config_path, stat.S_IRUSR | stat.S_IWUSR) |
| 962 | cfp.write(prelude_config_content) | 1007 | cfp.write(prelude_config_content) |
| 963 | cfp.write('# This file was created by gsutil version "%s"\n# at %s.\n\n\n' | 1008 | cfp.write('# This file was created by gsutil version "%s"\n# at %s.\n\n\n' |
| 964 | % (LoadVersionString(), | 1009 | % (LoadVersionString(), |
| 965 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) | 1010 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) |
| 966 | cfp.write('[Credentials]\n\n') | 1011 | cfp.write('[Credentials]\n\n') |
| 967 | for provider in provider_map: | 1012 | for provider in provider_map: |
| 968 | prefix = provider_map[provider] | 1013 | prefix = provider_map[provider] |
| 969 | uri_scheme = uri_map[provider] | 1014 | uri_scheme = uri_map[provider] |
| 970 | if provider in key_ids and provider in sec_keys: | 1015 | if provider in key_ids and provider in sec_keys: |
| 971 | cfp.write('# %s credentials ("%s://" URIs):\n' % | 1016 | cfp.write('# %s credentials ("%s://" URIs):\n' % |
| 972 | (provider, uri_scheme)) | 1017 | (provider, uri_scheme)) |
| 973 | cfp.write('%s_access_key_id = %s\n' % (prefix, key_ids[provider])) | 1018 | cfp.write('%s_access_key_id = %s\n' % (prefix, key_ids[provider])) |
| 974 | cfp.write('%s_secret_access_key = %s\n' % (prefix, sec_keys[provider])) | 1019 | cfp.write('%s_secret_access_key = %s\n' % (prefix, sec_keys[provider])) |
| 975 | else: | 1020 | else: |
| 976 | cfp.write('# To add %s credentials ("%s://" URIs), edit and ' | 1021 | cfp.write('# To add %s credentials ("%s://" URIs), edit and ' |
| 977 | 'uncomment the\n# following two lines:\n' | 1022 | 'uncomment the\n# following two lines:\n' |
| 978 | '#%s_access_key_id = <your %s access key ID>\n' | 1023 | '#%s_access_key_id = <your %s access key ID>\n' |
| 979 | '#%s_secret_access_key = <your %s secret access key>\n' % | 1024 | '#%s_secret_access_key = <your %s secret access key>\n' % |
| 980 | (provider, uri_scheme, prefix, provider, prefix, provider)) | 1025 | (provider, uri_scheme, prefix, provider, prefix, provider)) |
| 981 | cfp.write('# The ability to specify an alternate storage host is primarily ' | 1026 | cfp.write('# The ability to specify an alternate storage host is primarily ' |
| 982 | 'for developers.\n' | 1027 | 'for developers.\n' |
| 983 | '#%s_host = <alternate storage host address>\n\n' % (prefix)) | 1028 | '#%s_host = <alternate storage host address>\n\n' % (prefix)) |
| 984 | cfp.write(additional_config_content) | 1029 | cfp.write(additional_config_content) |
| 985 | cfp.close() | 1030 | cfp.close() |
| 986 | OutputAndExit('\nConfiguration file "%s" created. If you need to use\na ' | 1031 | OutputAndExit('\nConfiguration file "%s" created. If you need to use\na ' |
| 987 | 'proxy to access the Internet please see the instructions in ' | 1032 | 'proxy to access the Internet please see the instructions in ' |
| 988 | 'that file.\nPlease try running gsutil again now.' % | 1033 | 'that file.\nPlease try running gsutil again now.' % |
| 989 | config_path) | 1034 | config_path) |
| 990 | 1035 | ||
| 991 | 1036 | ||
| 992 | def SetupConfigIfNeeded(): | 1037 | def SetupConfigIfNeeded(): |
| 993 | """Interactively creates boto credential/config file if needed.""" | 1038 | """Interactively creates boto credential/config file if needed.""" |
| 994 | 1039 | ||
| 995 | config = boto.config | 1040 | config = boto.config |
| 996 | has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and | 1041 | has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and |
| 997 | config.has_option('Credentials', 'gs_secret_access_key')) | 1042 | config.has_option('Credentials', 'gs_secret_access_key')) |
| 998 | has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and | 1043 | has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and |
| 999 | config.has_option('Credentials', 'aws_secret_access_key')) | 1044 | config.has_option('Credentials', 'aws_secret_access_key')) |
| 1000 | if not has_goog_creds and not has_amzn_creds: | 1045 | if not has_goog_creds and not has_amzn_creds: |
| 1001 | config_file_list = GetBotoConfigFileList() | 1046 | config_file_list = GetBotoConfigFileList() |
| 1002 | if config_file_list: | 1047 | if config_file_list: |
| 1003 | OutputAndExit('You have no storage service credentials in any of the ' | 1048 | OutputAndExit('You have no storage service credentials in any of the ' |
| 1004 | 'following boto config\nfiles. Please add your credentials ' | 1049 | 'following boto config\nfiles. Please add your credentials ' |
| 1005 | 'as described in the gsutil README file, or else\ndelete' | 1050 | 'as described in the gsutil README file, or else\ndelete' |
| 1006 | 'these files and re-run this script to re-create them:\n%s' | 1051 | 'these files and re-run this script to re-create them:\n%s' |
| 1007 | % config_file_list) | 1052 | % config_file_list) |
| 1008 | else: | 1053 | else: |
| 1009 | CreateBotoConfigFile() | 1054 | CreateBotoConfigFile() |
| 1010 | 1055 | ||
| 1011 | 1056 | ||
| 1012 | # Maps command name to | 1057 | # Maps command name to |
| 1013 | # [function, min_args, max_args, supported_sub_args, file_uri_ok, | 1058 | # [function, min_args, max_args, supported_sub_args, file_uri_ok, |
| 1014 | # provider_uri_ok, uris_start_arg] | 1059 | # provider_uri_ok, uris_start_arg] |
| 1015 | NO_MAX = sys.maxint | 1060 | NO_MAX = sys.maxint |
| 1016 | commands = { | 1061 | commands = { |
| 1017 | 'cat': [CatCommand, 0, NO_MAX, 'h', False, True, 0], | 1062 | 'cat': [CatCommand, 0, NO_MAX, 'h', False, True, 0], |
| 1018 | 'cp': [CopyObjsCommand, 2, NO_MAX, '', True, False, 0], | 1063 | 'cp': [CopyObjsCommand, 2, NO_MAX, 'a:tz:', True, False, 0], |
| 1019 | 'getacl': [GetAclCommand, 1, 1, '', False, False, 0], | 1064 | 'getacl': [GetAclCommand, 1, 1, '', False, False, 0], |
| 1020 | 'help': [HelpCommand, 0, 0, '', False, False, 0], | 1065 | 'help': [HelpCommand, 0, 0, '', False, False, 0], |
| 1021 | 'ls': [ListCommand, 0, NO_MAX, 'l', False, True, 0], | 1066 | 'ls': [ListCommand, 0, NO_MAX, 'l', False, True, 0], |
| 1022 | 'mb': [MakeBucketsCommand, 1, NO_MAX, '', False, False, 0], | 1067 | 'mb': [MakeBucketsCommand, 1, NO_MAX, '', False, False, 0], |
| 1023 | 'mv': [MoveObjsCommand, 2, NO_MAX, '', True, False, 0], | 1068 | 'mv': [MoveObjsCommand, 2, NO_MAX, '', True, False, 0], |
| 1024 | 'rb': [RemoveBucketsCommand, 1, NO_MAX, '', False, False, 0], | 1069 | 'rb': [RemoveBucketsCommand, 1, NO_MAX, '', False, False, 0], |
| 1025 | 'rm': [RemoveObjsCommand, 1, NO_MAX, '', False, False, 0], | 1070 | 'rm': [RemoveObjsCommand, 1, NO_MAX, '', False, False, 0], |
| 1026 | 'setacl': [SetAclCommand, 2, NO_MAX, '', False, False, 1], | 1071 | 'setacl': [SetAclCommand, 2, NO_MAX, '', False, False, 1], |
| 1027 | 'update': [UpdateCommand, 0, 0, 'f', False, False, 0] | 1072 | 'update': [UpdateCommand, 0, 0, 'f', False, False, 0] |
| 1028 | } | 1073 | } |
| 1029 | 1074 | ||
| 1030 | 1075 | ||
| 1031 | def main(): | 1076 | def main(): |
| 1032 | if float('%d.%d%d' | 1077 | if float('%d.%d%d' |
| 1033 | %(sys.version_info[0], sys.version_info[1], sys.version_info[2]) | 1078 | %(sys.version_info[0], sys.version_info[1], sys.version_info[2]) |
| 1034 | ) < 2.51: | 1079 | ) < 2.51: |
| 1035 | OutputAndExit('This tool requires Python 2.5.1 or higher.') | 1080 | OutputAndExit('This tool requires Python 2.5.1 or higher.') |
| 1036 | 1081 | ||
| 1037 | # If user enters no commands just print the usage info. | 1082 | # If user enters no commands just print the usage info. |
| 1038 | if len(sys.argv) == 1: | 1083 | if len(sys.argv) == 1: |
| 1039 | OutputUsageAndExit() | 1084 | OutputUsageAndExit() |
| 1040 | try: | 1085 | try: |
| 1041 | opts, args = getopt.getopt(sys.argv[1:], 'dh', ['debug', 'help']) | 1086 | opts, args = getopt.getopt(sys.argv[1:], 'd?h:', |
| 1042 | if len(args) == 0: | 1087 | ['debug', 'help', 'header']) |
| 1088 | if not args: | ||
| 1043 | command = 'help' | 1089 | command = 'help' |
| 1044 | else: | 1090 | else: |
| 1045 | command = args[0] | 1091 | command = args[0] |
| 1046 | if command not in commands: | 1092 | if command not in commands: |
| 1047 | OutputAndExit('Invalid command "%s".' % command) | 1093 | OutputAndExit('Invalid command "%s".' % command) |
| 1048 | command_info = commands[command] | 1094 | command_info = commands[command] |
| 1049 | command_func = command_info[0] | 1095 | command_func = command_info[0] |
| 1050 | min_arg_count = command_info[1] | 1096 | min_arg_count = command_info[1] |
| 1051 | max_arg_count = command_info[2] | 1097 | max_arg_count = command_info[2] |
| 1052 | supported_sub_args = command_info[3] | 1098 | supported_sub_args = command_info[3] |
| 1053 | file_uri_ok = command_info[4] | 1099 | file_uri_ok = command_info[4] |
| 1054 | provider_uri_ok = command_info[5] | 1100 | provider_uri_ok = command_info[5] |
| 1055 | uris_start_arg = command_info[6] | 1101 | uris_start_arg = command_info[6] |
| 1056 | sub_opts, args = getopt.getopt(args[1:], supported_sub_args) | 1102 | sub_opts, args = getopt.getopt(args[1:], supported_sub_args) |
| 1057 | if len(args) < min_arg_count or len(args) > max_arg_count: | 1103 | if len(args) < min_arg_count or len(args) > max_arg_count: |
| 1058 | OutputAndExit('Wrong number of arguments for "%s" command.' % command) | 1104 | OutputAndExit('Wrong number of arguments for "%s" command.' % command) |
| 1059 | if not file_uri_ok and HaveFileUris(args[uris_start_arg:]): | 1105 | if not file_uri_ok and HaveFileUris(args[uris_start_arg:]): |
| 1060 | OutputAndExit('"%s" command does not support "file://" URIs.' % command) | 1106 | OutputAndExit('"%s" command does not support "file://" URIs.' % command) |
| 1061 | if not provider_uri_ok and HaveProviderUris(args[uris_start_arg:]): | 1107 | if not provider_uri_ok and HaveProviderUris(args[uris_start_arg:]): |
| 1062 | OutputAndExit('"%s" command does not support provider-only URIs.' % | 1108 | OutputAndExit('"%s" command does not support provider-only URIs.' % |
| 1063 | command) | 1109 | command) |
| 1064 | except InvalidUriError, e: | 1110 | except InvalidUriError, e: |
| 1065 | OutputAndExit(e.message) | 1111 | OutputAndExit(e.message) |
| 1066 | except getopt.GetoptError, e: | 1112 | except getopt.GetoptError, e: |
| 1067 | OutputAndExit(e.msg) | 1113 | OutputAndExit(e.msg) |
| 1068 | 1114 | ||
| 1069 | SetupConfigIfNeeded() | 1115 | SetupConfigIfNeeded() |
| 1070 | 1116 | ||
| 1071 | debug = 0 | 1117 | debug = 0 |
| 1072 | for o, unused_a in opts: | 1118 | headers = {} |
| 1119 | for o, a in opts: | ||
| 1073 | if o in ('-d', '--debug'): | 1120 | if o in ('-d', '--debug'): |
| 1074 | # Debug level 2 causes httplib output plus boto debug output | 1121 | # Debug level 2 causes httplib output plus boto debug output |
| 1075 | debug = 2 | 1122 | debug = 2 |
| 1076 | if o in ('-h', '--help'): | 1123 | if o in ('-?', '--help'): |
| 1077 | OutputUsageAndExit() | 1124 | OutputUsageAndExit() |
| 1125 | if o in ('-h', '--header'): | ||
| 1126 | (hdr_name, unused_ptn, hdr_val) = a.partition(':') | ||
| 1127 | if not hdr_name or not hdr_val: | ||
| 1128 | OutputUsageAndExit() | ||
| 1129 | headers[hdr_name] = hdr_val | ||
| 1078 | 1130 | ||
| 1079 | try: | 1131 | try: |
| 1080 | command_func(args, sub_opts, debug) | 1132 | command_func(args, sub_opts, headers, debug) |
| 1081 | except BotoClientError, e: | 1133 | except BotoClientError, e: |
| 1082 | OutputAndExit('Failure: %s.' % e.reason) | 1134 | OutputAndExit('BotoClientError,: %s.' % e.reason) |
| 1083 | except boto.exception.InvalidUriError, e: | 1135 | except InvalidAclError, e: |
| 1084 | OutputAndExit('Failure: %s.' % e.message) | 1136 | OutputAndExit('InvalidAclError: %s.' % e.message) |
| 1137 | except InvalidUriError, e: | ||
| 1138 | OutputAndExit('InvalidUriError: %s.' % e.message) | ||
| 1085 | except S3ResponseError, e: | 1139 | except S3ResponseError, e: |
| 1086 | detail_start = e.body.find('<Details>') | 1140 | detail_start = e.body.find('<Details>') |
| 1087 | detail_end = e.body.find('</Details>') | 1141 | detail_end = e.body.find('</Details>') |
| 1088 | if detail_start != -1 and detail_end != -1: | 1142 | if detail_start != -1 and detail_end != -1: |
| 1089 | detail = e.body[detail_start+9:detail_end] | 1143 | detail = e.body[detail_start+9:detail_end] |
| 1090 | OutputAndExit('Failure: status=%d, code=%s, reason=%s, detail=%s.' % | 1144 | OutputAndExit('S3ResponseError: status=%d, code=%s, reason=%s, detail=%s.' |
| 1091 | (e.status, e.code, e.reason, detail)) | 1145 | % (e.status, e.code, e.reason, detail)) |
| 1092 | else: | 1146 | else: |
| 1093 | OutputAndExit('Failure: status=%d, code=%s, reason=%s.' % | 1147 | OutputAndExit('S3ResponseError:: status=%d, code=%s, reason=%s.' % |
| 1094 | (e.status, e.code, e.reason)) | 1148 | (e.status, e.code, e.reason)) |
| 1095 | except AttributeError, e: | 1149 | except AttributeError, e: |
| 1096 | if e.message.find('aws_secret_access_key') != -1: | 1150 | if e.message.find('aws_secret_access_key') != -1: |
| 1097 | OutputAndExit('Missing credentials for the given URI(s). Does your ' | 1151 | OutputAndExit('Missing credentials for the given URI(s). Does your ' |
| 1098 | 'boto config file contain all needed credentials?') | 1152 | 'boto config file contain all needed credentials?') |
| 1099 | else: | 1153 | else: |
| 1100 | OutputAndExit(e.message) | 1154 | OutputAndExit(e.message) |
| 1101 | except Exception, e: | 1155 | except Exception, e: |
| 1102 | OutputAndExit('Failure: %s.' % e) | 1156 | OutputAndExit('Failure: %s.' % e) |
| 1103 | 1157 | ||
| 1104 | if __name__ == '__main__': | 1158 | if __name__ == '__main__': |
| 1105 | main() | 1159 | main() |
| 1106 | sys.exit(0) | 1160 | sys.exit(0) |
|