[DEBUG] [2015-08-27 10:41:19,082] [main] [org.jets3t.service.Jets3tProperties]: s3service.https-only=true [DEBUG] [2015-08-27 10:41:19,082] [main] [org.jets3t.service.Jets3tProperties]: storage-service.internal-error-retry-max=5 [DEBUG] [2015-08-27 10:41:19,082] [main] [org.jets3t.service.Jets3tProperties]: storage-service.apply-recommended-dns-caching-defaults=true [DEBUG] [2015-08-27 10:41:19,083] [main] [org.jets3t.service.StorageService]: Setting recommended DNS caching defaults [DEBUG] [2015-08-27 10:41:19,083] [main] [org.jets3t.service.StorageService]: Using DNS caching system settings: networkaddress.cache.ttl=300, networkaddress.cache.negative.ttl=1 [DEBUG] [2015-08-27 10:41:19,090] [main] [org.jets3t.service.Jets3tProperties]: http.connection-manager.factory-class-name=org.jets3t.service.utils.RestUtils$ConnManagerFactory [DEBUG] [2015-08-27 10:41:19,090] [main] [org.jets3t.service.Jets3tProperties]: httpclient.connection-timeout-ms=60000 [DEBUG] [2015-08-27 10:41:19,090] [main] [org.jets3t.service.Jets3tProperties]: httpclient.socket-timeout-ms=60000 [DEBUG] [2015-08-27 10:41:19,090] [main] [org.jets3t.service.Jets3tProperties]: httpclient.stale-checking-enabled=true [DEBUG] [2015-08-27 10:41:19,090] [main] [org.jets3t.service.Jets3tProperties]: httpclient.useragent=null [DEBUG] [2015-08-27 10:41:19,093] [main] [org.jets3t.service.utils.RestUtils]: Setting user agent string: JetS3t/0.9.4 (Linux/3.5.0-61-generic; amd64; en; JVM 1.7.0_79) [DEBUG] [2015-08-27 10:41:19,093] [main] [org.jets3t.service.Jets3tProperties]: http.protocol.expect-continue=true [DEBUG] [2015-08-27 10:41:19,093] [main] [org.jets3t.service.Jets3tProperties]: httpclient.connection-manager-timeout=0 [DEBUG] [2015-08-27 10:41:19,102] [main] [org.jets3t.service.Jets3tProperties]: httpclient.retry-max=5 [DEBUG] [2015-08-27 10:41:19,102] [main] [org.jets3t.service.Jets3tProperties]: httpclient.proxy-autodetect=true [DEBUG] [2015-08-27 10:41:19,102] [main] [org.jets3t.service.Jets3tProperties]: s3service.s3-endpoint=s3.amazonaws.com [DEBUG] [2015-08-27 10:41:19,104] [main] [org.apache.commons.httpclient.contrib.proxy.PluginProxyUtil]: About to attempt auto proxy detection under Java version:1.7.0_79-b14 [DEBUG] [2015-08-27 10:41:19,104] [main] [org.apache.commons.httpclient.contrib.proxy.PluginProxyUtil]: Sun Plugin reported java version not 1.3.X, 1.4.X, 1.5.X or 1.6.X - trying failover detection... [DEBUG] [2015-08-27 10:41:19,104] [main] [org.apache.commons.httpclient.contrib.proxy.PluginProxyUtil]: Using failover proxy detection... [DEBUG] [2015-08-27 10:41:19,104] [main] [org.apache.commons.httpclient.contrib.proxy.PluginProxyUtil]: Plugin Proxy Config List Property:null [DEBUG] [2015-08-27 10:41:19,104] [main] [org.apache.commons.httpclient.contrib.proxy.PluginProxyUtil]: No configured plugin proxy list [DEBUG] [2015-08-27 10:41:19,104] [main] [org.jets3t.service.Jets3tProperties]: s3service.default-storage-class=null [DEBUG] [2015-08-27 10:41:19,105] [main] [org.jets3t.service.Jets3tProperties]: s3service.server-side-encryption=null [DEBUG] [2015-08-27 10:41:19,105] [main] [org.jets3t.service.Jets3tProperties]: devpay.user-token=null [DEBUG] [2015-08-27 10:41:19,105] [main] [org.jets3t.service.Jets3tProperties]: devpay.product-token=null [DEBUG] [2015-08-27 10:41:19,105] [main] [org.jets3t.service.Jets3tProperties]: httpclient.requester-pays-buckets-enabled=false [DEBUG] [2015-08-27 10:41:19,116] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Retrieving All information for bucket some_bucket and object /some_folder [DEBUG] [2015-08-27 10:41:19,116] [main] [org.jets3t.service.Jets3tProperties]: s3service.disable-dns-buckets=false [DEBUG] [2015-08-27 10:41:19,116] [main] [org.jets3t.service.Jets3tProperties]: s3service.s3-endpoint=s3.amazonaws.com [DEBUG] [2015-08-27 10:41:19,117] [main] [org.jets3t.service.Jets3tProperties]: s3service.s3-endpoint-virtual-path= [DEBUG] [2015-08-27 10:41:19,117] [main] [org.jets3t.service.Jets3tProperties]: s3service.s3-endpoint-https-port=443 [DEBUG] [2015-08-27 10:41:19,117] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: S3 URL: https://some_bucket.s3.amazonaws.com:443//some_folder [DEBUG] [2015-08-27 10:41:19,121] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Performing GET request for 'https://some_bucket.s3.amazonaws.com:443//some_folder', expecting response codes: [200] [DEBUG] [2015-08-27 10:41:19,123] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Headers: [Date: Thu, 27 Aug 2015 14:41:19 GMT] [DEBUG] [2015-08-27 10:41:19,123] [main] [org.jets3t.service.Jets3tProperties]: httpclient.retry-max=5 [DEBUG] [2015-08-27 10:41:19,123] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Adding authorization for Access Key 'ACCESS_KEY_REMOVED'. [DEBUG] [2015-08-27 10:41:19,123] [main] [org.jets3t.service.Jets3tProperties]: s3service.s3-endpoint=s3.amazonaws.com [DEBUG] [2015-08-27 10:41:19,123] [main] [org.jets3t.service.Jets3tProperties]: storage-service.request-signature-version=AWS2 [DEBUG] [2015-08-27 10:41:19,123] [main] [org.jets3t.service.Jets3tProperties]: s3service.s3-endpoint=s3.amazonaws.com [DEBUG] [2015-08-27 10:41:19,124] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Canonical string ('|' is a newline): GET|||Thu, 27 Aug 2015 14:41:19 GMT|/some_bucket//some_folder [DEBUG] [2015-08-27 10:41:19,270] [main] [org.jets3t.service.Jets3tProperties]: httpclient.max-connections=20 [DEBUG] [2015-08-27 10:41:19,270] [main] [org.jets3t.service.Jets3tProperties]: httpclient.max-connections-per-host=0 [DEBUG] [2015-08-27 10:41:19,271] [main] [org.jets3t.service.Jets3tProperties]: httpclient.connection.ttl=-1 [DEBUG] [2015-08-27 10:41:19,303] [main] [org.jets3t.service.utils.RestUtils$ThreadSafeConnManager]: Get connection: {s}->https://some_bucket.s3.amazonaws.com:443, timeout = 0 [DEBUG] [2015-08-27 10:41:19,303] [main] [org.apache.http.impl.conn.tsccm.ConnPoolByRoute]: [{s}->https://some_bucket.s3.amazonaws.com:443] total kept alive: 0, total issued: 0, total allocated: 0 out of 20 [DEBUG] [2015-08-27 10:41:19,303] [main] [org.apache.http.impl.conn.tsccm.ConnPoolByRoute]: No free connections [{s}->https://some_bucket.s3.amazonaws.com:443][null] [DEBUG] [2015-08-27 10:41:19,303] [main] [org.apache.http.impl.conn.tsccm.ConnPoolByRoute]: Available capacity: 20 out of 20 [{s}->https://some_bucket.s3.amazonaws.com:443][null] [DEBUG] [2015-08-27 10:41:19,303] [main] [org.apache.http.impl.conn.tsccm.ConnPoolByRoute]: Creating new connection [{s}->https://some_bucket.s3.amazonaws.com:443] [DEBUG] [2015-08-27 10:41:19,380] [main] [org.apache.http.impl.conn.DefaultClientConnectionOperator]: Connecting to some_bucket.s3.amazonaws.com:443 [DEBUG] [2015-08-27 10:41:19,539] [main] [org.apache.http.client.protocol.RequestAddCookies]: CookieSpec selected: default [DEBUG] [2015-08-27 10:41:19,539] [main] [org.apache.http.client.protocol.RequestAuthCache]: Auth cache not set in the context [DEBUG] [2015-08-27 10:41:19,539] [main] [org.apache.http.client.protocol.RequestProxyAuthentication]: Proxy auth state: UNCHALLENGED [DEBUG] [2015-08-27 10:41:19,540] [main] [org.apache.http.impl.client.DefaultHttpClient]: Attempt 1 to execute request [DEBUG] [2015-08-27 10:41:19,540] [main] [org.apache.http.impl.conn.DefaultClientConnection]: Sending request: GET /some_folder HTTP/1.1 [DEBUG] [2015-08-27 10:41:19,540] [main] [org.apache.http.wire]: >> "GET /some_folder HTTP/1.1[\r][\n]" [DEBUG] [2015-08-27 10:41:19,540] [main] [org.apache.http.wire]: >> "Date: Thu, 27 Aug 2015 14:41:19 GMT[\r][\n]" [DEBUG] [2015-08-27 10:41:19,540] [main] [org.apache.http.wire]: >> "Authorization: AWS ACCESS_KEY_REMOVED:ACCESS_SECRET_REMOVED[\r][\n]" [DEBUG] [2015-08-27 10:41:19,540] [main] [org.apache.http.wire]: >> "Host: some_bucket.s3.amazonaws.com:443[\r][\n]" [DEBUG] [2015-08-27 10:41:19,540] [main] [org.apache.http.wire]: >> "Connection: Keep-Alive[\r][\n]" [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.wire]: >> "User-Agent: JetS3t/0.9.4 (Linux/3.5.0-61-generic; amd64; en; JVM 1.7.0_79)[\r][\n]" [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.wire]: >> "[\r][\n]" [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.headers]: >> GET /some_folder HTTP/1.1 [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.headers]: >> Date: Thu, 27 Aug 2015 14:41:19 GMT [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.headers]: >> Authorization: AWS ACCESS_KEY_REMOVED:ACCESS_SECRET_REMOVED [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.headers]: >> Host: some_bucket.s3.amazonaws.com:443 [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.headers]: >> Connection: Keep-Alive [DEBUG] [2015-08-27 10:41:19,541] [main] [org.apache.http.headers]: >> User-Agent: JetS3t/0.9.4 (Linux/3.5.0-61-generic; amd64; en; JVM 1.7.0_79) [DEBUG] [2015-08-27 10:41:19,553] [main] [org.apache.http.wire]: << "HTTP/1.1 403 Forbidden[\r][\n]" [DEBUG] [2015-08-27 10:41:19,554] [main] [org.apache.http.wire]: << "x-amz-request-id: REQUEST_ID_REMOVED[\r][\n]" [DEBUG] [2015-08-27 10:41:19,554] [main] [org.apache.http.wire]: << "x-amz-id-2: HEADER_REMOVED[\r][\n]" [DEBUG] [2015-08-27 10:41:19,554] [main] [org.apache.http.wire]: << "Content-Type: application/xml[\r][\n]" [DEBUG] [2015-08-27 10:41:19,554] [main] [org.apache.http.wire]: << "Transfer-Encoding: chunked[\r][\n]" [DEBUG] [2015-08-27 10:41:19,554] [main] [org.apache.http.wire]: << "Date: Thu, 27 Aug 2015 14:41:26 GMT[\r][\n]" [DEBUG] [2015-08-27 10:41:19,554] [main] [org.apache.http.wire]: << "Server: AmazonS3[\r][\n]" [DEBUG] [2015-08-27 10:41:19,554] [main] [org.apache.http.wire]: << "[\r][\n]" [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.impl.conn.DefaultClientConnection]: Receiving response: HTTP/1.1 403 Forbidden [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.headers]: << HTTP/1.1 403 Forbidden [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.headers]: << x-amz-request-id: REQUEST_ID_REMOVED [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.headers]: << x-amz-id-2: HEADER_REMOVED [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.headers]: << Content-Type: application/xml [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.headers]: << Transfer-Encoding: chunked [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.headers]: << Date: Thu, 27 Aug 2015 14:41:26 GMT [DEBUG] [2015-08-27 10:41:19,555] [main] [org.apache.http.headers]: << Server: AmazonS3 [DEBUG] [2015-08-27 10:41:19,558] [main] [org.apache.http.impl.client.DefaultHttpClient]: Connection can be kept alive indefinitely [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Response for 'GET'. Content-Type: application/xml, Headers: [x-amz-request-id: REQUEST_ID_REMOVED, x-amz-id-2: HEADER_REMOVED, Content-Type: application/xml, Transfer-Encoding: chunked, Date: Thu, 27 Aug 2015 14:41:26 GMT, Server: AmazonS3] [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Response entity: org.apache.http.conn.BasicManagedEntity@1d8cdb5c [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Entity length: -1 [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Received response code 403; matches one of expected set ([200])? false [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Error response xml: true [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Error response entity: org.apache.http.conn.BasicManagedEntity@1d8cdb5c [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Error response entity length: -1 [DEBUG] [2015-08-27 10:41:19,565] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Response '//some_folder' - Received error response with XML message [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "327[\r][\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "[\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "SignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.ACCESS_KEY_REMOVEDGET[\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "[\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "[\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "Thu, 27 Aug 2015 14:41:19 GMT[\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "/some_bucket/some_folderACCESS_SECRET_REMOVEDSTRING_TO_SIGN_BYTES_REMOVEDREQUEST_ID_REMOVEDHEADER_REMOVED" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "[\r][\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "0[\r][\n]" [DEBUG] [2015-08-27 10:41:19,566] [main] [org.apache.http.wire]: << "[\r][\n]" [DEBUG] [2015-08-27 10:41:19,567] [main] [org.jets3t.service.utils.RestUtils$ThreadSafeConnManager]: Released connection is reusable. [DEBUG] [2015-08-27 10:41:19,567] [main] [org.apache.http.impl.conn.tsccm.ConnPoolByRoute]: Releasing connection [{s}->https://some_bucket.s3.amazonaws.com:443][null] [DEBUG] [2015-08-27 10:41:19,567] [main] [org.apache.http.impl.conn.tsccm.ConnPoolByRoute]: Pooling connection [{s}->https://some_bucket.s3.amazonaws.com:443][null]; keep alive indefinitely [DEBUG] [2015-08-27 10:41:19,567] [main] [org.apache.http.impl.conn.tsccm.ConnPoolByRoute]: Notifying no-one, there are no waiting threads [DEBUG] [2015-08-27 10:41:19,567] [main] [org.jets3t.service.impl.rest.httpclient.HttpMethodReleaseInputStream]: Released HttpMethod as its response data stream is fully consumed [TRACE] [2015-08-27 10:41:19,571] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Rethrowing as a ServiceException error in performRequest: org.jets3t.service.ServiceException: Service Error Message. -- ResponseCode: 403, ResponseStatus: null, XML Error Message: SignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.ACCESS_KEY_REMOVEDGETThu, 27 Aug 2015 14:41:19 GMT/some_bucket/some_folderACCESS_SECRET_REMOVEDSTRING_TO_SIGN_BYTES_REMOVEDREQUEST_ID_REMOVEDHEADER_REMOVED org.jets3t.service.ServiceException: Service Error Message. -- ResponseCode: 403, ResponseStatus: null, XML Error Message: SignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.ACCESS_KEY_REMOVEDGETThu, 27 Aug 2015 14:41:19 GMT/some_bucket/some_folderACCESS_SECRET_REMOVEDSTRING_TO_SIGN_BYTES_REMOVEDREQUEST_ID_REMOVEDHEADER_REMOVED at org.jets3t.service.impl.rest.httpclient.RestStorageService.performRequest(RestStorageService.java:409) at org.jets3t.service.impl.rest.httpclient.RestStorageService.performRequest(RestStorageService.java:279) at org.jets3t.service.impl.rest.httpclient.RestStorageService.performRestGet(RestStorageService.java:1104) at org.jets3t.service.impl.rest.httpclient.RestStorageService.performRestGet(RestStorageService.java:1076) at org.jets3t.service.impl.rest.httpclient.RestStorageService.getObjectImpl(RestStorageService.java:2267) at org.jets3t.service.impl.rest.httpclient.RestStorageService.getObjectImpl(RestStorageService.java:2204) at org.jets3t.service.StorageService.getObject(StorageService.java:1167) at org.jets3t.service.S3Service.getObject(S3Service.java:2674) at org.jets3t.service.S3Service.getObject(S3Service.java:89) at org.jets3t.service.StorageService.getObject(StorageService.java:552) at org.jets3t.service.S3Service.getObject(S3Service.java:1468) at org.apache.hadoop.fs.s3.Jets3tFileSystemStore.get(Jets3tFileSystemStore.java:163) at org.apache.hadoop.fs.s3.Jets3tFileSystemStore.retrieveINode(Jets3tFileSystemStore.java:221) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy4.retrieveINode(Unknown Source) at org.apache.hadoop.fs.s3.S3FileSystem.listStatus(S3FileSystem.java:193) at org.apache.hadoop.fs.Globber.listStatus(Globber.java:69) at org.apache.hadoop.fs.Globber.glob(Globber.java:217) at org.apache.hadoop.fs.FileSystem.globStatus(FileSystem.java:1625) at .run(MAIN.java:99) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at .main(MAIN.java:39) [DEBUG] [2015-08-27 10:41:19,572] [main] [org.jets3t.service.impl.rest.httpclient.RestStorageService]: Releasing HttpClient connection after error: Service Error Message.