AWS S3 - 发送签名请求失败

Question

---

Clarifications:

Using the header method works for both regular (AKA permanent) credentials and temporary ones. Using the query string (AKS stick all headers as part of the uri), works with the permanent credentials but fails for the temporary one with the following error:

The request signature we calculated does not match the signature you provided. Check your key and signing method.

It's not related to any clock difference, and I know that the temporary credentials works with this S3 bucket for a fact.

the format of the temporary credentials is as fallow

[default]
aws_access_key_id = A********
aws_secret_access_key = U*******
aws_session_token = F******

End of clarifications

---

I'm writing a code snippet that needs to handle a get requests from a private S3 bucket. I have a working code that uses headers for both permanent and temporary credentials

my goal now is to achieve the same using query string but I'm unable to get it to work for temporary credentials, it's working just fine with the permanent one, so I'm pretty sure that the problem relays in the canonical_querystring somewhere, but nothing I've tried seems to be working

I can't use boto3 because it needs to be independent from any external package (the requests package is used only for debugging it won't be a part of the final code) if anyone can tell me what am I doing wrong it would be highly appreciated

hers is my attempt using query string

import datetime
import hashlib
import hmac
import re

try:
    import httplib
except ImportError:
    import http.client as httplib
import requests
import urllib


def get_region(url, host):
    conn = httplib.HTTPConnection(url)
    headers = {'Host': host}
    conn.request('HEAD', '/', headers=headers)
    res = conn.getresponse()
    status = res.status
    if 400 <= status:
        return None
    return res.getheader('x-amz-bucket-region')


def _sign(key, msg):
    return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()


def _get_signature_key(key, date_stamp, region_name, service_name):
    k_date = _sign(('AWS4' + key).encode('utf-8'), date_stamp)
    k_region = _sign(k_date, region_name)
    k_service = _sign(k_region, service_name)
    k_signing = _sign(k_service, 'aws4_request')
    return k_signing


def get_sign_headers(access_key, secret_key, url, session_token=None, method='GET', request_parameters=''):
    regex = r"^(https://)(([a-zA-z0-9\-]+)\.)((\w+.*)\.amazonaws\.com)([^:^/]*)?(.*)$"
    matches = re.match(regex, url, re.MULTILINE)
    service = 's3'
    host = matches.group(2) + matches.group(4)
    canonical_uri = matches.group(7)
    region = get_region('s3.us-east-2.amazonaws.com', host)
    if matches.group(3) == 's3':
        if not region:
            region = matches.group(5)
        host = service + '.' + region + '.amazonaws.com'
    endpoint = 'https://' + host + matches.group(7)

    t = datetime.datetime.utcnow()
    amz_date = t.strftime('%Y%m%dT%H%M%SZ')  # Format date as YYYYMMDD'T'HHMMSS'Z'
    datestamp = t.strftime('%Y%m%d')  # Date w/o time, used in credential scope
    canonical_headers = 'host:' + host + '\n'
    signed_headers = 'host'
    if session_token:
        canonical_headers += 'x-amz-security-token:' + session_token + '\n'
        signed_headers += ';x-amz-security-token'
    algorithm = 'AWS4-HMAC-SHA256'
    credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
    canonical_querystring = 'X-Amz-Algorithm=AWS4-HMAC-SHA256'
    try:
        credential = urllib.quote_plus(access_key + '/' + credential_scope)
    except AttributeError:
        credential = urllib.parse.quote_plus(access_key + '/' + credential_scope)
    canonical_querystring += '&X-Amz-Credential=' + credential
    canonical_querystring += '&X-Amz-Date=' + amz_date
    canonical_querystring += '&X-Amz-Expires=30'
    canonical_querystring += '&X-Amz-SignedHeaders=' + signed_headers
    payload_hash = 'UNSIGNED-PAYLOAD'
    canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + \
                                 '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
    string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(
        canonical_request.encode('utf-8')).hexdigest()
    signing_key = _get_signature_key(secret_key, datestamp, region, service)
    signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
    if session_token:
        try:
            session_token = urllib.quote_plus(session_token)
        except AttributeError:
            session_token = urllib.parse.quote_plus(session_token)
        canonical_querystring += '&X-Amz-Security-Token={TOKEN}'.format(TOKEN=session_token)
    canonical_querystring += '&X-Amz-Signature=' + signature
    request_url = endpoint + "?" + canonical_querystring

    print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
    print('Request URL = ' + request_url)
    r = requests.get(request_url)

    print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
    print('Response code: %d\n' % r.status_code)
    print(r.text)

and this is the code that uses headers approach (that works for both):

import datetime
import hashlib
import hmac
import re
import requests
try:
    import httplib
except ImportError:
    import http.client as httplib


def get_region(url, host):
    conn = httplib.HTTPConnection(url)
    headers = {'Host': host}
    conn.request('HEAD', '/', headers=headers)
    res = conn.getresponse()
    status = res.status
    if 400 <= status:
        return None
    return res.getheader('x-amz-bucket-region')


def _sign(key, msg):
    return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()


def _get_signature_key(key, date_stamp, region_name, service_name):
    k_date = _sign(('AWS4' + key).encode('utf-8'), date_stamp)
    k_region = _sign(k_date, region_name)
    k_service = _sign(k_region, service_name)
    k_signing = _sign(k_service, 'aws4_request')
    return k_signing


def get_sign_headers(access_key, secret_key, url, session_token=None, method='GET', request_parameters=''):
    regex = r"^(https://)(([a-zA-z0-9\-]+)\.)((\w+.*)\.amazonaws\.com)([^:^/]*)?(.*)$"
    matches = re.match(regex, url, re.MULTILINE)
    service = 's3'
    host = matches.group(2) + matches.group(4)
    canonical_uri = matches.group(7)
    region = get_region('s3.us-east-2.amazonaws.com', host)
    if matches.group(3) == 's3':
        if not region:
            region = matches.group(5)
        host = service + '.' + region + '.amazonaws.com'
    endpoint = 'https://' + host
    t = datetime.datetime.utcnow()
    amz_date = t.strftime('%Y%m%dT%H%M%SZ')
    datestamp = t.strftime('%Y%m%d')  # Date w/o time, used in credential scope

    canonical_querystring = request_parameters
    canonical_headers = 'host:' + host + '\n' + 'x-amz-content-sha256:UNSIGNED-PAYLOAD' + \
                        '\n' + 'x-amz-date:' + amz_date + '\n'
    signed_headers = 'host;x-amz-content-sha256;x-amz-date'
    if session_token:
        signed_headers += ';x-amz-security-token'
        canonical_headers += 'x-amz-security-token:' + session_token + '\n'
    payload_hash = 'UNSIGNED-PAYLOAD'
    canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + \
                                 '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash

    algorithm = 'AWS4-HMAC-SHA256'
    credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
    string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(
        canonical_request.encode('utf-8')).hexdigest()
    signing_key = _get_signature_key(secret_key, datestamp, region, service)
    signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest()
    authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + \
                                       'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature

    headers = {'x-amz-date': amz_date, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD',
               'Authorization': authorization_header}
    if session_token:
        headers['x-amz-security-token'] = session_token
    request_url = endpoint + canonical_uri
​
    print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
    print('Request URL = ' + request_url)
    print('Headers = {}'.format(headers))
    r = requests.get(request_url, headers=headers)
​
    print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
    print('Response code: %d\n' % r.status_code)

Answer 1

have you validated that your server time is accurate. Even a few minutes difference causes signing failures.