Mocking Tornado AsyncHTTPClient requests/responses

Question

I have a Tornado web application that sends requests to external services and can't seem to be able to mock the responses of those requests.

I've tried tornado-http-mock , and mock libs but with no luck.

The app starting point app.py :

import tornado.ioloop
import tornado.web

from handlers import HealthCheckHandler, MainHandler, LoginHandler, PaymentHandler, UserDetailsHandler
from settings import PORT, tornado_settings


def make_app(settings):
    handlers = [
        ('/static/(.*)', tornado.web.StaticFileHandler, {'path': './public'}),
        ('/', MainHandler),
        ('/health', HealthCheckHandler),
        ('/login', LoginHandler),
        ('/user', UserDetailsHandler),
        ('/payment', PaymentHandler),
    ]
    return tornado.web.Application(handlers=handlers, **settings)


if __name__ == '__main__':
    app = make_app(tornado_settings)
    app.listen(PORT)
    tornado.ioloop.IOLoop.current().start()

I'm trying to test the login functionality (it's an OAuth2 server) which redirects the user when there is no code GET param passed (if the user is not logged in yet), or it tries to exchange the code with the access token. Here's the login handler.

import base64
import urllib.parse
import json
import traceback
import tornado.web
import tornado.httpclient

from .base import BaseHandler
from settings import OID_AUTH_API, OID_REDIRECT_URI, OID_CLIENT_ID, OID_CLIENT_PASSWORD
from lib import logger


class LoginHandler(BaseHandler):
    _redirect_uri = urllib.parse.quote(OID_REDIRECT_URI, safe='')
    _scope = 'openid+profile+email'
    _response_type = 'code'
    _http_client = tornado.httpclient.AsyncHTTPClient()

    async def get(self):
        try:
            code = self.get_argument('code', None)

            if (code is None):
                self.redirect('%s/authorization?client_id=%s&scope=%s&response_type=%s&redirect_uri=%s' % (
                    OID_AUTH_API, OID_CLIENT_ID, self._scope, self._response_type, self._redirect_uri), self.request.uri)
                return

            # exchange the authorization code with the access token
            grant_type = 'authorization_code'
            redirect_uri = self._redirect_uri
            authorization_header = '%s:%s' % (
                OID_CLIENT_ID, OID_CLIENT_PASSWORD)
            authorization_header_encoded = base64.b64encode(
                authorization_header.encode('UTF-8')).decode('UTF-8')
            url = '%s/token?grant_type=%s&code=%s&redirect_uri=%s' % (
                OID_AUTH_API, grant_type, code, redirect_uri)
            token_exchange_response = await self._http_client.fetch(
                url,
                method='POST',
                headers={
                    'Content-Type': 'application/x-www-form-urlencoded',
                    'Authorization': 'Basic %s' % authorization_header_encoded,
                    'Accept': 'application/json'
                },
                body='')

            token_exchange_response_body_dict = json.loads(
                token_exchange_response.body)

            access_token = token_exchange_response_body_dict.get('access_token')

            self.send_response({
                'access_token': access_token
            })
        except Exception as error:
            logger.log_error_with_traceback(error)
            self.send_response({
                'success': False,
                'message': 'Internal server error. Please try again later.'
            }, 500)

I have two questions: 1. How to test the redirect functionality in case the authorization code was not presented? 2. How to mock requests to the OAuth2 server in this case?

I tried with tornado-http-mock , but I'm getting errors.

import app
import json

from tornado.httpclient import HTTPClient, HTTPResponse, HTTPRequest, HTTPError
from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, AsyncHTTPClient

from tornado_mock.httpclient import get_response_stub, patch_http_client, set_stub

from .base import TestHandlerBase
from settings import OID_AUTH_API


class TestLoginHandler(AsyncHTTPTestCase):
    def get_app(self):
        test_app = app.make_app({})
        self.app_http_client = test_app.http_client = AsyncHTTPClient(force_instance=True)
        return test_app

    def test_token_code_exchange(self):        
        patch_http_client(self.app_http_client)

        set_stub(self.app_http_client, '%s/token' % (OID_AUTH_API), request_method='POST', response_body='oauth_server_token')

        response = self.fetch('/login?code=123')

        self.assertEqual(response.code, 200)
        print(response.body)

I'm getting the following error which indicates that the POST method is not supported (it seems that the request is actually being sent to the server and not being mocked).

Login Error code: 405 | Response body: 
[E 190626 13:29:33 web:2246] 500 GET /login?code=123 (127.0.0.1) 238.10ms
======================================================================
FAIL: test_token_code_exchange (tests.handlers.login.TestLoginHandler)
----------------------------------------------------------------------

Traceback (most recent call last):
  File "/Users/.../venv/lib/python3.7/site-packages/tornado/testing.py", line 98, in __call__
    result = self.orig_method(*args, **kwargs)
  File "/Users/.../tests/handlers/login.py", line 60, in test_token_code_exchange
    self.assertEqual(response.code, 200)
AssertionError: 500 != 200

I'm expecting to get the stubbed response, but apparently, I'm not getting that. What am I missing here? Is there any other solution for this?

Answer 1

You can use mock.patch and gen.coroutine to mock external request in tornado. You can try something like this:

Extract the external request to a method like...

async def new_fetch(self, url, authorization_header_encoded):
    return await self._http_client.fetch(
        url,
        method='POST',
        headers={
            'Content-Type': 'application/x-www-form-urlencoded',
            'Authorization': 'Basic %s' % authorization_header_encoded,
            'Accept': 'application/json'
        },
        body='')

Change your LoginHandler to call this new method:

token_exchange_response = await new_fetch(url, authorization_header_encoded)

In your TestLoginHandler create a new method to mock a http response and decorate this method with the gen.coroutine decorator and use the mock.patch decorator do mock the external request method on your test method:

import app
import json

from tornado.httpclient import HTTPClient, HTTPResponse, HTTPRequest, HTTPError
from tornado.testing import AsyncHTTPTestCase, AsyncTestCase, AsyncHTTPClient
# from tornado_mock.httpclient import get_response_stub, patch_http_client, 
from tornado import gen 

from .base import TestHandlerBase
from settings import OID_AUTH_API


class TestLoginHandler(AsyncHTTPTestCase):
    @gen.coroutine
    def mock_fetch(self, url, authorization_header_encoded)
        request = HTTPRequest(
            headers=authorization_header_encoded,
            method='POST',
            body='',
            url=url)
        resp = HTTPResponse(request, HTTPStatus.OK, buffer=json.dumps({}))
        resp._body = json.dumps({"YOUR_RESPONSE_BODY":"YOUR_RESPONSE_BODY"})
        return resp

    def get_app(self):
        test_app = app.make_app({})
        # self.app_http_client = test_app.http_client = AsyncHTTPClient(force_instance=True)
        return test_app

    @mock.patch("full.path.to.LoginHandler.new_fetch")
    def test_token_code_exchange(self, mocked_method):
        mocked_method.return_value = self.mock_fetch('optional_url', 'optional_header') 

        # patch_http_client(self.app_http_client)

        # set_stub(self.app_http_client, '%s/token' % (OID_AUTH_API), request_method='POST', response_body='oauth_server_token')

        response = self.fetch('/login?code=123')

        self.assertEqual(response.code, 200)
        print(response.body)

I didn't test this code, I write just to pass you and idea to make this mock, so maybe you will need to ajust a few things in this code