How to Scrape the AWS SSO Credentials via Selenium Python

Question

So there is a process that is used by my organization that I work for, whereby which we have to scrape the following from an AWS SSO Console:

AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_SESSION_TOKEN=

My first thought was to try this with Curl, but there is a redirect because it is integrated via SAML to our G-Suite Organization creating LDAP Connections. As a result, my only thought was to use selenium to log in, grab the output, and sign in. Unfortunately, I am having issues with getting past the payload options because of the Way OAuth Workflows occur.

My Overall Goal is to Automate the process of

1. Logging in
2. Clicking on the a Specific Action
3. Clicking on the SSO Console App
4. Copying the Access_Key, Secret_Access_Key, and Session_Token and then writing it out to file.

I can inject the username without an issue, but I cannot inject the Password using the ID, Name, and XPATH elements. Here is the python traceback:

 rbarrett@MacBook-Pro  ~/Projects/Mirantis/Train  python scrape_creds.py                       1 ↵  3201  13:06:17
The file does not exist
[<Response [301]>, <Response [302]>]
https://mirantis.awsapps.com/start/#/
Traceback (most recent call last):
  File "scrape_creds.py", line 99, in <module>
    password = browser.find_element_by_xpath("//*[@id='password']")
  File "/usr/local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 394, in find_element_by_xpath
    return self.find_element(by=By.XPATH, value=xpath)
  File "/usr/local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 976, in find_element
    return self.execute(Command.FIND_ELEMENT, {
  File "/usr/local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "/usr/local/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//*[@id='password']"}
  (Session info: chrome=86.0.4240.75)

Here is the code:

 rbarrett@MacBook-Pro  ~/Projects/Mirantis/Train  cat scrape_creds.py                            ✔  3206  13:25:30
#!/bin/python
# ===========================================================
# Created By: Richard Barrett
# Organization: Mirantis
# Department: Customer Success
# Purpose: AWS SSO Credential Scrape
# Date: 10/01/2020
# ===========================================================

import selenium
import shutil
import xlsxwriter
import os
import unittest
import requests
import subprocess
import getpass
import platform
import logging
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from datetime import date

decrypt = "gpg --output secrets_test.json --decrypt secrets.gpg"

if os.path.exists("secrets.gpg"):
      returned_value = subprocess.call(decrypt, shell=True)
else:
        print("The file does not exist")

import json
with open('secrets.json','r') as f:
      config = json.load(f)

r = requests.get('http://' + 'mirantis.awsapps.com/start#/')

print(r.history)
print(r.url)

# Definitions
# find_elements_by_name
# find_elements_by_xpath
# find_elements_by_link_text
# find_elements_by_partial_link_text
# find_elements_by_tag_name
# find_elements_by_class_name
# find_elements_by_css_selector

# System Variables
today = date.today()
date = today.strftime("%m/%d/%Y")
node = platform.node()
system = platform.system()
username = getpass.getuser()
version = platform.version()
current_directory = os.getcwd()

# URL Variables
login_url = 'https://mirantis.awsapps.com/start#/'

# Check for Version of Chrome

# Options
#options = webdriver.ChromeOptions()
#options.add_argument("download.default_directory=current_directory", "--headless")

# WebDriver Path for System
if platform.system() == ('Windows'):
    browser = webdriver.Chrome("C:\Program Files (x86)\Google\Chrome\chromedriver.exe")
elif platform.system() == ('Linux'):
    browser = webdriver.Chrome(executable_path='/home/rbarrett/Drivers/Google/Chrome/chromedriver')
elif platform.system() == ('Darwin'):
    browser = webdriver.Chrome(executable_path='/Users/rbarrett/Drivers/Google/Chrome/chromedriver')
else:
    print("Are you sure you have the Selenium Webdriver installed in the correct path?")

# Parent URL
browser.get("https://mirantis.awsapps.com/start#/")

time.sleep(10)

# Credentials NEEDS UNIT TEST
# EMAIL XPATH /html/body/div[1]/div[1]/div[2]/div/div[2]/div/div/div[2]/div/div[1]/div/form/span/section/div/div/div[1]/div/div[1]/div/div[1]/input
# EMAIL ID identifierId
username = browser.find_element_by_id("identifierId")
username.send_keys(config['user']['name'])
# Click Next
# Next Button ID XPATH //*[@id='identifierNext']/div/button/span
element = WebDriverWait(browser, 20).until(
        EC.element_to_be_clickable((By.ID, "identifierNext")))
element.click();

# Password Payload
# Password XPTAH //*[@id='password']
password = browser.find_element_by_xpath("//*[@id='password']")
password.send_keys(config['user']['password'])
# Click Next
# Next Button XPATH //*[@id="passwordNext"]/div/button/div[2]
#element = WebDriverWait(browser, 20).until(
#        EC.element_to_be_clickable((By.ID, "passwordNext")))
#element.click();
#time.sleep(15)

# Authentication submit.click()
# For XPATH = //portal-application
#element = WebDriverWait(browser, 20).until(
#        EC.element_to_be_clickable((By.XPATH, "//portal-application")))
#element.click();
#print("Opening AWS SSO for Train!")

The problem is that the password doesn't get inject like it should. Is this because of a Windows Switch? Perhaps there is a better way to scrape these credentials, because these credentials expire every 12 hours.

Answer 1

So I did find a way on how to handle this issue. The reason why I couldn't get past it was that I needed to wait for the specific elements to become available. You can accomplish this by using a wait or putting in a wait for the element to be clickable.

# Credentials NEEDS UNIT TEST
# EMAIL XPATH /html/body/div[1]/div[1]/div[2]/div/div[2]/div/div/div[2]/div/div[1]/div/form/span/section/div/div/div[1]/div/div[1]/div/div[1]/input
# EMAIL ID identifierId
# XPATH //*[@id='identifierId']
#username = browser.find_element_by_xpath("//*[@id='identifierId']")
#username = browser.find_element_by_id("identifierId")
#username.send_keys(config['user']['name'])
element = WebDriverWait(browser, 20).until(
        EC.element_to_be_clickable((By.XPATH, "//*[@id='identifierId']")))
element.send_keys(config['user']['name'])
print("Sending Username...")


# Click Next
# Next Button ID XPATH //*[@id='identifierNext']/div/button/span
element = WebDriverWait(browser, 20).until(
        EC.element_to_be_clickable((By.ID, "identifierNext")))
element.click();
print("Clicking Next Button...")
time.sleep(2)

# Password Payload
# Password XPTAH //*[@id='password']
# XPATH //*[@id='password']/div[1]/div/div[1]/input
#password = browser.find_element_by_xpath("//*[@id='password']/div[1]/div/div[1]/input")
#password.send_keys(config['user']['password'])
element = WebDriverWait(browser, 10).until(
        EC.element_to_be_clickable((By.XPATH, "//*[@id='password']/div[1]/div/div[1]/input")))
element.send_keys(config['user']['password'])

The main component is the following block

element = WebDriverWait(browser, 10).until(
        EC.element_to_be_clickable((By.XPATH, "//*[@id='password']/div[1]/div/div[1]/input")))
element.send_keys(config['user']['password'])

If you provide a time.sleep(2) it should also work as well with single lines:

# Load Username
username = browser.find_element_by_id("identifierId")
username.send_keys(config['user']['name'])
time.sleep(2)

# Click Next
# Next Button ID XPATH //*[@id='identifierNext']/div/button/span
element = WebDriverWait(browser, 20).until(
        EC.element_to_be_clickable((By.ID, "identifierNext")))
element.click();

# Load Password
password = browser.find_element_by_xpath("//[@id='password']/div[1]/div/div[1]/input")
password.send_keys(config['user']['password'])

# Click Next
# Next Button XPATH //*[@id='passwordNext']/div/button/div[2]
# Full XPATH /html/body/div[1]/div[1]/div[2]/div/div[2]/div/div/div[2]/div/div[1]/div/form/span/section/div/div/div[1]/div[1]/div/div/div/div/div[1]/div/div[1]/input
element = WebDriverWait(browser, 20).until(
        EC.element_to_be_clickable((By.XPATH, "//*[@id='passwordNext']/div/button/div[2]")))
element.click();