在 AWS Glue Python 脚本中从 Secrets Manager 检索 RDS 凭证

Question

我有一个 Glue 脚本，它试图读取我存储在 Secrets manager 中的 RDS 凭据。 但是脚本继续运行并且永远不会完成。 此外，运行此 Glue 脚本的 IAM 角色包含SecretsManagerReadWrite策略（AWS 托管）

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
from awsglue.dynamicframe import DynamicFrameCollection
from awsglue.dynamicframe import DynamicFrame
import boto3
import botocore
from botocore.errorfactory import ClientError
# import org.apache.spark.sql.functions.concat_ws
from pyspark.sql.types import *
from pyspark.sql.functions import udf
from datetime import date
today = date.today()
current_day = today.strftime("%Y%m%d")

def str_to_arr(my_list):
    str = ""
    for item in my_list:
        if item:
            str += item
    str = str.split(" ")
    return '{"' + ' '.join([elem for elem in str])  + '"}'

str_to_arr_udf = udf(str_to_arr,StringType())

def AddPartitionKeys(glueContext, dfc) -> DynamicFrameCollection:
    df = dfc.select(list(dfc.keys())[0]).toDF()
    df = glueContext.add_ingestion_time_columns(df, "day")
    glue_df = DynamicFrame.fromDF(df, glueContext, "transform_date")
    return(DynamicFrameCollection({"CustomTransform0": glue_df}, glueContext))

## @params: [JOB_NAME]
args = getResolvedOptions(sys.argv, ['JOB_NAME', 'days', 's3_bucket', 'rds_endpoint', 'region_name', 'secret_name'])

region_name = args['region_name']
session = boto3.session.Session()
client = session.client("secretsmanager", region_name=region_name)
get_secret_value_response = client.get_secret_value(SecretId=args['secret_name'])
secret = get_secret_value_response['SecretString']
secret = json.loads(secret)
db_username = secret.get('username')
db_password = secret.get('password')
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
print("Below are the creds")
# print("DB USERNAME IS " , db_username)
# print("DB PWD IS " , db_password)
job = Job(glueContext)

job.init(args['JOB_NAME'], args)

job.commit()

我在这里想念什么？

我对照这个博客检查了我的工作，但我无法成功完成这个脚本。

Answer 1

在 Mark 的建议下，我发现我必须为 Secrets Manager 创建一个 VPC 接口端点。 AWS 在此处概述了这些步骤，只需确保终端节点中的策略具有提及我想从 Secrets Manager 访问的资源的访问权限/ARN。