堆栈内存溢出
  繁体   English   中英

Docker.network：从本地的 DataHub 或 OpenMetadata 连接到 Postgres DB

[英]Docker network: Connect to Postgres DB from DataHub or OpenMetadata locally

 原文  2022-12-04 14:05:46  6   1    postgresql/ docker/ docker-networking

问题描述

我尝试在本地运行 DataHub ( https://datahub.io/ ) 和/或 OpenMetaData ( https://open-metadata.org/ ) 进行测试。 两者都通过 docker-compose 文件使用。

对于我使用的 OpenMetaData：

version: "3.9"
volumes:
  ingestion-volume-dag-airflow:
  ingestion-volume-dags:
  ingestion-volume-tmp:
services:
  mysql:
    container_name: openmetadata_mysql
    image: openmetadata/db:0.13.0
    restart: always
    environment:
      MYSQL_ROOT_PASSWORD: password
    expose:
      - 3306
    volumes:
     - ./docker-volume/db-data:/var/lib/mysql
    networks:
      - app_net
    healthcheck:
      test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
      interval: 15s
      timeout: 10s
      retries: 10

  elasticsearch:
    container_name: openmetadata_elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
    environment:
      - discovery.type=single-node
      - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
    networks:
      - app_net
    ports:
      - "9200:9200"
      - "9300:9300"

  openmetadata-server:
    container_name: openmetadata_server
    restart: always
    image: openmetadata/server:0.13.0
    environment:
      # OpenMetadata Server Authentication Configuration
      AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
      AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
      AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
      AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
      AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
      AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
      AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
      AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
      AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
      CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
      AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/config/jwks]}
      AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
      AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
      AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
      AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
      AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
      # JWT Configuration
      RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
      RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
      JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
      JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
      # OpenMetadata Server Airflow Configuration
      AIRFLOW_HOST: ${AIRFLOW_HOST:-http://ingestion:8080}
      SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
      AIRFLOW_AUTH_PROVIDER: ${AIRFLOW_AUTH_PROVIDER:-no-auth}
      # OpenMetadata Airflow Azure SSO Configuration
      OM_AUTH_AIRFLOW_AZURE_CLIENT_SECRET: ${OM_AUTH_AIRFLOW_AZURE_CLIENT_SECRET:-""}
      OM_AUTH_AIRFLOW_AZURE_AUTHORITY_URL: ${OM_AUTH_AIRFLOW_AZURE_AUTHORITY_URL:-""}
      OM_AUTH_AIRFLOW_AZURE_SCOPES: ${OM_AUTH_AIRFLOW_AZURE_SCOPES:-[]}
      OM_AUTH_AIRFLOW_AZURE_CLIENT_ID: ${OM_AUTH_AIRFLOW_AZURE_CLIENT_ID:-""}
      # OpenMetadata Airflow Google SSO Configuration
      OM_AUTH_AIRFLOW_GOOGLE_SECRET_KEY_PATH: ${OM_AUTH_AIRFLOW_GOOGLE_SECRET_KEY_PATH:- ""}
      OM_AUTH_AIRFLOW_GOOGLE_AUDIENCE: ${OM_AUTH_AIRFLOW_GOOGLE_AUDIENCE:-"https://www.googleapis.com/oauth2/v4/token"}
      # OpenMetadata Airflow Okta SSO Configuration
      OM_AUTH_AIRFLOW_OKTA_CLIENT_ID: ${OM_AUTH_AIRFLOW_OKTA_CLIENT_ID:-""}
      OM_AUTH_AIRFLOW_OKTA_ORGANIZATION_URL: ${OM_AUTH_AIRFLOW_OKTA_ORGANIZATION_URL:-""}
      OM_AUTH_AIRFLOW_OKTA_PRIVATE_KEY: ${OM_AUTH_AIRFLOW_OKTA_PRIVATE_KEY:-""}
      OM_AUTH_AIRFLOW_OKTA_SA_EMAIL: ${OM_AUTH_AIRFLOW_OKTA_SA_EMAIL:-""}
      OM_AUTH_AIRFLOW_OKTA_SCOPES: ${OM_AUTH_AIRFLOW_OKTA_SCOPES:-[]}
      # OpenMetadata Airflow Auth0 SSO Configuration
      OM_AUTH_AIRFLOW_AUTH0_CLIENT_ID: ${OM_AUTH_AIRFLOW_AUTH0_CLIENT_ID:-""}
      OM_AUTH_AIRFLOW_AUTH0_CLIENT_SECRET: ${OM_AUTH_AIRFLOW_AUTH0_CLIENT_SECRET:-""}
      OM_AUTH_AIRFLOW_AUTH0_DOMAIN_URL: ${OM_AUTH_AIRFLOW_AUTH0_DOMAIN_URL:-""}
      # OpenMetadata Airflow Custom OIDC SSO Configuration
      OM_AUTH_AIRFLOW_CUSTOM_OIDC_CLIENT_ID: ${OM_AUTH_AIRFLOW_CUSTOM_OIDC_CLIENT_ID:-""}
      OM_AUTH_AIRFLOW_CUSTOM_OIDC_SECRET_KEY: ${OM_AUTH_AIRFLOW_CUSTOM_OIDC_SECRET_KEY:-""}
      OM_AUTH_AIRFLOW_CUSTOM_OIDC_TOKEN_ENDPOINT_URL: ${OM_AUTH_AIRFLOW_CUSTOM_OIDC_TOKEN_ENDPOINT_URL:-""}
      # OpenMetadata Airflow JWT Token Configuration
      OM_AUTH_JWT_TOKEN: ${OM_AUTH_JWT_TOKEN:-""}
      # Database configuration for MySQL
      DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
      DB_SCHEME: ${DB_SCHEME:-mysql}
      DB_USE_SSL: ${DB_USE_SSL:-false}
      DB_USER: ${DB_USER:-openmetadata_user}
      DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
      DB_HOST: ${DB_HOST:-mysql}
      DB_PORT: ${DB_PORT:-3306}
      OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
      # Airflow SSL Configurations
      AIRFLOW_VERIFY_SSL: ${AIRFLOW_VERIFY_SSL:-"no-ssl"}
      AIRFLOW_SSL_CERT_PATH: ${AIRFLOW_SSL_CERT_PATH:-""}
      # ElasticSearch Configurations
      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
      ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
      ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
      ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
      # Heap OPTS Configurations
      OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}

    expose:
      - 8585
      - 8586
    ports:
      - "8585:8585"
      - "8586:8586"
    depends_on:
      elasticsearch:
        condition: service_started
      mysql:
        condition: service_healthy
    networks:
      - app_net
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:8586/healthcheck" ]

  ingestion:
    container_name: openmetadata_ingestion
    image: openmetadata/ingestion:0.13.0
    depends_on:
      elasticsearch:
        condition: service_started
      mysql:
        condition: service_healthy
      openmetadata-server:
        condition: service_healthy
    environment:
      AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
      DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
      DB_PORT: ${AIRFLOW_DB_PORT:-3306}
      AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
      AIRFLOW_DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
      DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
      DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}
    entrypoint: /bin/bash
    command:
      - "/opt/airflow/ingestion_dependency.sh"
    expose:
      - 8080
    ports:
      - "8080:8080"
    networks:
      - app_net
    volumes:
      - ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
      - ingestion-volume-dags:/opt/airflow/dags
      - ingestion-volume-tmp:/tmp


networks:
  app_net:
    ipam:
      driver: default
      config:
        - subnet: "172.16.240.0/24"

对于 DataHub，我遵循了此处描述的步骤： https://datahubproject.io/docs/quickstart/

一切都在我的 Mac 上本地运行。

现在我的问题 - 我也有一个本地 postgres 启动并运行。 我可以通过 PGAdmin 访问数据库。

version: '3.8'
services:
  db:
    container_name: pg_container
    image: postgres
    restart: always
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin
      POSTGRES_DB: phd_test
    ports:
      - "5432:5432"
  pgadmin:
    container_name: pgadmin4_container
    image: dpage/pgadmin4
    restart: always
    environment:
      PGADMIN_DEFAULT_EMAIL: admin@admin.com
      PGADMIN_DEFAULT_PASSWORD: admin
    ports:
      - "5050:80"

在这两种情况下，DataHub 和 OpenMetaData 我都无法连接到这个本地 Postgres 数据库。 在这两种情况下，我都会收到错误，例如：

'sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "127.0.0.1", port 5432 failed: Connection refused\n'
       '\tIs the server running on that host and accepting TCP/IP connections?\n'
       '\n'
       '(Background on this error at: https://sqlalche.me/e/14/e3q8)\n',
       "2022-12-04 13:58:08.971818 [exec_id=9e2a2163-24e0-4afb-8d2d-cb1b3033bc91] INFO: Failed to execute 'datahub ingest'",

我还在 DataHub 和 OpenMetadata 中为 Postgres 尝试了不同的端点，比如

  • 127.0.0.1:5432
  • 0.0.0.0:5432
  • pg_container:5432（postgres 容器的名称 - 这在 PG Admin 中有效）
  • 172.25.0.2:5432（pg_container 的 IP 地址）

其他人是否在这些工具之一的启动和运行之间建立了本地连接？ 由于两种工具中的错误相似，我认为这可能是 docker.network 错误，因此其他容器（如 DataHub 或 OpenMetaData）无法看到 postgres 容器。

1 个解决方案

解决方案1
 0  2022-12-27 09:53:42

也许你想通了，但这里的问题是开放元数据在 docker 容器中运行，因此，localhost 它是开放元数据容器，如果你在本地运行 postgres，你需要访问主机的 localhost 而不是你自己的容器地址，因此你需要使用主机host.docker.internal

同样，如果您为 postgres 使用 docker 容器并且没有将端口暴露给 local.network，您需要找出该容器的主机，通常是 id，为此您可以运行docker.network ls来查看所有容器在 docker bridge.network 然后检查容器以查看 IP 地址和容器 ID 使用docker inspect .NETWORK ID>

暂无
暂无

声明:本站的技术帖子网页，遵循CC BY-SA 4.0协议，如果您需要转载，请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.

相关问题 从来宾游民游箱连接到主机上Docker中的PostgreSQL中的db 无法从pgadmin连接到postgres数据库（都在docker上运行）？ 无法连接到Postgres数据库-docker swarm 如何连接到 OpenMapTiles Docker Postgres DB 无法将Npgsql连接到在Docker上运行的Postgres DB 无法从 Docker 连接到 postgres 我们如何将Docker应用连接到本地运行的postgres？ 如何通过 docker-compose 网络连接到 postgres？ .net核心应用程序在docker-compose期间将不会连接到postgres db 无法从DataGrid连接到Postgres数据库
相关标签
 
粤ICP备18138465号  © 2020-2024 STACKOOM.COM