Python Subprocess Module | Postgres pg_dump with password

Question

I have a database that I want to back up with my a python code.

I tried to base my code from the code in this discussion that uses the subprocess module and pg_dump . My problem now is that I have to manually type in the password to get the backup file. I read somewhere that doing a .pgpass but I do want to see if it is possible to do it within the subprocess module.

My code follows below:

from subprocess import Popen, PIPE
from pathlib import Path, PureWindowsPath

def backup():
    version = 11
    postgresDir = Path("C:/Program Files/PostgreSQL/{}/bin/".format(version))
    directory = PureWindowsPath(postgresDir)
    filename = 'myBackUp2'  # output filename here
    saveDir = Path("D:/Desktop/{}.tar".format(filename))  # output directory here
    file = PureWindowsPath(saveDir)

    host = 'localhost'
    user = 'postgres'
    port = '5434'
    dbname = 'database_name'  # database name here
    proc = Popen(['pg_dump', '-h', host, '-U', user, '-W', '-p', port,
                   '-F', 't', '-f', str(file), '-d', dbname],
                    cwd=directory, shell=True, stdin=PIPE)
    proc.wait()

backup()

The code above works and the backup is created is I type in the password. I tried to replace proc.wait() with the code below to remove the need of typing the password manually:

return proc.communicate('{}\n'.format(database_password))

But I would receive this error:

TypeError: a bytes-like object is required, not 'str'

Is this possible to do within subprocess? If so, how?

Answer 1

Use a password file.

On Microsoft Windows the file is named %APPDATA%\\postgresql\\pgpass.conf (where %APPDATA% refers to the Application Data subdirectory in the user's profile).

and the -w or --no-password command line option (instead of -W )

-w

--no-password

Never issue a password prompt. If the server requires password authentication and a password is not available by other means such as a .pgpass file, the connection attempt will fail. This option can be useful in batch jobs and scripts where no user is present to enter a password.

Answer 2

最简单的方法是使用PGPASSWORD 环境变量。

Answer 3

There is two classes:

1. First class needed for create dsn string. Then try to connect with dsn parameters. If cannot connect go to Second class.
2. Second class needed for create for create DataBase and restore all tables from file. You need to remake this strings:

for correctly open your DataBase dump_file

__folder_name = Path(__file__).parent.parent
__folder_name_data = os.path.join(__folder_name, 'data')
__file_to_open = os.path.join(__folder_name_data, 'bd.backup')

import os
import textwrap
from pathlib import Path
from subprocess import Popen, PIPE


class DataBaseAPI:
    __slots__ = ('__dsn', 'cur')

    def __init__(self):
        self.__dsn = self.__dsn_string()
        self.cur = self.__connection()

    @staticmethod
    def __dsn_string() -> dict:
        print(f'INPUT name of DataBase')
        name = input()
        print(f'INPUT password of DataBase')
        password = input()
        print(f'INPUT user_name of DataBase or press ENTER if user_name="postgres"')
        user_name = input()
        if len(user_name) == 0:
            user_name = 'postgres'
        print(f'INPUT host_name of DataBase or press ENTER if host_name="localhost"')
        host_name = input()
        if len(host_name) == 0:
            host_name = 'localhost'
        return {'dbname': name, 'user': user_name, 'password': password, 'host': host_name}

    def __connection(self):
        try:
            conn = psycopg2.connect(dbname=self.__dsn['dbname'], user=self.__dsn['user'],
                                    host=self.__dsn['host'], password=self.__dsn['password'], port=5432)
        except psycopg2.OperationalError:
            print(textwrap.fill(f'There is no existing DataBase. Creating new DataBase', 80,
                                   subsequent_indent='                   '))
            DataBaseCreator(self.__dsn)
            conn = psycopg2.connect(dbname=self.__dsn['dbname'], user=self.__dsn['user'],
                                    host=self.__dsn['host'], password=self.__dsn['password'], port=5432)
        finally:
            conn.autocommit = True
            cur = conn.cursor()
            print(f'DataBase connection complete')
            return cur



class DataBaseCreator:
    def __init__(self, dsn):
        self.__dsn = dsn
        self.__check_conf_file()
        self.__create_data_base()
        self.__restore_data_base()

    def __check_conf_file(self):
        __app_data = os.environ.copy()["APPDATA"]
        __postgres_path = Path(f'{__app_data}\postgresql')
        __pgpass_file = Path(f'{__postgres_path}\pgpass.conf')
        parameters = f'{self.__dsn["host"]}:{5432}:{self.__dsn["dbname"]}:' \
                     f'{self.__dsn["user"]}:{int(self.__dsn["password"])}\n'
        if not os.path.isdir(__postgres_path):
            os.makedirs(__postgres_path)
        if os.path.isfile(__pgpass_file):
            log.debug(f'File "pgpass.conf" already exists')
            with open(__pgpass_file, 'r+') as f:
                content = f.readlines()
                if parameters not in content:
                    # сервер: порт:база_данных: имя_пользователя:пароль
                    f.write(parameters)
                else:
                    log.info(f' {parameters} already in "pgpass.conf" file')
        else:
            log.debug(f'File "pgpass.conf" not exists')
            with open(__pgpass_file, 'x') as f:
                # сервер: порт:база_данных: имя_пользователя:пароль
                f.write(parameters)

    def __create_data_base(self):
        try:
            __conn = psycopg2.connect(dbname='postgres', user=self.__dsn['user'],
                                      host=self.__dsn['host'], password=self.__dsn['password'], port=5432)
        except Exception as _:
            log.exception(f'{_}')
        else:
            __conn.autocommit = True
            __cur = __conn.cursor()
            __query = f'CREATE DATABASE "{self.__dsn["dbname"]}"'
            __cur.execute(__query)
            log.info(f'{__query}')

    def __restore_data_base(self):
        __col = [x for x in self.__dsn.values()]
        __folder_name = Path(__file__).parent.parent
        __folder_name_data = os.path.join(__folder_name, 'data')
        __file_to_open = os.path.join(__folder_name_data, 'bd.backup')
        __cmd = f'pg_restore --host={__col[3]} --dbname={__col[0]} --username={__col[1]} ' \
                f'--verbose=True --no-password ' \
                f'{__file_to_open}'
        try:
            __proc = Popen(__cmd, stdout=PIPE, stderr=PIPE)
        except FileNotFoundError:
            log.info(f'FileNotFoundError: [WinError 2] Не удается найти указанный файл')
            log.info(textwrap.fill(f'You need to SET Windows $PATH for use "pg_restore" in cmd', 80,
                                   subsequent_indent='                   '))
        else:
            __stderr = __proc.communicate()[1].decode('utf-8', errors="ignore").strip()
            log.debug(textwrap.fill(f'{__stderr}', 80))

Answer 4

dbname一种选择是使用dbname参数

'pg_dump --dbname=postgresql://{}:{}@{}:{}/{}'.format(user, password, host, port, database_name)