Python 3 XML xml.etree解析标准输出bash cli命令

Question

Greetings good Python people.

I am trying to build a python script that is doing two things. It will take an input.xml file with some commands (that the users can edit later and add more) and then create another output.xml file with the results of the commands and then start some other .py program that inserts the output via API to a web application. These commands are all BASH commands, like dmidecode, cat, hpconfg and similar. The Input.xml looks like this:

    <?xml version="1.0"?>
<platform>
  <vendor name="HP">
     <tests type = "Hardware" name="bios_versions">
        <command>dmidecode --type bios | grep -e Version -e "Release Date"</command>
     </tests>
     <tests type = "Hardware" name="iLo">
        <command>hponcfg -f /tmp/Get_ILO_Firmware_Version.xml | awk -F"\"" '/LICENSE_TYPE/ {print $2}'</command>
     </tests>
     <tests type = "Hardware" name="hard_disks">
        <command>hpssacli ctrl slot=0 ld all show detail | egrep "Status|Size"</command>
        <command>hpssacli ctrl slot=1 ld all show detail | egrep "Status|Size"</command>
        <command>hpssacli ctrl slot=2 ld all show detail | egrep "Status|Size"</command>
        <command>hpssacli ctrl slot=3 ld all show detail | egrep "Status|Size"</command>
     </tests>

  </vendor>
</platform>

I would need to iterate trough this input.xml (that will be later maybe bigger with more commands) and i will use the Paramiko ssh library to connect remotely to the servers and collect the stdout from the commands and parse the stdout to an output.xml file.

This is my older script to collect and print the stdout of the commands but I think I will need a subprocess module for this more complex task.

import atexit
import paramiko
import sys

if len(sys.argv) < 2:
    print("args missing")
    sys.exit(1)

hostname = sys.argv[1]
password = sys.argv[2]
#user = sys.argv[3]
username = "root"

#Check Vendor
client = paramiko.SSHClient()

client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname, port = 22, username=username, password=password)
stdin, stdout, stderr = client.exec_command("dmidecode | egrep Vendor | awk '{print $2}'")
vendor = stdout.read().decode('utf-8')
vendor = vendor.strip('\n') 
client.close()
#finish check vendor

class myssh:

    def __init__(self, hostname, username, password, port = 22):
        client = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        client.connect(hostname, port=port, username=username, password=password)
        atexit.register(client.close)
        self.client = client

    def __call__(self, command):
        stdin,stdout,stderr = self.client.exec_command(command)
        sshdata = stdout.readlines()
        for line in sshdata:
            #end = '' -  to  loose space after the loops new lines
            print(line,end = '')

remote = myssh(hostname,username,password)

#CHECK HARDWARE COMMANDS
if vendor == "Dell":
   print("1. HARDWARE VENDOR:"+vendor)
   print("2. CPU INFO")
   remote("cat /proc/cpuinfo | grep -e 'model name' -e 'physical id' | cut -c 1- | sort | uniq -c")
   remote("cat /etc/release | grep PLATFORM")
   print("3. PHYSICAL DISK SMART PREDICABLE STATUS: NO is Ok!")
   remote("omreport storage pdisk controller=0 | egrep '^ID|Status|State|Failure|Capacity' | grep -v Power | grep -v 'Status' | grep 'Failure Predicted'")
   print("4. BATERRY STATUS:")
   remote("omreport storage battery | grep H740 -A 4  | grep -oh 'Ok'")
   remote("omreport storage battery | grep H740 -A 4  | grep -oh 'Ready'")
   print("5. Bios Version:")
   remote("omreport chassis bios | grep 'Version' | awk '{print $3}'")

elif vendor == "HP":
   print("1. HARDWARE VENDOR:"+vendor)
   print("2. CPU INFO")
   remote("cat /proc/cpuinfo | grep -e 'model name' -e 'physical id' | cut -c 1- | sort | uniq -c")
   remote("cat /etc/release | grep PLATFORM")
   print("3. PHYSICAL DISK STATUS")
   remote("hpssacli ctrl all show config | egrep -i '(ok|failed|error|offline|rebuild|ignoring|degraded|skipping|nok)'")
   print("4. BATERRY STATUS")
   remote("hpssacli ctrl all show status | grep 'OK'")
   print("5. Bios Version:")
   remote("dmidecode --type bios | grep -e Version -e 'Release Date'")

else:
   print("1. HARDWARE VENDOR:"+vendor)
   print(vendor+" is this")

Any help and hints would be appreciated.

!!! UPDATE !!!

I decided to use CSV to create output.csv file with some ID which I can use later to call upon external api.py file to insert data with API to a Web Application.

The code looks like this, but still now finished and needs more work. Any help would be appreciated:

class myssh:

    def __init__(self, hostname, username, password, port=22):
        client = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        client.connect(hostname, port=port, username=username, password=password)
        atexit.register(client.close)
        self.client = client

    def __call__(self, command):
        stdin, stdout, stderr = self.client.exec_command(command)
        sshdata = stdout.readlines()
        for line in sshdata:
            # end = '' -  to  loose space after the loops new lines
            print(line, end='')


remote = myssh(hostname, username, password)


if vendor == "Dell":
    f = open('dellcsv.csv', 'r')
    csv_f = csv.reader(f, delimiter=';')
    for row in csv_f:
        remote(row[4])
        # print(row[4])
    f.close()

elif vendor == "HP":
    f = open('dellcsv.csv', 'r')
    csv_f = csv.reader(f, delimiter=';')
    for row in csv_f:
        remote(row[4])
        # print(row[4])
    f.close()

else:
    print("1. HARDWARE VENDOR:" + vendor)
    print(vendor + " is this")

Answer 1

As per comments, you can append every line to an empty list, open the csv file in desired format in w (write) mode and write the list to the csv. This will create the csv file with the data in sshdata , if that's what you are looking for:

import csv
filename = 'ssh_output.csv'
def __call__():
    #your code here#
    hostdata_list = []
    hardware_data = []
    for line in sshdata:
        if 'HARDWARE' in line:
            row = 'some_data'
            hardware_data.append(row)
        hostdata_list.append(line)

    with open('input.csv', 'r') as f:
        reader = csv.reader(f)
        header = next(reader)

    with open(filename, 'a+') as f:
        writer = csv.writer(f)  
        writer.writerow(header)  
        if hardware_data is not None:
            writer.writerow(hardware_data)
        writer.writerow(hostdata_list)
    print('sshdata written to {} for host {}'.format(filename, hostname))

A proposed XML solution :

The following solution provides a workflow that accepts hp_input.xml for vendor HP. Of course, you may have dell_input.xml if the vendor is Dell.

I've used the xmltodict and dicttoxml libraries to convert the input to a dict, add an output key for each command, update the value for this key based on output for the command, then convert the output dict to xml again and write to an hp_output.xml or dell_output.xml file. To write the xml in pretty print, I used lxml . Make sure you install the required libraries with pip before proceeding.

First, return the stdout from __call__() in the ssh class:

class ssh:
    #your code#
    def __call__():
        stdout_outputs = []
        #your code#
        stdout_outputs.append(stdout.read())
        return stdout_outputs

Next, the following function will accept the input xml and convert it to dict:

import xmltodict
import lxml.etree as etree
import dicttoxml
import json

def xml_to_dict(xml_doc):
    with open(xml_doc) as f:
        data = xmltodict.parse(f.read())
        # print(json.dumps(data, indent=4)) #check dict
    return data

The following function gets the command value from the above dict and sends it for execution to remote , receives the output and updates the dict received from above.

def get_output_dict():    
    if vendor == 'HP':    
        data = xml_to_dict(xml_doc='hp_input.xml')      
    elif vendor == 'Dell':
        data = xml_to_dict(xml_doc='dell_input.xml')          
    for test in data['platform']['vendor']['tests']:        
        command = test.get('command',[]) #continue if command is not present
        if isinstance(command, list):
            for i in command:
                output = list(dict(i=remote(i)))                
        output = list(dict(command=remote(command)))              
        test['output'] = output
    return json.loads(json.dumps(data))

Now that we have the final dict from the above function, the following function will convert it to xml again and write it to an output xml file:

def get_output_xml(output_dict):
    output_xml = dicttoxml.dicttoxml(output_dict,custom_root='output', attr_type=False, root=False)
    if vendor == 'HP':
        filename = 'hp_output.xml'
    elif vendor == 'Dell':
        filename = 'dell_output.xml'
    tree = etree.fromstring(output_xml)
    output_xml_string = etree.tostring(tree, pretty_print=True)        
    with open(filename, 'wb') as f:
        f.write(output_xml_string)
    print('Output for hostname server: {} written to: {}'.format(hostname, filename))        
    return output_xml

You may put all the functions into one class and add exception handling as you like. We can call the functions like this:

data = get_output_dict()
xml = get_output_xml(data)

The code was tested with input xml provided and a dummy value for output. Although, I haven't tested with paramiko , this code should give you a basic workflow to get your desired output xml and if any errors crop up, they can be debugged.