I ma newbie to python. I want to pass a command-line argument to my presto query which is inside a function and then writes the result as a CSV file. But when I try to run it on the terminal it says ' Traceback (most recent call last): File "function2.py", line 3, in <module> from pyhive import presto ModuleNotFoundError: No module named 'pyhive'
The pyhive requirement is already satisfied. Please find attached my code:
from sys import argv
import argparse
from pyhive import presto
import prestodb
import csv
import sys
import pandas as pd
connection = presto.connect(host='xyz',port=8889,username='test')
cur = connection.cursor()
print('Connection Established')
def func1(object,start,end):
object = argv[1]
start = argv[2]
end = argv[3]
result = cur.execute("""
with map_date as
(
SELECT
object,
epoch,
timestamp,
date,
map_agg(name, value) as map_values
from hive.schema.test1
where object = '${object}'
and (epoch >= '${start}' and epoch <= '${end}')
and name in ('x','y')
GROUP BY object,epoch,timestamp,date
order by timestamp asc
)
SELECT
epoch
, timestamp
, CASE WHEN element_at(map_values, 'x') IS NOT NULL THEN map_values['x'] ELSE NULL END AS x
, CASE WHEN element_at(map_values, 'y') IS NOT NULL THEN map_values['y'] ELSE NULL END AS y
, object
, date AS date
from map_date
""")
rows = cur.fetchall()
print('Query Finished') #Returns the list with one entry for each record
fp = open('/Users/xyz/Desktop/Python/function.csv', 'w')
print('File Created')
myFile = csv.writer(fp)
colnames = [desc[0] for desc in cur.description] #store the headers in variable called 'colnames'
myFile.writerow(colnames) #write the header to the file
myFile.writerows(rows)
fp.close()
func1(object,start,end)
cur.close()
connection.close()
How can I pass the command line argument to my Presto query which is written inside a function? Any help is much appreciated. Thank you In advance!
I only describe how to pass command line arguments to function and query.
If you define function
def func1(object, start, end):
# code
then you have to send values as varaibles and you have to use sys.argv
outside function
connection = presto.connect(host='xyz', port=8889, username='test') # PEP8: spaces after commas
cur = connection.cursor()
print('Connection Established')
object_ = sys.argv[1] # PEP8: there is class `object` so I add `_` to create different name
start = sys.argv[2]
end = sys.argv[3]
func1(object_, start, end)
cur.close()
connection.close()
You don't have to use the same names outside function
args1 = sys.argv[1]
args2 = sys.argv[2]
args3 = sys.argv[3]
func1(args1, args2, args3)
and you can even do
func1(sys.argv[1], sys.argv[2], sys.argv[3])
becuse when you run this line then python gets definition def func1(object, start, end):
and it creates local variables with names object, start, end
inside func1
and it assigns external value to these local variables
object=objec_, start=start, end=end
or
object=args1, start=args2, end=args2
or
object=sys.argv[1], start=sys.argv[1], end=sys.argv[1]
It would be good to send explicitly also cur
to function
def func1(cur, object_, start, end):
# code
and
func1(cur, sys.argv[1], sys.argv[2], sys.argv[3])
I don't know what you try to do in SQL query but Python uses {start}
(without $
) to put value in string (Bash uses ${start}
) and it needs prefix f
to create f-string
- f"""... {start}...."""
. Without f
you have to use normal string formatting """... {start}....""".format(start=start)
import sys
import csv
from pyhive import presto
# --- functions ----
def func1(cur, object_, start, end): # PEP8: spaces after commas
# Python use `{star} {end}`, Bash uses `${start} ${end}`
# String needs prefix `f` to use `{name} {end}` in f-string
# or you have to use `"{start} {end}".format(start=value1, end=value2)`
result = cur.execute(f"""
WITH map_date AS
(
SELECT
object,
epoch,
timestamp,
date,
map_agg(name, value) AS map_values
FROM hive.schema.test1
WHERE object = '{object_}'
AND (epoch >= '{start}' AND epoch <= '{end}')
AND name IN ('x','y')
GROUP BY object,epoch,timestamp,date
ORDER BY timestamp asc
)
SELECT
epoch,
timestamp,
CASE WHEN element_at(map_values, 'x') IS NOT NULL THEN map_values['x'] ELSE NULL END AS x,
CASE WHEN element_at(map_values, 'y') IS NOT NULL THEN map_values['y'] ELSE NULL END AS y,
object,
date AS date
FROM map_date
""")
rows = cur.fetchall()
colnames = [desc[0] for desc in cur.description] # store the headers in variable called 'colnames'
print('Query Finished') # returns the list with one entry for each record
fp = open('/Users/xyz/Desktop/Python/function.csv', 'w')
my_file = csv.writer(fp) # PEP8: lower_case_names for variables
my_file.writerow(colnames) # write the header to the file
my_file.writerows(rows)
fp.close()
print('File Created')
# --- main ---
connection = presto.connect(host='xyz', port=8889, username='test') # PEP8: spaces after commas
cur = connection.cursor()
print('Connection Established')
#object_ = sys.argv[1] # PEP8: there is class `object` so I add `_` to create different name
#start = sys.argv[2]
#end = sys.argv[3]
#func1(cur, object_, start, end)
func1(cur, sys.argv[1], sys.argv[2], sys.argv[3])
cur.close()
connection.close()
If you plan to use argparse
parser = argparse.ArgumentParser()
parser.add_argument('-o', '--object', help='object to search')
parser.add_argument('-s', '--start', help='epoch start')
parser.add_argument('-e', '--end', help='epoch end')
args = parser.parse_args()
and then
func1(cur, args.object, args.start, args.end)
import argparse
# ... imports and functions ...
# --- main ---
parser = argparse.ArgumentParser()
parser.add_argument('-o', '--object', help='object to search')
parser.add_argument('-s', '--start', help='epoch start')
parser.add_argument('-e', '--end', help='epoch end')
#parser.add_argument('-D', '--debug', action='store_true', help='debug (display extra info)')
args = parser.parse_args()
#if args.debug:
# print(args)
connection = presto.connect(host='xyz', port=8889, username='test') # PEP8: spaces after commas
cur = connection.cursor()
print('Connection Established')
func1(cur, args.object, args.start, args.end)
cur.close()
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.