Python create dynamic/loop based on variables and CSV

Question

How do I fix this code to response dynamically to multiple entries (any number of keywords and cvs files) as asked here ?

It is a Python 2.x script that works for 2 keywords located in keywords.txt and 2 CSV files as seen opened withing the code.

import csv
from itertools import chain


with open("keywords.txt", "rb") as keywords, open('folding umbrella-sort-highest.csv', 'rb') as g, open('lego minecraft-sort-highest.csv', 'rb') as f, open('filename1.csv', 'wb') as myfile1, open('filename2.csv', 'wb') as myfile2:

# Step1: Read contents of keywords.tex in variables
    ky = list(keywords.readlines())
    ky1, ky2 = ky[0], ky[1]

# Step2: Read and process folding umbrella-sort-highest.csv
    reader = csv.reader(g)
    umbrella_list = list(reader)
    list1 = filter(lambda e: e[0] in ky1, umbrella_list)

    list2 = list(chain(*list1))
    # or better: (available since Python 2.6)
    # print list(chain.from_iterable(list1))

    ind_prt1 = umbrella_list.index(list2) +1 
    mylist1 = umbrella_list[:ind_prt1]

    for r in mylist1:
        wr = csv.writer(myfile1, quoting=csv.QUOTE_ALL)
        wr.writerow(r)

# Step3: Read and process lego minecraft-sort-highest.csv
    reader = csv.reader(f)
    minecraft_list = list(reader)
    list3 = filter(lambda e: e[0] in ky2, minecraft_list)

    list4 = list(chain(*list3))
    # or better: (available since Python 2.6)
    # print list(chain.from_iterable(list4))

    ind_prt2 = minecraft_list.index(list4) +1 
    mylist2 = minecraft_list[:ind_prt2]

    for r in mylist2:
        wr = csv.writer(myfile2, quoting=csv.QUOTE_ALL)
        wr.writerow(r)

print "Task completed, check your working directory."

The code above works for 2 keywords located in keywords.txt and 2 CSV files as seen. I want it to be dynamic to handle any numbers of keywords and CSV file?

Answer 1

I can't test it but it can be

import csv
from itertools import chain
import glob

# --- read keywords ----

with open("keywords.txt", "rb") as keywords:
    # remove '\n'
    all_keys = [x.strip() for x in keywords] 

# --- ---

#all_filenames = [
#    'folding umbrella-sort-highest.csv',
#    'lego minecraft-sort-highest.csv',
#]

all_filenames = glob.glob('csv/*.csv')

print all_filenames

# get pair: one key and one filename + current index/number
for idx, (key, input_name) in enumerate(zip(all_keys, all_filenames), 1):

    print idx, key, input_name

    # - read all data from input file -

    with open(input_name, 'rb') as f_in:
        reader = csv.reader(f_in)
        data = list(reader)

    # - change data -

    sublist = filter(lambda e: e[0] in key, data)

    sublist = list(chain(*sublist))
    # or better: (available since Python 2.6)
    # print list(chain.from_iterable(sublist))

    idx_prt = data.index(sublist)+1 
    output = data[:idx_prt]

    # - write all data to output file -

    output_name = 'filename{}.csv'.format(idx)

    with open(output_name, 'wb') as f_out:
        writer = csv.writer(f_out, quoting=csv.QUOTE_ALL)
        writer.writerows(output)

# --- the end ---

print "Task completed, check your working directory."