Python for loop output to a dataframe

Question

I'm new to python and have started to learn programming during the lockdowns of COVID-19. I thought this would be a good dataset to practice with so have built a program that graphs the data based on input parameters. I have built a for loop that iterates through the data, plots a line for each country specified and then performs linear regression, outputting a predicted value for a given date. The outputting print statement works fine for what I want, which is the data, but I want to turn this into a dataframe called prediction. I'm having difficulty doing it, and have tried a numpy array as well as using pandas. I am able to output into an array or a dataframe, but it only will input the data from the very last entry on the loop iteration. I have read other posts with a similar issue, but I can't read their code accurately enough to draw an analogous solution for my code. Any help would be really appreciated.

prediction = pd.DataFrame()

countries = [country for country in covid4]

for country in covid4:
    plt.plot(covid4.index, covid4[country], marker = ',', linestyle = '-', linewidth = 2)

    slope, intercept, rval, pval, stder = linregress(covid4.index, covid4[country])
    y2 = (slope * days_from_covid_start) + intercept
    print(country, y2, (rval**2), pval, stder)

    #prediction = pd.DataFrame({'country': country, 'predicted': y2, 'rval': (rval**2), 'pval': pval, 'stder': stder}, index = [len(country)])
    prediction.append(pd.DataFrame([country,y2, (rval**2), pval, stder], index=[0]), ignore_index=True)

print output = US 5179943.844549271 0.9006676283944509 2.6485459147090845e-114 621.5097854328127
Brazil 2936531.6824269355 0.7832301173896118 2.574767200468765e-76 594.2400982216876
India 2073088.1203383887 0.6345313255372067 7.298314985773991e-51 624.2194955678399
Russia 978300.4286278635 0.9142926929617138 1.7522978381731647e-121 108.50766510347822
Peru 505978.0715319741 0.858987519325372 2.99136171150524e-97 76.57252904070529
Colombia 356724.59233558143 0.6333783192991587 1.0395048549255093e-50 107.7550260556157
South Africa 472308.1410419348 0.6966400875418044 6.071717833597016e-60 122.57207991971943
Mexico 466476.80000000005 0.8003861479274035 2.486332913502741e-80 89.06795808066232
Spain 406391.36529570003 0.9113259379908633 7.934640637615308e-120 39.7913275775704
Argentina 231457.53997115127 0.6268247164508722 7.600868471677846e-50 70.99075237604976
Chile 400397.3610073682 0.8667991679346673 5.033317538752307e-100 58.15733490786391
Iran 361576.26469403406 0.964179243093371 6.270093847731166e-164 23.965905293313885
United Kingdom 396538.81819552195 0.9237619398832538 3.5207778505031074e-127 36.6462363789314
France 307895.94598781073 0.9160909120260833 1.628524516693465e-122 29.273492653868225
Bangladesh 258210.0824529258 0.8134667793006155 1.245704530630672e-83 47.16886076960728
Saudi Arabia 301026.7291359661 0.8720838496759029 5.387824647150151e-102 42.7760188991517

***edit covid4 has columns for each country and the index of the date, the rows indicate the cases per country for that date.

for context the full code is below

#specifies countries that have above 'n' number
n = 300000

#search for a specific country's COVID data from the date specified.
country_to_find = ['']
value = 'cases'
today = '7/30/20'
#specifies how many days from covid first onset to map linear regression model for country to find
days_from_covid_start = 230


import matplotlib.pyplot as plt 
import pandas as pd
import numpy as np
from datetime import date
import seaborn as sns
from scipy.stats import linregress

sns.set()

df = pd.read_csv('https://data.humdata.org/hxlproxy/api/data-preview.csv?url=https%3A%2F%2Fraw.githubusercontent.com%2FCSSEGISandData%2FCOVID-19%2Fmaster%2Fcsse_covid_19_data%2Fcsse_covid_19_time_series%2Ftime_series_covid19_confirmed_global.csv&filename=time_series_covid19_confirmed_global.csv')
covid = df.groupby('Country/Region', as_index = False).sum()

date_i = covid[(today)]
G = str(date_i.sum())
covid = covid.sort_values((today), ascending=False)

covid.drop(columns=['Lat', 'Long'], inplace=True)
sig_cases = covid[date_i <= n].index
covid.drop(sig_cases, inplace=True)
plt.figure(figsize=(14,7.2))
sns.barplot(y = covid['Country/Region'], x = date_i, data = covid, )


covid = covid.sort_values((today), ascending=True)
x_values = range(len(covid['Country/Region']))
countries = covid['Country/Region'].map(str)


print(date_i)
#matplotlib graph, shows the same data as previous seaborn plot

'''cases = covid[(today)]
plt.figure(figsize=(14,7.2))
plt.ylabel('Countries')
plt.xlabel('Total Recorded')
plt.title('Covid Cases By Country', size = 10)
group = covid.groupby('Country/Region').sum()
plt.yticks(x_values, countries, rotation= 0, size = 10)
plt.barh(x_values, cases,  color='Red')
plt.grid()'''



covid.sort_values((today), ascending = False, inplace = True)

covid_transposed = covid.transpose()

covid_transposed.columns = covid_transposed.iloc[0]
covid_transposed2 = covid_transposed[1:]

covid_transposed2.to_csv('covid_transposed_new_index.csv', index = False)
covid_transposed2.to_csv('covid_transposed_new.csv')


covid5 = pd.read_csv('covid_transposed_new.csv')
covid5.rename( columns={'Unnamed: 0':'Date'}, inplace=True)
sns.set_style("ticks")
plt.subplot()
plt.figure(figsize=(14,7.2))

for country in covid5:
    if country in country_to_find:
        plt.plot(covid5.index, covid5[country], marker = ',', linestyle = '-', linewidth = 2)
        slope, intercept, rval, pval, stder = linregress(covid5.index, covid5[country])
        y = (slope * days_from_covid_start) + intercept
        print(y, rval **2 , pval, stder)

plt.xticks(covid5.index[::30], covid5.Date[::30])
plt.legend(country_to_find, loc=2, prop={'size': 15})
plt.xticks(rotation=32, size = 15)
plt.yticks(size = 15)
plt.title('Covid ' +str(value)+ ' per selected country', size=20)
plt.ylabel('Num ' +str(value), size=20)
plt.xlabel('Date', size=20)
plt.grid(True)

covid4 = pd.read_csv('covid_transposed_new_index.csv')

plt.figure(figsize=(14,7.2))

plt.subplot()
prediction = pd.DataFrame()

countries = [country for country in covid4]

for country in covid4:
    plt.plot(covid4.index, covid4[country], marker = ',', linestyle = '-', linewidth = 2)
    slope, intercept, rval, pval, stder = linregress(covid4.index, covid4[country])
    y2 = (slope * days_from_covid_start) + intercept
    prediction.append(pd.DataFrame(np.array([[country], [y2], [(rval**2)], [pval], [stder]])))
    print(country, y2, (rval**2), pval, stder)
    

plt.xticks(covid4.index[::30])
plt.legend(countries, loc=2, prop={'size': 9})
plt.xticks(rotation=32, size = 15)
plt.yticks(size = 15)
plt.title('Covid ' +str(value)+ ' per selected country', size=20)
plt.ylabel('Num ' +str(value), size=20)
plt.xlabel('Days from first COVID-19 identification', size=20)
plt.grid(True)

sns.set_style("darkgrid")


plt.show()

print(str('Total cases as of date ' + str(today) + ' is ')+ G)
print(prediction)

ps I know the regression model doesn't work haha

Answer 1

• 1. Extract all necessary data from the covid4 .

---

• Same as you did countries = [country for country in covid4]

• Do for the other data

• ie y2 = [y2 for y2 in covid4]

• rval = [rval for rval in covid4]

• pval = [pval for pval in covid4]

• stder = [stder for stder in covid4]

• 2. Create a dictionary to store the extracted data

---

* `data = {'country': country, 'y2': y2, 'rval': rval, 'pval': pval, 'stder': stder}`

• If the above doesn't work, then add data with array []

• For instance: data = {'country': [country], ... }

• 3. Now you can create DataFrame

---

* `df = pd.DataFrame(data, columns=['country', 'y2', 'rval', 'pval', 'stder'])`

Answer 2

DataFrame.append() appends to the specified dataframe and then returns a new object.

Documentation

You can create your initial object with the columns you desire, and then when appending to the dataframe make sure you specify a new object for it.

prediction = pd.DataFrame(columns=['country', 'y2', 'rval', 'pval', 'stder'])

for country in covid4:
    plt.plot(covid4.index, covid4[country], marker = ',', linestyle = '-', linewidth = 2)
    slope, intercept, rval, pval, stder = linregress(covid4.index, covid4[country])
    y2 = (slope * days_from_covid_start) + intercept
    prediction = prediction.append(pd.DataFrame(data={'country': country, 'y2': y2, 'rval': (rval**2), 'pval': pval, 'stder': stder}, index = [0]), ignore_index = True)
    print(country, y2, (rval**2), pval, stder)

This outputs the result as follows;

          country            y2      rval           pval       stder
0              US  5.179944e+06  0.900668  2.648546e-114  621.509785
1          Brazil  2.936532e+06  0.783230   2.574767e-76  594.240098
2           India  2.073088e+06  0.634531   7.298315e-51  624.219496
3          Russia  9.783004e+05  0.914293  1.752298e-121  108.507665
4    South Africa  4.723081e+05  0.696640   6.071718e-60  122.572080
5          Mexico  4.664768e+05  0.800386   2.486333e-80   89.067958
6            Peru  5.059781e+05  0.858988   2.991362e-97   76.572529
7           Chile  4.003974e+05  0.866799  5.033318e-100   58.157335
8  United Kingdom  3.965388e+05  0.923762  3.520778e-127   36.646236
9            Iran  3.615763e+05  0.964179  6.270094e-164   23.965905