problem in choosing K-Fold cross validation value

Question

for a course in informatics, it has been given to implement the Cholesky decomposition and to choose the best value for a KFold cross validation, for the LS solution. The problem is that when I try some values for the KFold, different from 2, I receive the error below, ie the dimensions of the two matrices don't match. The fact is that I can't reshape the matrices. Any idea? Thanks

        
 # import libraries
 %matplotlib inline
 import numpy as np
 import matplotlib as cm
 import matplotlib.pyplot as plt
 import scipy.linalg
 import os
 import pickle
            
        # functions definitions
            def linearRegrFunction(n, D, low_D, high_D, W, sigma_noise):
                X = np.zeros((n,D))
                for i in range(0, D):
                    X[:,i] = np.random.uniform(low_D[i], high_D[i], size=n)
                
                gauss_noise = np.random.normal(0, sigma_noise, size=(n,1))
            
                Y = np.dot(X, W) + gauss_noise
                
                return X, Y
            
            def regularizedLSTrain(Xtr, Ytr, reg_par):
                A = Xtr.T * Xtr + reg_par * len(Xtr) * np.identity(len(Xtr))
                b = Xtr.T * Ytr
                L = scipy.linalg.cholesky(A, lower=True, check_finite=True)
                L_star = L.conj().T
                y = np.divide(b, L, out=np.zeros_like(L), where=L!=0)
                x = np.divide(y, L_star, out=np.zeros_like(L_star), where=L_star!=0)
                return x
            
            def calcError(Ypred, Ytrue):
                return np.mean((Ypred-Ytrue)**2)
            
            def regularizedLSTest(w, Xtest):
                return np.dot(Xtest.T, w)
         
        # first part of main program
            n = 100
            D = 1
            sigma_noise = 10
        # w matrix, Aw = b
            truew = np.array([1]).transpose()
            truew.shape = (D,1)
            
            low_D = np.array([-100])
            high_D = np.array([100])
            
            Xtr, Ytr = linearRegrFunction(n, D, low_D, high_D, truew, sigma_noise)
            
            reg_par = 100
    
    # estimated w        
            estw = regularizedLSTrain(Xtr, Ytr, reg_par)
            
            Xte, Yte = linearRegrFunction(n, D, low_D, high_D, truew, sigma_noise)
    
    # estimated error on the training and test set        
            test_err = regularizedLSTest(estw, Xte)
            print(test_err)
            train_err = regularizedLSTest(estw, Xtr)
                                          
            print("MSE on the test set "+str(test_err))
            print("MSE on the training set "+str(train_err))
            
            reg_pars = [10,20,30,40,50,60,70,100,200,500,1000,2000,3000,4000,5000,6000,10000,50000,100000]
            #reg_pars = [1000,10000]
            #reg_pars = (70, 100, 1000, 10000)
            #reg_pars = range(100, 100000, 100)
            
            errs_tr = np.zeros(np.shape(reg_pars))
            errs_te = np.zeros(np.shape(reg_pars))
            
            for i in range(len(reg_pars)):
            
                w = regularizedLSTrain(Xtr, Ytr, reg_pars[i])
                
                Ypred_tr = regularizedLSTest(w, Xtr)
                Ypred_te = regularizedLSTest(w, Xte)
                
                errs_tr[i] = calcError(Ypred_tr, Ytr)
                errs_te[i] = calcError(Ypred_te, Yte)
            
            plt.plot(reg_pars, errs_te, '-o')
            plt.plot(reg_pars, errs_tr, '-o')
            plt.xlabel("Regularization parameter")
            plt.ylabel("MSE")
        
        
        # definition of the function for the least square solution (cross validation)    
            def KFoldCVRLS(Xtr, Ytr, KF, regpar_list):
            
                regpar_list = np.array(regpar_list)
                num_regpar = regpar_list.size
            
                n_tot = Xtr.shape[0]
                n_val = int(np.ceil(n_tot/KF))
            
                Tm = np.zeros(num_regpar)
                Ts = np.zeros(num_regpar)
                Vm = np.zeros(num_regpar)
                Vs = np.zeros(num_regpar)
            
                rand_idx = np.random.choice(n_tot, size=n_tot, replace=False)
                
                
                for kdx, regpar in enumerate(regpar_list):
                    first = 0
                    for fold in range(KF):
                        
                        flags = np.zeros(Xtr.shape[0])
                        flags[first:first+n_val]=1;
                        
                        X = Xtr[rand_idx[flags==0]]
                        Y = Ytr[rand_idx[flags==0]]
                        X_val = Xtr[rand_idx[flags==1]]
                        Y_val = Ytr[rand_idx[flags==1]]
                        
                        currW = regularizedLSTrain(X, Y, regpar)
                        YpredTR = regularizedLSTest(currW, X)
                        trError = calcError(YpredTR, Y)
                        Tm[kdx] = Tm[kdx] + trError
                        Ts[kdx] = Ts[kdx] + trError ** 2
                        
                        YpredVAL = regularizedLSTest(currW, X_val)
                        valError = calcError(YpredVAL, Y_val)
                        Vm[kdx] = Vm[kdx] + valError
                        Vs[kdx] = Vs[kdx] + valError ** 2
                        
                        first = first+n_val                
            
                Tm = Tm / KF
                Ts = Ts / KF - Tm ** 2
            
                Vm = Vm / KF
                Vs = Vs / KF - Vm ** 2
            
                best_regpar_idx = np.argmin(Vm)
                bestregpar = regpar_list[best_regpar_idx]
            
                return bestregpar, Vm, Vs, Tm, Ts
         
        # second part of "main"   
            KF = 10
            bestregpar, Vm, Vs, Tm, Ts = KFoldCVRLS(Xtr, Ytr, KF, reg_pars)
            
            plt.plot(reg_pars, Vm, '-o')
            plt.plot(reg_pars, Tm, '-x')
            plt.xlabel("Regularization parameter")
            plt.ylabel("MSE")
        
        

    <hr>
    <p>
    <pre><code>
    
        ---------------------------------------------------------------------------
        ValueError                                Traceback (most recent call last)
        <ipython-input-11-050d1c546189> in <module>
              1 KF = 10
        ----> 2 bestregpar, Vm, Vs, Tm, Ts = KFoldCVRLS(Xtr, Ytr, KF, reg_pars)
              3 
              4 plt.plot(reg_pars, Vm, '-o')
              5 plt.plot(reg_pars, Tm, '-x')
        
        <ipython-input-10-3dbc5e035cda> in KFoldCVRLS(Xtr, Ytr, KF, regpar_list)
             41 
             42             # Compute the validation error of the RLS regression for the given value of regpar
        ---> 43             YpredVAL = regularizedLSTest(currW, X_val)
             44             valError = calcError(YpredVAL, Y_val)
             45             Vm[kdx] = Vm[kdx] + valError
        
        <ipython-input-5-3fb9cc618491> in regularizedLSTest(w, Xtest)
              1 def regularizedLSTest(w, Xtest):
        ----> 2     return np.dot(Xtest.T, w)
        
        <__array_function__ internals> in dot(*args, **kwargs)
        
        ValueError: shapes (1,10) and (90,90) not aligned: 10 (dim 1) != 90 (dim 0)
    
    </code></pre>
    </p>

Answer 1

You can try to change this function from

      1 def regularizedLSTest(w, Xtest):
----> 2     return np.dot(Xtest.T, w)

to

      def regularizedLSTest(w, Xtest):
          return np.dot(w, Xtest.T)