Python - Mapreduce - PermissionError: [WinError 5] Access is denied

Question

I am getting this error, I tried admin rights, open as admin, UAC off..but still same problem, can anyone tell what is the problem? I am passing 2 files movies2.csv and ratings2.csv from terminal

from mrjob.job import MRJob
from mrjob.step import MRStep
from itertools import combinations
import numpy
from scipy import spatial
class movies_count(MRJob):
 #  steps function determine the sequence of operations   
    def steps(self):
        return [
            MRStep(mapper=self.moviedatasplit,
                reducer=self.joinfilereducer),
          MRStep(reducer=self.reducer_moviepairs),
          MRStep(reducer=self.reducer_pairs),
          MRStep(reducer=self.movie_similarity)
    ]

 # Passing two files (movies.csv and ratings.csv) to the first mapper  
    def moviedatasplit(self, _, line):
                dsplit = line.split(",")
                if (len(dsplit) == 3): # movie data
                          yield dsplit[0], dsplit[1]
                else: # rating data
                          yield dsplit[1], (dsplit[0], dsplit[2])
# generating user id as key and movie title, movierating as values with the help of first reducer
    def joinfilereducer(self, _, values):
                movielist = list(values)
                movietitle = movielist[0]
                tuplevalue = movielist[1:]
                for val in tuplevalue:
                        userid = val[0]
                        movierating = val[1]    
                        yield userid, (movietitle, movierating)      
# generating combination of two movies as key and their respective ratings as value for each user id with the second reducer   
    def reducer_moviepairs(self,userid,values):
         for pair1, pair2 in combinations(values,2):
            title1=pair1[0]
            rating1=pair1[1]
            title2=pair2[0]
            rating2=pair2[1]
            yield (title1,title2),(rating1,rating2)
# combining all the ratings for each movie pair by different users with the third reducer     
    def reducer_pairs(self,titles,ratings):
        rating=[]
        for r in ratings:
            rating.append(r)
        yield titles,rating

# finding similarity between movies using statistical coorelation and cosine similarity
    def movie_similarity(self,titles,ratings):     
        rating =list(ratings)
        for ratings in rating:
            n=len(ratings)
        q1=[]
        q2=[]
        for r1 in ratings:
            q1.append((float(r1[0])))
            q2.append((float(r1[1])))

        if(n>3):           
                cor = numpy.corrcoef(q1,q2)[0,1]
                cos_cor = 1-spatial.distance.cosine(q1,q2)
                avg_cor = 0.5*(cor+cos_cor)
                yield titles[0], (titles[1],avg_cor,cor,cos_cor,n)
# main function                               
if __name__ == '__main__':
            movies_count.run()

Error:

No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
Running step 1 of 4...
Running step 1 of 4...
Running step 1 of 4...
Creating temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579
Creating temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579
Creating temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579
Running step 2 of 4...
Running step 2 of 4...
Running step 2 of 4...
Running step 3 of 4...
Running step 3 of 4...
Running step 3 of 4...
Running step 4 of 4...
Running step 4 of 4...
Running step 4 of 4...
Streaming final output from C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579\output...
Streaming final output from C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579\output...
Streaming final output from C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579\output...
Removing temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579...
Removing temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579...
Removing temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579...
[WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
Traceback (most recent call last):
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\mrjob\runner.py", line 606, in _cleanup_local_tmp
    shutil.rmtree(self._local_tmp_dir)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 494, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 389, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 387, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
[WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
Traceback (most recent call last):
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\mrjob\runner.py", line 606, in _cleanup_local_tmp
    shutil.rmtree(self._local_tmp_dir)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 494, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 389, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 387, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
[WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
Traceback (most recent call last):
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\mrjob\runner.py", line 606, in _cleanup_local_tmp
    shutil.rmtree(self._local_tmp_dir)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 494, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 389, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 387, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'

Answer 1

It seems to have an issue with permission. Try changing the path for temp folder.

In the windows system, the temporary folder is determine by

1. The directory named by the TMPDIR environment variable.
2. The directory named by the TEMP environment variable.
3. The directory named by the TMP environment variable.
4. Platform-specific location - the directories C:\\TEMP, C:\\TMP, \\TEMP, and \\TMP, in that order.

Create/Edit an environment variable TMPDIR/TEMP/TMP and set value as C:\\temp . Try re-running the program.