在Python中從網站下載csv文件的問題

Question

因此，我嘗試從需要我的電子郵件地址和密碼作為站點身份驗證的站點下載並將csv文件寫入我的計算機。 我有以下代碼：

import cStringIO
import pycurl
import urllib

url = 'http://www.riglocator.ca/report=rig%2Frig%2D150226%2Ecsv'



def GetPage(url, proxy=None):
  if proxy:
    port = 8888
    proxy = proxy.replace("socks://", "")
    if ":" in proxy:
      port = int(proxy.rsplit(":", 1)[1])
      proxy = proxy.rsplit(":", 1)[0]
  try:
    buf = cStringIO.StringIO()
    c = pycurl.Curl()
    c.setopt(c.URL, url)
    c.setopt(c.WRITEFUNCTION, buf.write)
    c.setopt(c.CONNECTTIMEOUT, 5)
    c.setopt(c.TIMEOUT, 8)
    if proxy:
      c.setopt(pycurl.PROXY, proxy)
      c.setopt(pycurl.PROXYPORT, port)
      c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5)
      c.setopt(pycurl.USERPWD, 'john@mail.com:password123')
    c.setopt(c.FOLLOWLOCATION, True)
    c.perform()
    c.close()
    results = buf.getvalue()
    buf.close()
  except:
    results = ""
  return results

GetPage(url,"socks://127.0.0.1:8888")

def loader():
    csv_url = GetPage(url,"socks://127.0.0.1:8888")
    r = urllib.urlopen(csv_url)
    print(r)
    csv = r.read()
    csv_str = str(csv)
    lines = csv_str.split('\\n')
    dest_url = r'mapfile.csv'
    fx = open(dest_url, 'w')
    for line in lines:
        fx.write(line + '\n')
    fx.close()

loader()

但是這仍然會從登錄頁面返回HTML代碼，有什么建議嗎？

我收到此錯誤：

     File "C:/Users/cevans/PycharmProjects/RigLocatorMapPull/rigmapscrape.py", line 55, in <module>
loader()
  File "C:/Users/cevans/PycharmProjects/RigLocatorMapPull/rigmapscrape.py", line 44, in loader
    r = urllib.urlopen(csv_url)
  File "C:\Python27\lib\urllib.py", line 87, in urlopen
    return opener.open(url)
  File "C:\Python27\lib\urllib.py", line 208, in open
    return getattr(self, name)(url)
  File "C:\Python27\lib\urllib.py", line 463, in open_file
    return self.open_local_file(url)
  File "C:\Python27\lib\urllib.py", line 477, in open_local_file
    raise IOError(e.errno, e.strerror, e.filename)
IOError: [Errno 2] The system cannot find the path specified: ''

Process finished with exit code 1

Answer 1

這是我用pycurl抓取文件的一些代碼的鏈接，它基本上應該做你需要做的事情。 你只需要添加選項c.setopt（pycurl.USERPWD，'username：userpass'）做我的代碼來設置你的用戶名和密碼。

http://prestongarrison.com/proper-python-pycurl-example/

Answer 2

#This is a solution using the Mechanize browser library which takes the url, 
#changes it to the current date, submits the username/password in a form, 
#downloads a csv and writes it to a folder location: 

__author__ = 'cevans'

import mechanize
import os
import cookielib
import datetime, string

USERNAME = 'xxxx'
PASSWORD = 'xxxxx'
OLDURL = 'http://www.oldurl.com/report050301'
folder = r'\\Driver'

def loader():
#Takes current date and changes URL to grab correct datefile (Schedule only runs on day of week)
    cdate = str(datetime.date.today().strftime("%y%m%d"))
    DATAURL = string.replace(OLDURL,'150301',cdate)

# Browser and Cookie Jar
    br = mechanize.Browser()
    cj = cookielib.LWPCookieJar()
    br.set_cookiejar(cj)
# Browser options
    br.set_handle_equiv(True)
    br.set_handle_gzip(False)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(True)
# Follows refresh 0 but not hangs on refresh > 0
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
# Opens site:
    r = br.open(DATAURL)
    html = r.read()
    br.select_form(nr=0)
    br.form['nauthemail']= USERNAME
    br.form['password']=PASSWORD
    br.submit()
    r = br.open(DATAURL)
#Read and write file to csv, in folder

    csv = r.read()
    csv_str = str(csv)
    lines = csv_str.split('\\n')
    fname = 'map-'+ cdate
    base_filename=fname
    filename_suffix = '.csv'
    folder1 = os.path.join(folder, base_filename + filename_suffix)
    dest_url = folder1
    fx = open(dest_url, 'w')
    for line in lines:
    fx.write(line + '\n')
    fx.close()

loader()