How to import start_urls from CSV

Question

I have a program in scrapy that scrapes multiple URLs and each of those urls has multiple pages and right now it is outputting the out of order. It will output all of the page 1's then all of the page 2's and so on instead of page 1,2.3... of URL 1 and page 1,2,3 of URL 2 and so. And anyways after a lot of research it seems like the way to fix this is to import my URLS from my CSV and then set that equal to start_urls but I cannot figure out how to do that in my current program.

# Import from other python files and scrapy files and the needed csv file containing all URLs/proxies/ua
import csv
import scrapy
from scrapy.spiders import Spider
from scrapy_splash import SplashRequest
from ..items import GameItem
from random import randint
from time import sleep
from scrapy.http import FormRequest
##########################          SPALSHSPIDER.PY OVERVIEW      #####################################################
# process the csv file so the url + ip address + useragent pairs are the same as defined in the file
# returns a list of dictionaries, example:
# [ {'url': 'http://www.starcitygames.com/catalog/category/Rivals%20of%20Ixalan',
#    'ip': 'http://204.152.114.244:8050',
#    'ua': "Mozilla/5.0 (BlackBerry; U; BlackBerry 9320; en-GB) AppleWebKit/534.11"},
#    ...
# ]
# plus python file also scrapes all URLs returning needed info and goes to all apges associated with URL by clicking next button

# Function to read csv file that contains URLs that are paried with proxies and user agents
def process_csv(csv_file):
    # Initialize data
    data = []
    # Initialize reader
    reader = csv.reader(csv_file)
    next(reader)

    # While inside csv file and not at end of csv file
    for fields in reader:

        # Set URL
        if fields[0] != "":
            url = fields[0]
        else:
            continue # skip the whole row if the url column is empty
        #Set proxy and pair with correct URL
        if fields[1] != "":
            ip = "http://" + fields[1] + ":8050" # adding http and port because this is the needed scheme
        # Set user agent and pair with correct URL
        if fields[2] != "":
            useragent = fields[2]
        # Put all three together
        data.append({"url": url, "ip": ip, "ua": useragent})
    # Return URL paried with ua and proxy
    return data

# Spider clascrapy crawl splash_spider -o data.json
class MySpider(Spider):

    # Name of Spider
    name = 'splash_spider'
    # getting all the url + ip address + useragent pairs then request them
    def start_requests(self):

        # get the file path of the csv file that contains the pairs from the settings.py
        with open(self.settings["PROXY_CSV_FILE"], mode="r") as csv_file:
           # requests is a list of dictionaries like this -> {url: str, ua: str, ip: str}
            requests = process_csv(csv_file)
        for req in requests:
            # Return needed url with set delay of 3 seconds
            yield SplashRequest(url=req["url"], callback=self.parse, args={"wait": 3},
                    # Pair with user agent specified in csv file
                    headers={"User-Agent": req["ua"]},
                    # Sets splash_url to whatever the current proxy that goes with current URL  is instead of actual splash url
                    splash_url = req["ip"],
                    )

    # Scraping function that will scrape URLs for specified information
   # import pdb; pdb.set_trace()
    def parse(self, response):

        # Initialize item to function GameItem located in items.py, will be called multiple times
        item = GameItem()
        # Initialize saved_name
        saved_name = ""
        # Extract card category from URL using html code from website that identifies the category.  Will be outputted before rest of data

        item["Category"] = response.css("span.titletext::text").get()

        # For loop to loop through HTML code until all necessary data has been scraped
        for game in response.css("tr[class^=deckdbbody]"):

            # Initialize saved_name to the extracted card name
            saved_name  = game.css("a.card_popup::text").get() or saved_name
            # Now call item and set equal to saved_name and strip leading '\n' from output
            item["Card_Name"] = saved_name.strip()
            # Check to see if output is null, in the case that there are two different conditions for one card
            if item["Card_Name"] != None:
                # If not null than store value in saved_name
                saved_name = item["Card_Name"].strip()
            # If null then set null value to previous card name since if there is a null value you should have the same card name twice
            else:
                item["Card_Name"] = saved_name
            # Call item again in order to extract the condition, stock, and price using the corresponding html code from the website
            item["Condition"] = game.css("td[class^=deckdbbody].search_results_7 a::text").get()
            item["Stock"] = game.css("td[class^=deckdbbody].search_results_8::text").get()
            item["Price"] = game.css("td[class^=deckdbbody].search_results_9::text").get()
            if item["Price"] == None:
                item["Price"] = game.css("td[class^=deckdbbody].search_results_9 span[style*='color:red']::text").get()
            # Return values
            yield item

        # Finds next page button
        next_page = response.xpath('//a[contains(., "- Next>>")]/@href').get()
        # If it exists and there is a next page enter if statement
        if next_page is not None:
            # Go to next page
            yield response.follow(next_page, self.parse)

Partial CSV file

URL,IP Address,User Agent
http://www.starcitygames.com/catalog/category/Vanguard%20Oversized,199.89.192.78,"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-en) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4"
http://www.starcitygames.com/catalog/category/Visions,,
http://www.starcitygames.com/catalog/category/Zendikar,,
http://www.starcitygames.com/catalog/category/Duel%20Decks%20Blessed%20vs%20Cursed,,

Answer 1

You are doing everything in right way. The thing that will help you to keep your links in order is priority in request. Check the documentation here: https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.http.Request

priority (int) – the priority of this request (defaults to 0). The priority is used by the scheduler to define the order used to process requests. Requests with a higher priority value will execute earlier. Negative values are allowed in order to indicate relatively low-priority.

So in your start_requests you can set priority for example equal to int number from len(requests) to 1. And keep this priority for all the children in parse function.

This will help you to run all links from the first url with the highest priority equal to len(requests) , then all links from the second link with priority len(requests) - 1 and etc.