在数据库轨道中存储多个数组

Question

我正在从UCAS网站中提取数据，并将其存储在一系列数组中，如下所示。 我想将这些数组存储在数据库中，但不确定如何执行此操作。 有人可以帮忙吗？ 尽管我计划稍后将其更改为postgres，但在sqlite3中还是理想的选择。

class PagesController < ApplicationController


require 'mechanize'

class Scraper



    def process_unis
        @mechanize = Mechanize.new

        @unis_array = []
        page_num = 1
        next_page_link = true
        # Load initial search page
        @page = @mechanize.get('http://search.ucas.com/search/providers?CountryCode=3&RegionCode=&Lat=&Lng=&Feather=&Vac=2&Query=&ProviderQuery=&AcpId=&Location=scotland&IsFeatherProcessed=True&SubjectCode=&AvailableIn=2016')
        while  next_page_link
            puts "- Scraping uni page: #{page_num}" 
            # Loops through all unis on page
            @page.search('li.result').each do |uni|
                # Sets hash to store all uni data
                uni_info = {}
                uni_name = uni.search('h3').text
                uni_info[:name] = uni_name
                uni_more_courses_link = uni.search('.morecourseslink a').first
                if uni_more_courses_link
                    # If there is a link to a courses page for the uni go to that
                    uni_info[:courses] = process_courses_page(@mechanize.get(uni_more_courses_link["href"])) 
                else
                    # Otherwise process all the courses on the current page for that uni
                    uni_info[:courses] = process_inline_courses(uni)
                end
                # Add complete uni info to array. This may also be a good place to inset the uni in the database
                @unis_array.push(uni_info)
            end
            # Checks if there is a next page and navigates if so
            next_page_link = @page.search('.pager a[text()=">"]').first
            if(next_page_link) 
                @page = @mechanize.get(next_page_link["href"])
                page_num += 1
            end
        end
    end

    # Just a function to print what has been added. Can be removed if unneeded
    def print_course_info(course_info)
        puts "--- Adding course: #{course_info[:name]}" 
        puts "---- Duration: #{course_info[:duration]}"
        puts "---- Qualification: #{course_info[:qualification]}"

    end



    def print_details_info(details_info)

        puts "---- link: #{details_info[:url]}"

    end


    def print_entry_info(entry_info)

        puts "---- Requirements: #{entry_info[:req]}"
        puts ""
    end 


    # Removes excess spaces and new line characters from duration and qual text
    def clean_text(text)
        return text.gsub!('  ', '').gsub!("\n", '').gsub!("\r", '')
    end

    def clean_reqs(text)
        return text.gsub!('  ', '').gsub!("\n", '').gsub!("\r", '')
    end



    def process_inline_courses(uni)
        uni_name = uni.search('h3').text
        puts "-- Scraping #{uni_name} courses"
        courses_array = []

        # Loops through all courses, saves their info into a hash and pushes it to an array
        uni.search('.courseresult').each do |course|
            course_info = {}
            course_info[:name]          = course.search('.title').text
            course_info[:duration]      = course.search('.durationValue').text
            course_info[:qualification] = course.search('.outcomequalValue').text

            courses_array.push(course_info)
            print_course_info(course_info)
        end

        puts "--- Scraped #{courses_array.length} courses"
        return courses_array
    end





    def process_courses_page(course_page)
        courses_array = []
        page_num = 1
        next_page_link = true   
        details_link = true 
        uni_name = course_page.search('.providerinfo h3').text
        # Loops through all course pages for uni
        while next_page_link


            puts "-- Scraping #{uni_name} courses page: #{page_num}"
            # Loops through all courses and adds their info into a hash
            course_page.search('ol.resultscontainer li').each do |course|
                course_info = {}

                # Removes excess html which was interferring with text
                course.search(
                    '.courseinfoduration span, 
                     .courseinfoduration br, 
                     .courseinfooutcome span, 
                     .courseinfooutcome br').remove
                # Sets all data in hash
                course_info[:name]          = course.search('.courseTitle').text
                course_info[:duration]      = clean_text(course.search('.courseinfoduration').text)
                course_info[:qualification] = clean_text(course.search('.courseinfooutcome').text)
                # Pushes course hash to course array
                courses_array.push(course_info)
                print_course_info(course_info)


                details_link = course.search('div.coursenamearea a').first
                    if details_link
                     # If there is a link to a courses page for the uni go to that
                       course_info[:detail] = process_course_details(@mechanize.get(details_link["href"])) 

                    end
             end
                     # Checks if there is a next page and navigates to it if so
          next_page_link = course_page.search('.pager a[text()=">"]').first        
            if(next_page_link) 
                course_page = @mechanize.get(next_page_link["href"])
                page_num += 1
            end
        end
end


    def process_course_details(course_details)
        details_array =[]
        details_link = true 
        entry_link = true

                details_info = {}
                # Sets all data in hash
                details_info[:url]          = course_details.search('div.coursedetails_programmeurl a')

                details_array.push(details_info)
                print_details_info(details_info)


             entry_link = course_details.search('ul.details_tabs a').first
             if entry_link
                 details_info[:entry] = process_entry(@mechanize.get(entry_link["href"]))
             end 

     end


    def process_entry(entry_req)
        entry_array = []
        entry_link = true 

                entry_info = {}
                # Sets all data in hash
                entry_info[:req]          = entry_req.search('li.qual-element.qual_range').text.strip

                entry_array.push(entry_info)
                print_entry_info(entry_info)
    end



scraper = Scraper.new
scraper.process_unis

end


end 

Answer 1

您在此处粘贴了很多代码，以至于无法轻松阅读和理解。 话虽这么说，您可以采用几种方法将数组保存到数据库中。

1. 序列化它，然后将其存储为字符串。 然后在要阅读时反序列化它。 您可以为此使用json或yaml 。
2. 将数据分别保存在阵列中。 这可能是一种更明智的方法，因为您不必弄乱序列化。 因此，例如，如果您有一个数组阵列fruits = ["apple", "orange"]和meats = ["beef", "lamb"] ，则可以在数据库中创建fruits and meats表，并meats name列，然后创建适当的条目。

Answer 2

在模型中，您很可能希望使用序列化方法。 这将允许您将数据序列化为数组，哈希，json，HashieMash等。

http://api.rubyonrails.org/classes/ActiveRecord/AttributeMethods/Serialization/ClassMethods.html

Answer 3

您可以很容易地将其存储为JSON形式的字段。 这仅在不需要查询数组中的数据时才有用。 如果确实有此需要，则需要创建一个表来表示这些数组。

如果您决定将其存储为JSON，则它是一个非常简单的接口：

my_array = [1,2,3,4]
my_array_as_json = my_array.to_json

以后你可以用

my_array_again = JSON.parse(my_array_as_json)