通過python處理數據庫會產生錯誤的結果

Question

這種將數據放入數據庫的方法有什么問題嗎？

程序的部分是這個。 它將在LOOP中執行多次。

    db= MySQLdb.connect("localhost","root","ahmed","practice")
    cursor=db.cursor()

    #checking phase to stop scrapping
    sql = """SELECT Short_link FROM Properties WHERE Short_link=%s"""
    print rows
    rows = cursor.execute(sql,(link_result))
    print rows
    if rows>=1:
        print "Already present - The program is terminating"
        sys.exit()
    else:
        query="""INSERT INTO Properties (Sale_Rent, Type, Title,Price, PricePerSqrFt, Bedroom,Agency_Fee, Bathroom, Size,ZonedFor, Freehold, Prop_ref,Furnished_status,Rent_payment,Building_info,Amenities,Trade_name,Licence, RERA_ID,Phone_info,Short_link) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
        cursor.execute(query,(Sale_Rent_result,Type_result, title_result, price_result, Pricepersq_result, bedroom_result, agencyfee_result, bathroom_result, size_result,Zoned_for_result, Freehold_result, propertyref_result, furnished_result, rent_is_paid_result, building_result, Amenities_result, tradename_result, licencenum_result, reraid_result, phone_result, link_result))


    db.commit()
    cursor.close()
    db.close()

當我運行該程序時，它運行良好。 但是，當我使用下面的腳本以並行方式運行該程序的5個副本時，其中一個程序從一開始就具有row = 1（而數據庫為空，並且row不應為1）。

all.sh
python python1.py &
python python2.py &
python python3.py &
python python4.py &
python python5.py &

每個link_result's結果在提取后都是唯一的，並且一旦輸入數據庫，它就會創建一列唯一鏈接。

假設數據庫為空，並且我一起運行這些文件，則row永遠不應該等於1。當我再次運行程序時， row應等於1。 當我再次運行它們時，它將輸入新數據並在link_result與已經存在的鏈接結果沖突時停止（由sql的select方法檢查）

我假設數據庫的打開和關閉存在一些問題，即數據庫為空時row變量變為1。 我無法理解這種行為。

這是整個程序供參考

#!/usr/bin/python
import urllib
from bs4 import BeautifulSoup

import MySQLdb
import re
import pdb
import sys





def getting_urls_of_all_pages(): 

    i=1
    while i<=40: #40 is the total number of main pages
        url_rent_flat='http://dubai.dubizzle.com/property-for-rent/residential/apartmentflat/?page='+str(i) #url of the main page (iterating to 40)


        link=[]
        htmlfile=urllib.urlopen(url_rent_flat).read()
        soup=BeautifulSoup(htmlfile)

        link=soup.find_all('a',xtclib=re.compile("listing_list_\d+_title_link"),href=True) #stores all the links (25) links of the page

        """
        Part 2: passing each property url to process for data extraction
        """

        for a in link:
            every_property_in_a_page_data_extraction(a['href']) 

        i+=1


def every_property_in_a_page_data_extraction(url):






    title_result=""
    price_result=""
    bedroom_result="" 
    agencyfee_result="" 
    bathroom_result="" 
    size_result=""
    propertyref_result=""
    furnished_result=""
    rent_is_paid_result=""
    building_result=""
    Amenities_result=""
    tradename_result=""
    licencenum_result=""
    reraid_result=""
    phone_result=""
    link_result=""
    Zoned_for_result=""
    Freehold_result=""
    Pricepersq_result=""
    Type_result="Apartment"
    Sale_Rent_result="Rent"
    rows=0




    """
    Part1: Extracting data
    """

    htmlfile=urllib.urlopen(url).read()
    soup=BeautifulSoup(htmlfile)

    """
    Part2: Extracting the components
    """


    # Sale/Rent
    print "Sale/Rent: ", Sale_Rent_result

    # Type of property
    print "Type of property: ", Type_result

    #title
    try:
        title= soup.find('span',{'id':'listing-title-wrap'})
        title_result= str(title.get_text().strip().encode("utf-8"))
        print "Title: ",title_result
    except StandardError as e:
        title_result="Error was {0}".format(e)
        print title_result

    #price
    try:
        price = soup.find('span',{'id':'actualprice'})
        price_result= str(price.get_text())
        print "Price: ",price_result
    except StandardError as e:
        price_result="Error was {0}".format(e)
        print price_result

    #Agency Fee, Bedroom, Bathroom, Size
    spans_ABBS= []
    for a in soup.select(".important-fields li span"):
        spans_ABBS.append(a.text.strip())

    strongs_ABBS=[]
    for a in soup.select(".important-fields li strong"):
        strongs_ABBS.append(a.text.strip())


    for name, value in zip(spans_ABBS, strongs_ABBS):
        if name=="Agency Fees:":
            try:
                agencyfee_result= str(value)
                print "Agency Fees: ", agencyfee_result
            except StandardError as e:
                agencyfee_result="Error was {0}".format(e)
                print agencyfee_result

        elif name=="Bedrooms:":
            try:
                bedroom_result= str(value)
                print "Number of Bedrooms: ",bedroom_result
            except StandardError as e:
                bedroom_result="Error was {0}".format(e)
                print bedroom_result

        elif name=="Bathrooms:":
            try:
                bathroom_result= str(value)
                print "Number of Bathrooms: ", bathroom_result
            except StandardError as e:
                bathroom_result="Error was {0}".format(e)
                print bathroom_result

        elif name=="Size:":
            try:
                size_result= str(value)
                print "Size of the property: ",size_result
            except StandardError as e:
                size_result="Error was {0}".format(e)
                print size_result

        elif name=="Zoned For:":
            try:
                Zoned_for_result= str(value)
                print "Zoned For:",Zoned_for_result
            except StandardError as e:
                Zoned_for_result="Error was {0}".format(e)
                print Zoned_for_result

        elif name=="Freehold:":
            try:
                Freehold_result= str(value)
                print "Freehold: ",Freehold_result
            except StandardError as e:
                Freehold_result="Error was {0}".format(e)
                print Freehold_result

        elif name=="Price / SqFt:":
            try:
                Pricepersq_result= str(value)
                print "Price Per Sqft: ",Pricepersq_result
            except StandardError as e:
                Pricepersq_result="Error was {0}".format(e)
                print Pricepersq_result

    #Property Reference, Furnished, Listed By, Rent Is Paid, Building, Amenities: 
    spans_others=[]
    for a in soup.select("#listing-details-list li span"):
            spans_others.append(a.text.strip())

    strongs_others=[]
    for a in soup.select("#listing-details-list li strong"):
        strongs_others.append(a.text.strip())



    for name, value in zip(spans_others, strongs_others):
        if name=="Listed by:":
            break

        elif name=="Property Reference:":
            try:
                propertyref_result=str(value.strip())
                print "Property reference in Dubizel: ",propertyref_result
            except StandardError as e:
                propertyref_result="Error was {0}".format(e)
                print propertyref_result

        elif name=="Furnished:":
            try:
                furnished_result=str(value.strip())
                print "Furnished status: ",furnished_result
            except StandardError as e:
                furnished_result="Error was {0}".format(e)
                print furnished_result


        elif name=="Rent Is Paid:":
            try:
                rent_is_paid_result=str(value.strip())
                print "Rent payment: ",rent_is_paid_result
            except StandardError as e:
                rent_is_paid_result="Error was {0}".format(e)
                print rent_is_paid_result

        elif name=="Building:":
            try:
                building_result=str(value.strip())
                print "Building info: ",building_result
            except StandardError as e:
                building_result="Error was {0}".format(e)
                print building_result
        elif name=="Amenities:":
            try:
                for a in value.split(","):
                    Amenities_result+=a.strip()+","
                print Amenities_result
            except StandardError as e:
                Amenities_result="Error was {0}".format(e)
                print Amenities_result



    #Agents info --> TTrade Name, DED Licence Number, RERA Registration Number
    spans_broker=[]
    for a in soup.select("#broker-details li span"):
            spans_broker.append(a.text.strip())

    strongs_broker=[]
    for a in soup.select("#broker-details  li strong"):
        strongs_broker.append(a.text.strip())


    for name, value in zip(spans_broker, strongs_broker):
        if name=="Trade Name:":
            try:
                tradename_result=str(value.strip())
                print "Trade name: ",tradename_result
            except StandardError as e:
                tradename_result="Error was {0}".format(e)
                print tradename_result

        elif name=="DED Licence Number:":
            try:
                licencenum_result=str(value.strip())
                print "Licence #: ",licencenum_result
            except StandardError as e:
                licencenum_result="Error was {0}".format(e)
                print licencenum_result

        elif name=="RERA Registration Number:":
            try:
                reraid_result=str(value.strip())
                print "RERA ID #: ",reraid_result
            except StandardError as e:
                reraid_result="Error was {0}".format(e)
                print reraid_result


    # phone num
    try:
        phone=soup.find_all("div", "phone-content")
        for a in phone:
            phone_result= str(a.get_text().strip().encode("utf-8"))
        print "Phone information:", phone_result
    except StandardError as e:
        phone_result="Error was {0}".format(e)
        print phone_result



    #link
    try:
        link = soup.find('input',{'id':'short-link-input'})
        link_result= str(link.get('value'))
        print "Short Reference link: ", link_result
    except StandardError as e:
        link_result="Error was {0}".format(e)
        print link_result





    """
    Connecting to Database and putting data into in
    """

    db= MySQLdb.connect("localhost","root","ahmed","practice")
    cursor=db.cursor()

    #checking phase to stop scrapping
    sql = """SELECT Short_link FROM Properties WHERE Short_link=%s"""
    print rows
    rows = cursor.execute(sql,(link_result))
    print rows
    if rows>=1:
        print "Already present - The program is terminating"
        sys.exit()
    else:
        query="""INSERT INTO Properties (Sale_Rent, Type, Title,Price, PricePerSqrFt, Bedroom,Agency_Fee, Bathroom, Size,ZonedFor, Freehold, Prop_ref,Furnished_status,Rent_payment,Building_info,Amenities,Trade_name,Licence, RERA_ID,Phone_info,Short_link) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
        cursor.execute(query,(Sale_Rent_result,Type_result, title_result, price_result, Pricepersq_result, bedroom_result, agencyfee_result, bathroom_result, size_result,Zoned_for_result, Freehold_result, propertyref_result, furnished_result, rent_is_paid_result, building_result, Amenities_result, tradename_result, licencenum_result, reraid_result, phone_result, link_result))


    db.commit()
    cursor.close()
    db.close()



    #-----------------------------------------------------------





        getting_urls_of_all_pages()

Answer 1

您沒有將link_result正確傳遞給execute()方法：

rows = cursor.execute(sql,(link_result))

括號是可選的，Python將其視為：

rows = cursor.execute(sql, link_result)

因此查詢中僅使用link_result的第一個字符 （其他數據庫會告訴您傳遞了太多參數）。

您需要使用逗號使其成為適當的元組：

rows = cursor.execute(sql, (link_result,))