Python写入抓取数据的json文件

Question

I wrote a web scraping script and it is working great. I am trying to write the scraped data to json file but i failed.

this is my snippet:

def scrape_post_info(url):
    content = get_page_content(url)
    title, description, post_url = get_post_details(content, url)
    job_dict = {}
    job_dict['title'] = title
    job_dict['Description'] = description
    job_dict['url'] = post_url

    json_job = json.dumps(job_dict)
    with open('data.json', 'a') as f:
        json.dump(json_job, f)

if __name__ == '__main__':
    urls = ['url1', 'url2', 'url3', 'url4']
    for url in urls:
        scrape_post_info(url)

ignore two function i called inside the function, problem not with them

My problem only is writing to json.

Currently i am getting the scraped data like this below and there are wrong format

data.json are below:

{
    "title": "this is title",
    "Description": " Fendi is an Italian luxury labelarin. ",
    "url": "https:/~"
}

{
    "title": " - Furrocious Elegant Style", 
    "Description": " the Italian luxare vast. ", 
    "url": "https://www.s"
}
    
{
    "title": "Rome, Fountains and Fendi Sunglasses",
    "Description": " Fendi started off as a store. ",
    "url": "https://www.~"
}
    
{
    "title": "Tipsnglasses",
    "Description": "Whether irregular orn season.", 
    "url": "https://www.sooic"
}

but it should be like these:

[
{
    "title": "this is title",
    "Description": " Fendi is an Italian luxury labelarin. ",
    "url": "https:/~"
},

{
    "title": " - Furrocious Elegant Style", 
    "Description": " the Italian luxare vast. ", 
    "url": "https://www.s"
},
    
{
    "title": "Rome, Fountains and Fendi Sunglasses",
    "Description": " Fendi started off as a store. ",
    "url": "https://www.~"
},
    
{
    "title": "Tipsnglasses",
    "Description": "Whether irregular orn season.", 
    "url": "https://www.sooic"
},

]

I am not getting exactly why i am not getting data in json file in proper formate..

Can anyone help me in this?

Answer 1

You can try this code to solve your problem. you will get exact file as you expected above, following is the code:

import json
def scrape_post_info(url, f):
    content = get_page_content(url)
    title, description, post_url = get_post_details(content, url)
    job_dict = {}
    job_dict['title'] = title
    job_dict['Description'] = description
    job_dict['url'] = post_url 

    json_job = json.dumps(job_dict)
    f.seek(0)
    txt = f.readline()
    if txt.endswith("}"):
        f.write(",")
    f.write(json_job)

if __name__ == '__main__':
    urls = ['url1', 'url2', 'url3', 'url4']
    with open('data.json', 'r+') as f:
        f.write("[")
        for url in urls:
            scrape_post_info(url,f)
        f.write("]")