My script are not going to next page for scraping

2020-05-02 02:50发布

站内问答 / Python
1677 1 4

I wrote a code for web scraping everything is ok except next page activity. When I run my code to srape data from the website it just sraping first page not moving forward to scrape other pages data. Actually I'm new to web scraping using python so please guide me. could you please fix my code. have a look at my code and help me please, Thank you

here is my code:

import requests
from bs4 import BeautifulSoup
#import pandas as pd
#import pandas as pd
import csv

def get_page(url):
    response = requests.get(url)
    if not response.ok:
        print('server responded:', response.status_code)
    else:
        soup = BeautifulSoup(response.text, 'html.parser') # 1. html , 2. parser
    return soup

def get_detail_page(soup):

     try:
        title = (soup.find('h1',class_="cdm_style",id=False).text)
     except:
         title = 'Empty Title'
     try:
         collection = (soup.find('td',id="metadata_collec").find('a').text)
     except:
         collection = "Empty Collection"
     try:
         author = (soup.find('td',id="metadata_creato").text)
     except:
         author = "Empty Author"
     try:
         abstract = (soup.find('td',id="metadata_descri").text)
     except:
         abstract = "Empty Abstract"
     try:
         keywords = (soup.find('td',id="metadata_keywor").text)
     except:
         keywords = "Empty Keywords"
     try:
         publishers = (soup.find('td',id="metadata_publis").text)
     except:
         publishers = "Empty Publishers"
     try:
         date_original = (soup.find('td',id="metadata_contri").text)
     except:
         date_original = "Empty Date original"
     try:
        date_digital = (soup.find('td',id="metadata_date").text)
     except:
        date_digital = "Empty Date digital"
     try:
        formatt = (soup.find('td',id="metadata_source").text)
     except:
        formatt = "Empty Format"
     try:
        release_statement = (soup.find('td',id="metadata_rights").text)
     except:
        release_statement = "Empty Realease Statement"
     try:
        library = (soup.find('td',id="metadata_librar").text)
     except:
        library = "Empty Library"
     try:
        date_created = (soup.find('td',id="metadata_dmcreated").text)
     except:
        date_created = "Empty date Created"
     data = {
         'Title'        : title.strip(),
         'Collection'   : collection.strip(),
         'Author'       : author.strip(),
         'Abstract'     : abstract.strip(),
         'Keywords'     : keywords.strip(),
         'Publishers'   : publishers.strip(),
         'Date_original': date_original.strip(),
         'Date_digital' : date_digital.strip(),
         'Format'       : formatt.strip(),
         'Release-st'   : release_statement.strip(),
         'Library'      : library.strip(),
         'Date_created' : date_created.strip()


     }
     return data
def get_index_data(soup):
    try:
        titles_link = soup.find_all('a',class_="body_link_11")
    except:
        titles_link = []
    else:
        titles_link_output = []
        for link in titles_link:
            try:
                item_id = link.attrs.get('item_id', None) #All titles with valid links will have an item_id
                if item_id:
                    titles_link_output.append("{}{}".format("http://cgsc.cdmhost.com",link.attrs.get('href', None)))
            except:
                continue
    return titles_link_output
def write_csv(data,url):
    with open('11_to_55.csv','a') as csvfile:
        writer = csv.writer(csvfile)
        row = [data['Title'], data['Collection'], data['Author'],
        data['Abstract'], data['Keywords'], data['Publishers'], data['Date_original'],
        data['Date_digital'], data['Format'], data['Release-st'], data['Library'],
        data['Date_created'], url]
        writer.writerow(row)
def main():
    #url = "http://cgsc.cdmhost.com/cdm/singleitem/collection/p4013coll8/id/2653/rec/1"
    #get_page(url)
    for x in range(1,4):
        mainurl = ("http://cgsc.cdmhost.com/cdm/search/collection/p4013coll8/searchterm/1/field/all/mode/all/conn/and/order/nosort/page/")
        print(x)
        url = (mainurl + str(x))
        products = get_index_data(get_page(url))
        for product in products:
            data1 = get_detail_page(get_page(product))
            write_csv(data1,product)
    #write_csv(data,url)


if __name__ == '__main__':
    main()

1条回答
趁早两清
2楼-- · 2020-05-02 03:48

I started trying to find out why it wasn't loading the next page correctly, but before I found the answer I found another way to get the data you are looking for. On the page there is an option to change how many results you want to return. I changed this to 10000, and now all items from the collection load on one page.

If this isn't what you want, and instead still want to sove the page change problem let me know and I'll have another look.

cgsc.cdmhost.com/cdm/search/collection/p4013coll8/searchterm/1/field/all/mode/all/conn/and/display/10000/order/nosort/ad/asc

I tested loading the index page but have not tested the detailed pages. I didn't want to download the entire collection.

Here are some changes I made and some recommendations.

  1. Removed the need for looping index pages. Page now returns all items.
  2. Be specific with what exception you are catching. In this case its AttributeError
  3. Add a user-agent to your request, many websites will block requests that do not have it

Good luck!

import requests
from bs4 import BeautifulSoup
#import pandas as pd
#import pandas as pd
import csv

def get_page(url):
    response = requests.get(url, headers={'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.37"})
    if not response.ok:
        print('server responded:', response.status_code)
    else:
        soup = BeautifulSoup(response.text, 'html.parser') # 1. html , 2. parser
    return soup

def get_detail_page(soup):

    # Be specific with your exception capturing. 
    try:
        title = (soup.find('h1',class_="cdm_style",id=False).text)
    except AttributeError:
        title = 'Empty Title'
    try:
        collection = (soup.find('td',id="metadata_collec").find('a').text)
    except AttributeError:
        collection = "Empty Collection"
    try:
        author = (soup.find('td',id="metadata_creato").text)
    except AttributeError:
        author = "Empty Author"
    try:
        abstract = (soup.find('td',id="metadata_descri").text)
    except AttributeError:
        abstract = "Empty Abstract"
    try:
        keywords = (soup.find('td',id="metadata_keywor").text)
    except AttributeError:
        keywords = "Empty Keywords"
    try:
        publishers = (soup.find('td',id="metadata_publis").text)
    except AttributeError:
        publishers = "Empty Publishers"
    try:
        date_original = (soup.find('td',id="metadata_contri").text)
    except AttributeError:
        date_original = "Empty Date original"
    try:
        date_digital = (soup.find('td',id="metadata_date").text)
    except AttributeError:
        date_digital = "Empty Date digital"
    try:
        formatt = (soup.find('td',id="metadata_source").text)
    except AttributeError:
        formatt = "Empty Format"
    try:
        release_statement = (soup.find('td',id="metadata_rights").text)
    except AttributeError:
        release_statement = "Empty Realease Statement"
    try:
        library = (soup.find('td',id="metadata_librar").text)
    except AttributeError:
        library = "Empty Library"
    try:
        date_created = (soup.find('td',id="metadata_dmcreated").text)
    except AttributeError:
        date_created = "Empty date Created"
    data = {
    'Title'        : title.strip(),
    'Collection'   : collection.strip(),
    'Author'       : author.strip(),
    'Abstract'     : abstract.strip(),
    'Keywords'     : keywords.strip(),
    'Publishers'   : publishers.strip(),
    'Date_original': date_original.strip(),
    'Date_digital' : date_digital.strip(),
    'Format'       : formatt.strip(),
    'Release-st'   : release_statement.strip(),
    'Library'      : library.strip(),
    'Date_created' : date_created.strip()
    }
    return data

def get_index_data(soup):
    try:
        titles_link = soup.find_all('a',class_="body_link_11")
    except:
        titles_link = []
    else:
        titles_link_output = []
        for link in titles_link:
            try:
                item_id = link.attrs.get('item_id', None) #All titles with valid links will have an item_id
                if item_id:
                    titles_link_output.append("{}{}".format("http://cgsc.cdmhost.com",link.attrs.get('href', None)))
            except:
                continue
    return titles_link_output

def write_csv(data,url):
    with open('11_to_55.csv','a') as csvfile:
        writer = csv.writer(csvfile)
        row = [data['Title'], data['Collection'], data['Author'],
        data['Abstract'], data['Keywords'], data['Publishers'], data['Date_original'],
        data['Date_digital'], data['Format'], data['Release-st'], data['Library'],
        data['Date_created'], url]
        writer.writerow(row)

def main():
    main_url = ("http://cgsc.cdmhost.com/cdm/search/collection/p4013coll8/searchterm/1/field/all/mode/all/conn/and/display/10000/order/nosort/ad/asc")
    products = get_index_data(get_page(main_url))
    print(products)
#     for product in products:
#         data1 = get_detail_page(get_page(product))
#         write_csv(data1,product)
#     write_csv(data,url)


if __name__ == '__main__':
    main()
查看更多
0人赞 添加讨论(0) 举报
登录 后发表回答
相关问题
查看全部
相关文章
查看全部
收藏的人(4)