Fetching Data Recursively with Python APIs

Here is the standard pattern for retrieving list-type data via an API using Python.

def getAllContents(page_num=1):

    contents = []
    page_num = 1
    endpoint = "https://example.com/content?page_num=" + str(page_num)
    headers = {'Authorization': "Bearer xxxxxxxxxxxxxxxxxxxxxxxxxxxx", "Content-Type": "application/json"}
    r = requests.get(endpoint, headers=headers)
    res = r.json()
    contents = contents + res["data"]
    
    has_next = False
    if len(res["data"]) == 50: # 50 is the maximum number of contents per page
        has_next = True
    
    while has_next:
        page_num += 1
        endpoint = "https://example.com/content?page_num=" + str(page_num)
        r = requests.post(endpoint, headers=headers)
        res = r.json()
        contents = contents + res["data"]
        page_num += 1
        time.sleep(1)
        print(res)

    return contents



all_Contents = getAllContents()