MemoryError - how to download large file via Googl

问题:

I'm running out of memory when downloading big file from my Google Drive. I assume that tmp = content.read(1024) does not work, but how to fix it? Thank you.

def download_file(service, file_id):
  drive_file = service.files().get(fileId=file_id).execute()
  download_url = drive_file.get('downloadUrl')
  title = drive_file.get('title')
  originalFilename = drive_file.get('originalFilename')
  if download_url:
    resp, content = service._http.request(download_url)
    if resp.status == 200:
      file = 'tmp.mp4'
      with open(file, 'wb') as f:
          while True:
              tmp = content.read(1024)
              if not tmp:
                  break
              f.write(tmp)
      return title, file
    else:
      print 'An error occurred: %s' % resp
      return None
  else:
    return None

回答1:

The right solution would be to implement partial download. With this process, you will request chunks of different length of a file until it is completely downloaded. Here is a reference of the process: https://developers.google.com/drive/web/manage-downloads#partial_download.

回答2:

Here is complete working code.

def partial(total_byte_len, part_size_limit):
    s = []
    for p in range(0, total_byte_len, part_size_limit):
        last = min(total_byte_len - 1, p + part_size_limit - 1)
        s.append([p, last])
    return s



def GD_download_file(service, file_id):
  drive_file = service.files().get(fileId=file_id).execute()
  download_url = drive_file.get('downloadUrl')
  total_size = int(drive_file.get('fileSize'))
  s = partial(total_size, 100000000) // I'm downloading BIG files, so 100M chunk size is fine for me
  title = drive_file.get('title')
  originalFilename = drive_file.get('originalFilename')
  filename = '/some_path/' + originalFilename
  if download_url:
      with open(filename, 'wb') as file:
        for bytes in s:
          headers = {"Range" : 'bytes=%s-%s' % (bytes[0], bytes[1])}
          resp, content = service._http.request(download_url, headers=headers)
          if resp.status == 206 :
                file.write(content)
                file.flush()
          else:
            print 'An error occurred: %s' % resp
            return None
      return title, filename
  else:
    return None