How to list of more than 1000 records from Google

Question

This is the continuation of original question in this link.

Through the below code, I can able to fetch 1000 records but I have in total 6500++ records in my drive. Searching google but unable to find out the correct solution.

As per reference, the description value of Parameter "pageSize" is "The maximum number of files to return per page. Acceptable values are 1 to 1000, inclusive. (Default: 100)".

So it means, we can get only 1000 records or if possible, then what's the way. Also, I don't understand about the Parameter "pageToken", what's the usage of 'nextPageToken' value in realtime.

Code: (https://developers.google.com/drive/v3/web/quickstart/dotnet)

namespace gDrive
{
    class Program
    {
        static string[] Scopes = { DriveService.Scope.DriveReadonly };
        static string ApplicationName = "Drive API .NET Quickstart";

    static void Main(string[] args)
    {
        UserCredential credential;
        gDriveTableAdapter gDrive = new gDriveTableAdapter();

        using (var stream =
            new FileStream("client_secret.json", FileMode.Open, FileAccess.Read))
        {
            string credPath = System.Environment.GetFolderPath(
                System.Environment.SpecialFolder.Personal);
            credPath = Path.Combine(credPath, ".credentials/drive-dotnet-quickstart.json");

            credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
                GoogleClientSecrets.Load(stream).Secrets,
                Scopes,
                "user",
                CancellationToken.None,
                new FileDataStore(credPath, true)).Result;
            //Console.WriteLine("Credential file saved to: " + credPath);
        }

        // Create Drive API service.
        var service = new DriveService(new BaseClientService.Initializer()
        {
            HttpClientInitializer = credential,
            ApplicationName = ApplicationName,
        });

        // Define parameters of request.
        FilesResource.ListRequest listRequest = service.Files.List();
        listRequest.PageSize = 1000;
        listRequest.Fields = "nextPageToken, files(webViewLink, name)";

        // List files.
        IList<Google.Apis.Drive.v3.Data.File> files = listRequest.Execute()
            .Files;
        Console.WriteLine("Processing...\n");
        if (files != null && files.Count > 0)
        {
            foreach (var file in files)
            {
                gDrive.InsertQuery(file.Name, file.WebViewLink);
            }
            Console.WriteLine(files.Count + " records fetched.");
        }
        else
        {
            Console.WriteLine("No files found.");
        }
        Console.Read();
    }
  }
}

Answer 1

Here is an improved (IMO) version of EDR's great answer that does not repeat the listRequest code:

List<Google.Apis.Drive.v3.Data.File> allFiles = new List<Google.Apis.Drive.v3.Data.File>();

Google.Apis.Drive.v3.Data.FileList result = null;
while (true)
{
    if (result != null && string.IsNullOrWhiteSpace(result.NextPageToken))
        break;

    FilesResource.ListRequest listRequest = service.Files.List();
    listRequest.PageSize = 1000;
    listRequest.Fields = "nextPageToken, files(id, name)";
    if (result != null)
        listRequest.PageToken = result.NextPageToken;

    result = listRequest.Execute();
    allFiles.AddRange(result.Files);
}

Answer 2

Achieved thru Google Script. Thanks to mesgarpour (link).

var folderId = "Enter the Folder Id here";

// Main function 1: List all folders, & write into the current sheet.
function listFolders(){
  getFolderTree(folderId, false);
};

// Main function 2: List all files & folders, & write into the current sheet.
function listAll(){
  getFolderTree(folderId, true); 
};

// =================
// Get Folder Tree
function getFolderTree(folderId, listAll) {
  try {
    // If you want to search from the top (root) folder
    var parentFolder = DriveApp.getRootFolder();

    // If you want a tree of any sub folder
    //var parentFolder = DriveApp.getFolderById(folderId);

    // Initialise the sheet
    var file, data, sheet = SpreadsheetApp.getActiveSheet();
    sheet.clear();
    sheet.appendRow(["Full Path", "Name", "Date", "URL", "Last Updated", "Description", "Size"]);

    // Get files and folders
    getChildFolders(parentFolder.getName(), parentFolder, data, sheet, listAll);

  } catch (e) {
    Logger.log(e.toString());
  }
};

// Get the list of files and folders and their metadata in recursive mode
function getChildFolders(parentName, parent, data, sheet, listAll) {
  var childFolders = parent.getFolders();

  // List folders inside the folder
  while (childFolders.hasNext()) {
    var childFolder = childFolders.next();
    // Logger.log("Folder Name: " + childFolder.getName());
    data = [ 
      parentName + "/" + childFolder.getName(),
      childFolder.getName(),
      childFolder.getDateCreated(),
      childFolder.getUrl(),
      childFolder.getLastUpdated(),
      childFolder.getDescription(),
      childFolder.getSize()
    ];
    // Write
    sheet.appendRow(data);

    // List files inside the folder
    var files = childFolder.getFiles();
    while (listAll & files.hasNext()) {
      var childFile = files.next();
      // Logger.log("File Name: " + childFile.getName());
      data = [ 
        parentName + "/" + childFolder.getName() + "/" + childFile.getName(),
        childFile.getName(),
        childFile.getDateCreated(),
        childFile.getUrl(),
        childFile.getLastUpdated(),
        childFile.getDescription(),
        childFile.getSize()
      ];
      // Write
      sheet.appendRow(data);
    }

    // Recursive call of the subfolder
    getChildFolders(parentName + "/" + childFolder.getName(), childFolder, data, sheet, listAll);  
  }
};

Answer 3

You can make several request and get data from each page. The items are dived on pages by security reason

Answer 4

If you need to download paginating you can achieve it via C# SDK too. The trick is in keeping track of the listRequest.Execute() return value, that contains several variables and between them the NextPageToken. This part is "hidden" in the standard google example.

FilesResource.ListRequest listRequest = _service.Files.List();
listRequest.PageSize = 100;
listRequest.Fields = "nextPageToken, files(id, name)";

// List files.
var result =  listRequest.Execute();
IList<Google.Apis.Drive.v3.Data.File> files =result.Files;
Console.WriteLine("Files:");
while (files!=null && files.Count > 0)
{
    foreach (var file in files)
    {
        Console.WriteLine("{0} ({1})", file.Name, file.Id);
    }
    if (!string.IsNullOrWhiteSpace(result.NextPageToken))
    {
        listRequest = _service.Files.List();
        listRequest.PageToken = result.NextPageToken;
        listRequest.PageSize = 100;
        listRequest.Fields = "nextPageToken, files(id, name)";
        result = listRequest.Execute();
        files = result.Files;
    }
}