Blog

How to Efficiently Load Large Data Sets in a Mobile App With Data Paging

by on September 1, 2019

Data paging is the process of breaking up a larger set of objects into smaller chunks, commonly referred to as pages. Typically paging applies to the results of search queries when the server may return too many objects at once.

Since smaller pages of data can be returned to the client much faster, the user of the client application does not need to wait. As a result, the user experience of the app is significantly improved, which is the primary advantage of paging.

Backendless automatically breaks up the results of search queries into pages. The examples below demonstrate the usage of the API and various techniques for paged data retrieval.

All the examples in this article rely on the code obtained through Backendless code generator. For details see how to generate client-side code for your mobile application with Backendless. To setup your backend in order to run these examples, you should:
In another post, we showed how to load data from Backendless in the most basic API. The code in that post uses the basic find API to fetch a collection of the restaurant objects. Since Backendless automatically pages data, it will return only the first page. The default page size is 10 objects, however, that value can be changed using the following API (maximum allowed page size is 100):

    AsyncCallback<List<Restaurant>> callback = new AsyncCallback<List<Restaurant>>()
    { /* skipped for brevity */  };
    int PAGESIZE = 80;
    DataQueryBuilder dataQuery = DataQueryBuilder.create();
    dataQuery.setPageSize(PAGESIZE);
    Backendless.Data.of(Restaurant.class).find(dataQuery, callback);

    val callback = object : AsyncCallback<List<Restaurant>> { /* skipped for brevity */  }
    val PAGESIZE = 80
    val dataQuery = DataQueryBuilder.create()
    dataQuery.setPageSize(PAGESIZE)
    Backendless.Data.of(Restaurant::class.java).find(dataQuery, callback)

    NSInteger PAGESIZE = 80;
        
    DataQueryBuilder *queryBuilder = [DataQueryBuilder new];
    [queryBuilder setPageSizeWithPageSize:PAGESIZE];
        
    [[Backendless.shared.data of:[Restaurant class]] findWithQueryBuilder:queryBuilder responseHandler:^(NSArray *restaurants) {
        // skipped for brevity
    } errorHandler:^(Fault *fault) {
        NSLog(@"Error: %@", fault.message);
    }];

    let PAGESIZE = 80
            
    let queryBuilder = DataQueryBuilder()
    queryBuilder.setPageSize(pageSize: PAGESIZE)
            
    Backendless.shared.data.of(Restaurant.self).find(queryBuilder: queryBuilder, responseHandler: { restaurants in
        // skipped for brevity
    }, errorHandler: { fault in
        print("Error: \(fault.message ?? "")")
    })

    Backendless.initApp(APP_ID, API_KEY)
    
    const PAGE_SIZE = 80
    
    const dataQuery = Backendless.DataQueryBuilder.create().setPageSize(PAGE_SIZE)
    
    Backendless.Persistence.of('Restaurant').find(dataQuery)
      .then(console.log)
      .catch(console.error)
    

    int PAGESIZE = 80;
    DataQueryBuilder dataQuery = DataQueryBuilder()
      ..pageSize = PAGESIZE;
    
    Backendless.Data.of("Restaurant").find(dataQuery).then((response)
      { /* skipped for brevity */  });

    Now that we know how to fetch the first page, the question is how to load subsequent pages. Before we run any of the paging examples below, it would be worthwhile to populate the backend storage with additional objects. Without them, you would not be able to see the paging in action – all the objects would be returned in the first page. Consider the following code that adds 100 restaurant objects to the backend storage:

      private static void addRestaurants() {
         IDataStore dataStore = Backendless.Data.of(Restaurant.class);
         List restaurants = new ArrayList<>();
         for (int i = 0; i < 100; i++) { Restaurant restaurant = new Restaurant(); restaurant.setName("TastyBaaS " + i); restaurant.setCuisine("mBaaS"); restaurants.add(restaurant); } dataStore.create(restaurants, new AsyncCallback<List<String>>() {
             @Override
             public void handleResponse(List<String> ids) {
                 for (String id : ids) {
                     Log.i(TAG, "Object saved with ID - " + id);
                 }
             }
      
             @Override
             public void handleFault(BackendlessFault fault) {
                 Log.e(TAG, fault.getMessage());
             }
         });
      }

      private fun addRestaurants() {
         val dataStore = Backendless.Data.of(Restaurant::class.java)
         val restaurants = ArrayList()
         for (i in 0..99) {
             val restaurant = Restaurant()
             restaurant.name = "TastyBaaS $i"
             restaurant.cuisine = "mBaaS"
             restaurants.add(restaurant)
         }
      
         dataStore.create(restaurants, object : AsyncCallback<List<String>> {
             override fun handleResponse(ids: List<String>) {
                 for (id in ids) {
                     Log.i(TAG, "Object saved with ID - $id")
                 }
             }
      
             override fun handleFault(fault: BackendlessFault) {
                 Log.e(TAG, fault.message)
             }
         })
      }

      - (void)addRestaurants {
          DataStoreFactory *dataStore = [Backendless.shared.data of:[Restaurant class]];
          for (int i = 0; i < 300; i++) {
              Restaurant *restaurant = [Restaurant new];
              restaurant.name = [NSString stringWithFormat:@"TastyBaaS %i", i];
              restaurant.cuisine = @"mBaaS";
              [dataStore saveWithEntity:restaurant responseHandler:^(Restaurant *savedRestaurant) {
                  NSLog(@"Saved %@", savedRestaurant.name);
              } errorHandler:^(Fault *fault) {
                  NSLog(@"Error: %@", fault.message);
              }];
          }
      }

      func addRestaurants() {
          let dataStore = Backendless.shared.data.of(Restaurant.self)
          for i in 0..<300 {
              let restaurant = Restaurant()
              restaurant.name = "TastyBaaS " + String(i)
              restaurant.cuisine = "mBaaS"
              dataStore.save(entity: restaurant, responseHandler: { savedRestaurant in
                  if let savedRestaurant = savedRestaurant as? Restaurant {
                      print("Saved \(savedRestaurant.name ?? "")")
                  }
              }, errorHandler: { fault in
                  print("Error: \(fault.message ?? "")")
              })
          }
      }

      function addRestaurants() {
        for (let i = 0; i < 300; i++) { const newRestaurant = { name : `TastyBaaS ${i}`, cuisine: 'mBaaS', } Backendless.Persistence.of('Restaurant').save(newRestaurant) .then(result => console.log(`Saved ${result.name}`))
            .catch(console.error)
        }
      }
      

      static void _addRestaurants() {
         IDataStore
      
      
      
      
      
      
      
      
       dataStore = Backendless.Data.of("Restaurant");
         List
      
      
      
      
      
      
      
      
       restaurants = List();
         for (int i = 0; i < 100; i++) { Map restaurant = { "name": "TastyBaaS $i", "Cuisine": "mBaaS", }; restaurants.add(restaurant); } dataStore.create(restaurants).then((ids) { ids.forEach((id) => print("Object saved with ID - $id"));
         });
      }
       
      

       

       

       

       

      If you run the method above once, you should have 104 objects in the backend table (4 objects came from the data import and 100 created by the code).

      Now the backend is ready to show the power of paging. The following code  demonstrates just that:

        private static void basicPagingAsync() {
           final long startTime = System.currentTimeMillis();
        
           final int PAGESIZE = 80;
           final DataQueryBuilder dataQuery = DataQueryBuilder.create();
           dataQuery.setPageSize(PAGESIZE);
        
           final AsyncCallback<List<Restaurant>> callback = new AsyncCallback<List<Restaurant>>() {
        
               public void handleResponse(List<Restaurant> restaurants) {
                   int size = restaurants.size();
                   Log.i(TAG, "Loaded " + size + " restaurants in the current page");
        
                   if (size == PAGESIZE) {
                       dataQuery.prepareNextPage();
                       Backendless.Data.of(Restaurant.class).find(dataQuery, this);
                   } else {
                       Log.i(TAG, "Total time (ms) - " + (System.currentTimeMillis() - startTime));
                   }
               }
        
               @Override
               public void handleFault(BackendlessFault fault) {
                   Log.e(TAG, fault.getMessage());
               }
           };
        
           Backendless.Data.of(Restaurant.class).getObjectCount(new AsyncCallback<Integer>() {
               @Override
               public void handleResponse(Integer response) {
                   Log.i(TAG, "Total restaurants - " + response);
                   Backendless.Data.of(Restaurant.class).find(dataQuery, callback);
               }
        
               @Override
               public void handleFault(BackendlessFault fault) {
                   Log.e(TAG, fault.getMessage());
               }
           });
        }

        private fun basicPagingAsync() {
           val startTime = System.currentTimeMillis()
        
           val PAGESIZE = 80
           val dataQuery = DataQueryBuilder.create()
           dataQuery.setPageSize(PAGESIZE)
        
           val callback = object : AsyncCallback<List<Restaurant>> {
        
               override fun handleResponse(restaurants: List<Restaurant>) {
                   val size = restaurants.size
                   Log.i(TAG, "Loaded $size restaurants in the current page")
        
                   if (size == PAGESIZE) {
                       dataQuery.prepareNextPage()
                       Backendless.Data.of(Restaurant::class.java).find(dataQuery, this)
                   } else {
                       Log.i(TAG, "Total time (ms) - ${System.currentTimeMillis() - startTime}")
                   }
               }
        
               override fun handleFault(fault: BackendlessFault) {
                   Log.e(TAG, fault.message)
               }
           }
        
           Backendless.Data.of(Restaurant::class.java).getObjectCount(object : AsyncCallback<Int> {
               override fun handleResponse(response: Int?) {
                   Log.i(TAG, "Total restaurants - $response")
                   Backendless.Data.of(Restaurant::class.java).find(dataQuery, callback)
               }
        
               override fun handleFault(fault: BackendlessFault) {
                   Log.e(TAG, fault.message)
               }
           })
        }

        block BOOL firstResponse = YES;
        DataQueryBuilder *queryBuilder = [DataQueryBuilder new];
        
        - (void)basicPaging {
            NSDate *startTime = [NSDate date];
            DataStoreFactory *dataStore = [Backendless.shared.data of:[Restaurant class]];
            
            [dataStore findWithQueryBuilder:queryBuilder responseHandler:^(NSArray *restaurants) {
                [dataStore getObjectCountWithResponseHandler:^(NSInteger totalObjects) {
                    if (self->firstResponse) {
                        self->firstResponse = NO;
                        NSLog(@"Total restaurants - %li", (long)totalObjects);
                    }
                    NSInteger size = restaurants.count;
                    if (size == 0) {
                        NSLog(@"Total time (ms) - %g", 1000 * [[NSDate date] timeIntervalSinceDate:startTime]);
                    }
                    else {
                        NSLog(@"Loaded %li restaurants in the current page", (long)size);
                    }
                    if ([self->queryBuilder getOffset] < totalObjects) { [self->queryBuilder prepareNextPage];
                        [self basicPaging];
                    }
                } errorHandler:^(Fault *fault) {
                    NSLog(@"Error: %@", fault.message);
                }];
            } errorHandler:^(Fault *fault) {
                NSLog(@"Error: %@", fault.message);
            }];
        }

        var firstResponse = true
        let queryBuilder = DataQueryBuilder()
        
        func basicPaging() {
            let startTime = Date()
            let dataStore = Backendless.shared.data.of(Restaurant.self)
                
            dataStore.find(queryBuilder: queryBuilder, responseHandler: { restaurants in
                dataStore.getObjectCount(responseHandler: { totalObjects in
                    if self.firstResponse {
                        self.firstResponse = false
                        print("Total restaurants - \(totalObjects)")
                    }
                    let size = restaurants.count
                    if size == 0 {
                        print("Total time (ms) - \(Int(Date().timeIntervalSince(startTime) * 1000))")
                    }
                    else {
                        print("Loaded \(size) restaurants in the current page")
                    }
                    if self.queryBuilder.getOffset() < totalObjects {
                        self.queryBuilder.prepareNextPage()
                        self.basicPaging()
                    }
                }, errorHandler: { fault in
                    print("Error: \(fault.message ?? "")")
                })
            }, errorHandler: { fault in
                print("Error: \(fault.message ?? "")")
            })
        }

        Backendless.initApp(APP_ID, API_KEY)
        
        const PAGE_SIZE = 100 // max allowed value
        
        const fetchAllRestaurants = async () => {
          let lastLoadedItemsSize = 0
          let itemsCollection = []
        
          const startTime = Date.now()
        
          const pageQuery = Backendless.DataQueryBuilder.create()
        
          pageQuery.setPageSize(PAGE_SIZE)
        
          do {
            const items = await Backendless.Data.of('Restaurant').find(pageQuery)
        
            lastLoadedItemsSize = items.length
        
            itemsCollection = itemsCollection.concat(items)
        
            pageQuery.prepareNextPage()
        
          } while (lastLoadedItemsSize >= PAGE_SIZE)
        
          console.log(`Total time (ms) - ${Date.now() - startTime}`)
        
          return itemsCollection
        }
        
        const onSuccess = restaurants => {
          console.log(`Loaded ${ restaurants.length } restaurants objects`)
        
          restaurants.forEach(restaurant => {
            console.log(`Restaurant name = ${ restaurant.name }`)
          })
        }
        
        const onError = error => {
          console.error(`Server reported an error: [${error.code} - ${error.message}`)
        }
        
        Promise.resolve()
          .then(fetchAllRestaurants)
          .then(onSuccess)
          .catch(onError)
        

        static void _basicPagingAsync() {
           int startTime = DateTime.now().millisecondsSinceEpoch;
        
           int PAGESIZE = 80;
           DataQueryBuilder dataQuery = DataQueryBuilder()
             ..pageSize = PAGESIZE;
        
           Function getPagedData;
           getPagedData = () {
             Backendless.Data.of("Restaurant").find(dataQuery).then((restaurants) {
               int size = restaurants.length;
               print("Loaded $size restaurants in the current page");
        
               if (size == PAGESIZE) {
                 dataQuery.prepareNextPage();
                 getPagedData();
               } else {
                 print("Total time (ms) - ${DateTime.now().millisecondsSinceEpoch - startTime}");
               }
             });
           };
        
           Backendless.Data.of("Restaurant").getObjectCount().then((count) {
             print("Total restaurants - $count");
             getPagedData();
           });
         }

        The principle for loading a page of data is very simple – the code calls the prepareNextPage() method which adjusts the index of the next set of database records to retrieve.

        The prepareNextPage() API is very convenient, it uses the page size you specify with the setPageSize() call and automatically adjusts the offset/index of the subsequently requested page of data. However, there are scenarios when the client needs to load a page out of order. For example, the app UI may offer paging navigation with clickable numbers for the individual pages. In this case, Backendless provides an advanced mechanism where you can control the index/offset of the next block of data objects. The following example demonstrates the usage of the advanced paging – a mechanism which lets you download any block of data (or page) from the database:

          private static void advancedPagingAsync() {
             final long startTime = System.currentTimeMillis();
          
             final int PAGESIZE = 100;
             final DataQueryBuilder dataQuery = DataQueryBuilder.create();
             dataQuery.setPageSize(PAGESIZE);
          
             final AsyncCallback<List> callback = new AsyncCallback<List<Restaurant>>() {
                 private int offset = 0;
          
                 public void handleResponse(List<Restaurant> restaurants) {
                     int size = restaurants.size();
                     Log.i(TAG, "Loaded " + size + " restaurants in the current page");
          
                     if (size == PAGESIZE) {
                         offset += restaurants.size();
                         dataQuery.setOffset(offset);
                         Backendless.Data.of(Restaurant.class).find(dataQuery, this);
                     } else {
                         Log.i(TAG, "Total time (ms) - " + (System.currentTimeMillis() - startTime));
                     }
                 }
          
                 @Override
                 public void handleFault(BackendlessFault fault) {
                     Log.e(TAG, fault.getMessage());
                 }
             };
          
             Backendless.Data.of(Restaurant.class).getObjectCount(new AsyncCallback<Integer>() {
                 @Override
                 public void handleResponse(Integer response) {
                     Log.i(TAG, "Total restaurants - " + response);
                     Backendless.Data.of(Restaurant.class).find(dataQuery, callback);
                 }
          
                 @Override
                 public void handleFault(BackendlessFault fault) {
                     Log.e(TAG, fault.getMessage());
                 }
             });
          }

          private fun advancedPagingAsync() {
             val startTime = System.currentTimeMillis()
          
             val PAGESIZE = 100
             val dataQuery = DataQueryBuilder.create()
             dataQuery.setPageSize(PAGESIZE)
          
             val callback = object : AsyncCallback<List<Restaurant>> {
                 private var offset = 0
          
                 override fun handleResponse(restaurants: List<Restaurant>) {
                     val size = restaurants.size
                     Log.i(TAG, "Loaded $size restaurants in the current page")
          
                     if (size == PAGESIZE) {
                         offset += restaurants.size
                         dataQuery.setOffset(offset)
                         Backendless.Data.of(Restaurant::class.java).find(dataQuery, this)
                     } else {
                         Log.i(TAG, "Total time (ms) - ${System.currentTimeMillis() - startTime}")
                     }
                 }
          
                 override fun handleFault(fault: BackendlessFault) {
                     Log.e(TAG, fault.message)
                 }
             }
          
             Backendless.Data.of(Restaurant::class.java).getObjectCount(object : AsyncCallback<Int> {
                 override fun handleResponse(response: Int?) {
                     Log.i(TAG, "Total restaurants - $response")
                     Backendless.Data.of(Restaurant::class.java).find(dataQuery, callback)
                 }
          
                 override fun handleFault(fault: BackendlessFault) {
                     Log.e(TAG, fault.message)
                 }
             })
          }

          BOOL firstResponse = YES;
          NSInteger offset = 0;
          
          DataQueryBuilder *queryBuilder = [DataQueryBuilder new];
          [queryBuilder setPageSizeWithPageSize:100];
          [queryBuilder setOffsetWithOffset:offset];
          
          - (void)advancedPaging {
              NSDate *startTime = [NSDate date];
              DataStoreFactory *dataStore = [Backendless.shared.data of:[Restaurant class]];
              
              [dataStore findWithQueryBuilder:queryBuilder responseHandler:^(NSArray *restaurants) {
                  [dataStore getObjectCountWithResponseHandler:^(NSInteger totalObjects) {
                      if (self->firstResponse) {
                          self->firstResponse = NO;
                          NSLog(@"Total restaurants - %li", (long)totalObjects);
                      }
                      NSInteger size = restaurants.count;
                      if (size == 0) {
                          NSLog(@"Total time (ms) - %g", 1000 * [[NSDate date] timeIntervalSinceDate:startTime]);
                      }
                      else {
                          NSLog(@"Loaded %li restaurants in the current page", (long)size);
                          self->offset += size;
                      }
                      if ([self->queryBuilder getOffset] < totalObjects) { [self->queryBuilder setOffsetWithOffset:self->offset];
                          [self advancedPaging];
                      }
                  } errorHandler:^(Fault *fault) {
                      NSLog(@"Error: %@", fault.message);
                  }];
              } errorHandler:^(Fault *fault) {
                  NSLog(@"Error: %@", fault.message);
              }];
          }

          var firstResponse = true
          var offset = 0
          
          let queryBuilder = DataQueryBuilder()
          queryBuilder.setPageSize(pageSize: 100)
          queryBuilder.setOffset(offset: offset)
          
          func advancedPaging() {
              let startTime = Date()
              let dataStore = Backendless.shared.data.of(Restaurant.self)
                  
              dataStore.find(queryBuilder: queryBuilder, responseHandler: { restaurants in
                  dataStore.getObjectCount(responseHandler: { totalObjects in
                      if self.firstResponse {
                          self.firstResponse = false
                          print("Total restaurants - \(totalObjects)")
                      }
                      let size = restaurants.count
                      if size == 0 {
                          print("Total time (ms) - \(Int(Date().timeIntervalSince(startTime) * 1000))")
                      }
                      else {
                          print("Loaded \(size) restaurants in the current page")
                          self.offset += size
                      }
                      if self.queryBuilder.getOffset() < totalObjects {
                          self.queryBuilder.setOffset(offset: self.offset)
                          self.advancedPaging()
                      }
                  }, errorHandler: { fault in
                      print("Error: \(fault.message ?? "")")
                  })
              }, errorHandler: { fault in
                  print("Error: \(fault.message ?? "")")
              })
          }

          Backendless.initApp(APP_ID, API_KEY)
          
          const PAGE_SIZE = 100 // max allowed value
          
          const fetchAllRestaurants = async () => {
            let offset = 0
            let lastPageSize = 0
            let itemsCollection = []
          
            const startTime = Date.now()
          
            const pageQuery = Backendless.DataQueryBuilder.create()
            pageQuery.setPageSize(PAGE_SIZE)
          
            do {
              pageQuery.setOffset(offset)
          
              const items = await Backendless.Data.of('Restaurant').find(pageQuery)
          
              lastPageSize = items.length
          
              itemsCollection = itemsCollection.concat(items)
          
              offset += PAGE_SIZE
            } while (lastPageSize >= PAGE_SIZE)
          
            console.log(`Total time (ms) - ${Date.now() - startTime}`)
          
            return itemsCollection
          }
          
          const onSuccess = restaurants => {
            console.log(`Loaded ${ restaurants.length } restaurants objects`)
          
            restaurants.forEach(restaurant => {
              console.log(`Restaurant name = ${ restaurant.name }`)
            })
          }
          
          const onError = error => {
            console.error(`Server reported an error: [${error.code}] - ${error.message}`)
          }
          
          Promise.resolve()
            .then(fetchAllRestaurants)
            .then(onSuccess)
            .catch(onError)
          

          static void _advancedPagingAsync() {
             int startTime = DateTime.now().millisecondsSinceEpoch;
          
             int PAGESIZE = 100;
             DataQueryBuilder dataQuery = DataQueryBuilder()
               ..pageSize = PAGESIZE;
          
             int offset = 0;
          
             Function getPagedData;
             getPagedData = () {
               Backendless.Data.of("Restaurant").find(dataQuery).then((restaurants) {
                 int size = restaurants.length;
                 print("Loaded $size restaurants in the current page");
          
                 if (size == PAGESIZE) {
                   offset += restaurants.length;
                   dataQuery.offset = offset;
                   getPagedData();
                 } else {
                   print("Total time (ms) - ${DateTime.now().millisecondsSinceEpoch - startTime}");
                 }
               });
             };
          
             Backendless.Data.of("Restaurant").getObjectCount().then((count) {
               print("Total restaurants - $count");
               getPagedData();
             });
           }

          Leave a Reply