Blog

How to Efficiently Load Large Data Sets in a Mobile App With Data Paging

by Mark Piller on September 1, 2019

Data paging is the process of breaking up a larger set of objects into smaller chunks, commonly referred to as pages. Typically paging applies to the results of search queries when the server may return too many objects at once.

Since smaller pages of data can be returned to the client much faster, the user of the client application does not need to wait. As a result, the user experience of the app is significantly improved, which is the primary advantage of paging.

Backendless automatically breaks up the results of search queries into pages. The examples below demonstrate the usage of the API and various techniques for paged data retrieval.

All the examples in this article rely on the code obtained through Backendless code generator. For details see how to generate client-side code for your mobile application with Backendless. To setup your backend in order to run these examples, you should:

Register/login to Backendless Console and create an app.

Download the following file:
https://backendless.com/documentation/samples/restaraunt-app-tables-with-data.zip

Configure your backends storage by following the instructions on how to import schema and data from a backup file to a Backendless app. You will need to use the file from step 2.

Generate client-side source code using Backendless code generator.

In another post, we showed how to load data from Backendless in the most basic API. The code in that post uses the basic find API to fetch a collection of the restaurant objects. Since Backendless automatically pages data, it will return only the first page. The default page size is 10 objects, however, that value can be changed using the following API (maximum allowed page size is 100):

AsyncCallback<List<Restaurant>> callback = new AsyncCallback<List<Restaurant>>()
{ /* skipped for brevity */  };
int PAGESIZE = 80;
DataQueryBuilder dataQuery = DataQueryBuilder.create();
dataQuery.setPageSize(PAGESIZE);
Backendless.Data.of(Restaurant.class).find(dataQuery, callback);

val callback = object : AsyncCallback<List<Restaurant>> { /* skipped for brevity */  }
val PAGESIZE = 80
val dataQuery = DataQueryBuilder.create()
dataQuery.setPageSize(PAGESIZE)
Backendless.Data.of(Restaurant::class.java).find(dataQuery, callback)

NSInteger PAGESIZE = 80;
    
DataQueryBuilder *queryBuilder = [DataQueryBuilder new];
[queryBuilder setPageSizeWithPageSize:PAGESIZE];
    
[[Backendless.shared.data of:[Restaurant class]] findWithQueryBuilder:queryBuilder responseHandler:^(NSArray *restaurants) {
    // skipped for brevity
} errorHandler:^(Fault *fault) {
    NSLog(@"Error: %@", fault.message);
}];

let PAGESIZE = 80
        
let queryBuilder = DataQueryBuilder()
queryBuilder.setPageSize(pageSize: PAGESIZE)
        
Backendless.shared.data.of(Restaurant.self).find(queryBuilder: queryBuilder, responseHandler: { restaurants in
    // skipped for brevity
}, errorHandler: { fault in
    print("Error: \(fault.message ?? "")")
})

Backendless.initApp(APP_ID, API_KEY)

const PAGE_SIZE = 80

const dataQuery = Backendless.DataQueryBuilder.create().setPageSize(PAGE_SIZE)

Backendless.Persistence.of('Restaurant').find(dataQuery)
  .then(console.log)
  .catch(console.error)

int PAGESIZE = 80;
DataQueryBuilder dataQuery = DataQueryBuilder()
  ..pageSize = PAGESIZE;

Backendless.Data.of("Restaurant").find(dataQuery).then((response)
  { /* skipped for brevity */  });

Now that we know how to fetch the first page, the question is how to load subsequent pages. Before we run any of the paging examples below, it would be worthwhile to populate the backend storage with additional objects. Without them, you would not be able to see the paging in action – all the objects would be returned in the first page. Consider the following code that adds 100 restaurant objects to the backend storage:

private static void addRestaurants() {
   IDataStore dataStore = Backendless.Data.of(Restaurant.class);
   List restaurants = new ArrayList<>();
   for (int i = 0; i < 100; i++) { Restaurant restaurant = new Restaurant(); restaurant.setName("TastyBaaS " + i); restaurant.setCuisine("mBaaS"); restaurants.add(restaurant); } dataStore.create(restaurants, new AsyncCallback<List<String>>() {
       @Override
       public void handleResponse(List<String> ids) {
           for (String id : ids) {
               Log.i(TAG, "Object saved with ID - " + id);
           }
       }

       @Override
       public void handleFault(BackendlessFault fault) {
           Log.e(TAG, fault.getMessage());
       }
   });
}

private fun addRestaurants() {
   val dataStore = Backendless.Data.of(Restaurant::class.java)
   val restaurants = ArrayList()
   for (i in 0..99) {
       val restaurant = Restaurant()
       restaurant.name = "TastyBaaS $i"
       restaurant.cuisine = "mBaaS"
       restaurants.add(restaurant)
   }

   dataStore.create(restaurants, object : AsyncCallback<List<String>> {
       override fun handleResponse(ids: List<String>) {
           for (id in ids) {
               Log.i(TAG, "Object saved with ID - $id")
           }
       }

       override fun handleFault(fault: BackendlessFault) {
           Log.e(TAG, fault.message)
       }
   })
}

- (void)addRestaurants {
    DataStoreFactory *dataStore = [Backendless.shared.data of:[Restaurant class]];
    for (int i = 0; i < 300; i++) {
        Restaurant *restaurant = [Restaurant new];
        restaurant.name = [NSString stringWithFormat:@"TastyBaaS %i", i];
        restaurant.cuisine = @"mBaaS";
        [dataStore saveWithEntity:restaurant responseHandler:^(Restaurant *savedRestaurant) {
            NSLog(@"Saved %@", savedRestaurant.name);
        } errorHandler:^(Fault *fault) {
            NSLog(@"Error: %@", fault.message);
        }];
    }
}

func addRestaurants() {
    let dataStore = Backendless.shared.data.of(Restaurant.self)
    for i in 0..<300 {
        let restaurant = Restaurant()
        restaurant.name = "TastyBaaS " + String(i)
        restaurant.cuisine = "mBaaS"
        dataStore.save(entity: restaurant, responseHandler: { savedRestaurant in
            if let savedRestaurant = savedRestaurant as? Restaurant {
                print("Saved \(savedRestaurant.name ?? "")")
            }
        }, errorHandler: { fault in
            print("Error: \(fault.message ?? "")")
        })
    }
}

function addRestaurants() {
  for (let i = 0; i < 300; i++) { const newRestaurant = { name : `TastyBaaS ${i}`, cuisine: 'mBaaS', } Backendless.Persistence.of('Restaurant').save(newRestaurant) .then(result => console.log(`Saved ${result.name}`))
      .catch(console.error)
  }
}

static void _addRestaurants() {
   IDataStore








 dataStore = Backendless.Data.of("Restaurant");
   List








 restaurants = List();
   for (int i = 0; i < 100; i++) { Map restaurant = { "name": "TastyBaaS $i", "Cuisine": "mBaaS", }; restaurants.add(restaurant); } dataStore.create(restaurants).then((ids) { ids.forEach((id) => print("Object saved with ID - $id"));
   });
}

If you run the method above once, you should have 104 objects in the backend table (4 objects came from the data import and 100 created by the code).

Now the backend is ready to show the power of paging. The following code demonstrates just that:

private static void basicPagingAsync() {
   final long startTime = System.currentTimeMillis();

   final int PAGESIZE = 80;
   final DataQueryBuilder dataQuery = DataQueryBuilder.create();
   dataQuery.setPageSize(PAGESIZE);

   final AsyncCallback<List<Restaurant>> callback = new AsyncCallback<List<Restaurant>>() {

       public void handleResponse(List<Restaurant> restaurants) {
           int size = restaurants.size();
           Log.i(TAG, "Loaded " + size + " restaurants in the current page");

           if (size == PAGESIZE) {
               dataQuery.prepareNextPage();
               Backendless.Data.of(Restaurant.class).find(dataQuery, this);
           } else {
               Log.i(TAG, "Total time (ms) - " + (System.currentTimeMillis() - startTime));
           }
       }

       @Override
       public void handleFault(BackendlessFault fault) {
           Log.e(TAG, fault.getMessage());
       }
   };

   Backendless.Data.of(Restaurant.class).getObjectCount(new AsyncCallback<Integer>() {
       @Override
       public void handleResponse(Integer response) {
           Log.i(TAG, "Total restaurants - " + response);
           Backendless.Data.of(Restaurant.class).find(dataQuery, callback);
       }

       @Override
       public void handleFault(BackendlessFault fault) {
           Log.e(TAG, fault.getMessage());
       }
   });
}

private fun basicPagingAsync() {
   val startTime = System.currentTimeMillis()

   val PAGESIZE = 80
   val dataQuery = DataQueryBuilder.create()
   dataQuery.setPageSize(PAGESIZE)

   val callback = object : AsyncCallback<List<Restaurant>> {

       override fun handleResponse(restaurants: List<Restaurant>) {
           val size = restaurants.size
           Log.i(TAG, "Loaded $size restaurants in the current page")

           if (size == PAGESIZE) {
               dataQuery.prepareNextPage()
               Backendless.Data.of(Restaurant::class.java).find(dataQuery, this)
           } else {
               Log.i(TAG, "Total time (ms) - ${System.currentTimeMillis() - startTime}")
           }
       }

       override fun handleFault(fault: BackendlessFault) {
           Log.e(TAG, fault.message)
       }
   }

   Backendless.Data.of(Restaurant::class.java).getObjectCount(object : AsyncCallback<Int> {
       override fun handleResponse(response: Int?) {
           Log.i(TAG, "Total restaurants - $response")
           Backendless.Data.of(Restaurant::class.java).find(dataQuery, callback)
       }

       override fun handleFault(fault: BackendlessFault) {
           Log.e(TAG, fault.message)
       }
   })
}

block BOOL firstResponse = YES;
DataQueryBuilder *queryBuilder = [DataQueryBuilder new];

- (void)basicPaging {
    NSDate *startTime = [NSDate date];
    DataStoreFactory *dataStore = [Backendless.shared.data of:[Restaurant class]];
    
    [dataStore findWithQueryBuilder:queryBuilder responseHandler:^(NSArray *restaurants) {
        [dataStore getObjectCountWithResponseHandler:^(NSInteger totalObjects) {
            if (self->firstResponse) {
                self->firstResponse = NO;
                NSLog(@"Total restaurants - %li", (long)totalObjects);
            }
            NSInteger size = restaurants.count;
            if (size == 0) {
                NSLog(@"Total time (ms) - %g", 1000 * [[NSDate date] timeIntervalSinceDate:startTime]);
            }
            else {
                NSLog(@"Loaded %li restaurants in the current page", (long)size);
            }
            if ([self->queryBuilder getOffset] < totalObjects) { [self->queryBuilder prepareNextPage];
                [self basicPaging];
            }
        } errorHandler:^(Fault *fault) {
            NSLog(@"Error: %@", fault.message);
        }];
    } errorHandler:^(Fault *fault) {
        NSLog(@"Error: %@", fault.message);
    }];
}

var firstResponse = true
let queryBuilder = DataQueryBuilder()

func basicPaging() {
    let startTime = Date()
    let dataStore = Backendless.shared.data.of(Restaurant.self)
        
    dataStore.find(queryBuilder: queryBuilder, responseHandler: { restaurants in
        dataStore.getObjectCount(responseHandler: { totalObjects in
            if self.firstResponse {
                self.firstResponse = false
                print("Total restaurants - \(totalObjects)")
            }
            let size = restaurants.count
            if size == 0 {
                print("Total time (ms) - \(Int(Date().timeIntervalSince(startTime) * 1000))")
            }
            else {
                print("Loaded \(size) restaurants in the current page")
            }
            if self.queryBuilder.getOffset() < totalObjects {
                self.queryBuilder.prepareNextPage()
                self.basicPaging()
            }
        }, errorHandler: { fault in
            print("Error: \(fault.message ?? "")")
        })
    }, errorHandler: { fault in
        print("Error: \(fault.message ?? "")")
    })
}

Backendless.initApp(APP_ID, API_KEY)

const PAGE_SIZE = 100 // max allowed value

const fetchAllRestaurants = async () => {
  let lastLoadedItemsSize = 0
  let itemsCollection = []

  const startTime = Date.now()

  const pageQuery = Backendless.DataQueryBuilder.create()

  pageQuery.setPageSize(PAGE_SIZE)

  do {
    const items = await Backendless.Data.of('Restaurant').find(pageQuery)

    lastLoadedItemsSize = items.length

    itemsCollection = itemsCollection.concat(items)

    pageQuery.prepareNextPage()

  } while (lastLoadedItemsSize >= PAGE_SIZE)

  console.log(`Total time (ms) - ${Date.now() - startTime}`)

  return itemsCollection
}

const onSuccess = restaurants => {
  console.log(`Loaded ${ restaurants.length } restaurants objects`)

  restaurants.forEach(restaurant => {
    console.log(`Restaurant name = ${ restaurant.name }`)
  })
}

const onError = error => {
  console.error(`Server reported an error: [${error.code} - ${error.message}`)
}

Promise.resolve()
  .then(fetchAllRestaurants)
  .then(onSuccess)
  .catch(onError)

static void _basicPagingAsync() {
   int startTime = DateTime.now().millisecondsSinceEpoch;

   int PAGESIZE = 80;
   DataQueryBuilder dataQuery = DataQueryBuilder()
     ..pageSize = PAGESIZE;

   Function getPagedData;
   getPagedData = () {
     Backendless.Data.of("Restaurant").find(dataQuery).then((restaurants) {
       int size = restaurants.length;
       print("Loaded $size restaurants in the current page");

       if (size == PAGESIZE) {
         dataQuery.prepareNextPage();
         getPagedData();
       } else {
         print("Total time (ms) - ${DateTime.now().millisecondsSinceEpoch - startTime}");
       }
     });
   };

   Backendless.Data.of("Restaurant").getObjectCount().then((count) {
     print("Total restaurants - $count");
     getPagedData();
   });
 }

The principle for loading a page of data is very simple – the code calls the prepareNextPage() method which adjusts the index of the next set of database records to retrieve.

The prepareNextPage() API is very convenient, it uses the page size you specify with the setPageSize() call and automatically adjusts the offset/index of the subsequently requested page of data. However, there are scenarios when the client needs to load a page out of order. For example, the app UI may offer paging navigation with clickable numbers for the individual pages. In this case, Backendless provides an advanced mechanism where you can control the index/offset of the next block of data objects. The following example demonstrates the usage of the advanced paging – a mechanism which lets you download any block of data (or page) from the database:

private static void advancedPagingAsync() {
   final long startTime = System.currentTimeMillis();

   final int PAGESIZE = 100;
   final DataQueryBuilder dataQuery = DataQueryBuilder.create();
   dataQuery.setPageSize(PAGESIZE);

   final AsyncCallback<List> callback = new AsyncCallback<List<Restaurant>>() {
       private int offset = 0;

       public void handleResponse(List<Restaurant> restaurants) {
           int size = restaurants.size();
           Log.i(TAG, "Loaded " + size + " restaurants in the current page");

           if (size == PAGESIZE) {
               offset += restaurants.size();
               dataQuery.setOffset(offset);
               Backendless.Data.of(Restaurant.class).find(dataQuery, this);
           } else {
               Log.i(TAG, "Total time (ms) - " + (System.currentTimeMillis() - startTime));
           }
       }

       @Override
       public void handleFault(BackendlessFault fault) {
           Log.e(TAG, fault.getMessage());
       }
   };

   Backendless.Data.of(Restaurant.class).getObjectCount(new AsyncCallback<Integer>() {
       @Override
       public void handleResponse(Integer response) {
           Log.i(TAG, "Total restaurants - " + response);
           Backendless.Data.of(Restaurant.class).find(dataQuery, callback);
       }

       @Override
       public void handleFault(BackendlessFault fault) {
           Log.e(TAG, fault.getMessage());
       }
   });
}

private fun advancedPagingAsync() {
   val startTime = System.currentTimeMillis()

   val PAGESIZE = 100
   val dataQuery = DataQueryBuilder.create()
   dataQuery.setPageSize(PAGESIZE)

   val callback = object : AsyncCallback<List<Restaurant>> {
       private var offset = 0

       override fun handleResponse(restaurants: List<Restaurant>) {
           val size = restaurants.size
           Log.i(TAG, "Loaded $size restaurants in the current page")

           if (size == PAGESIZE) {
               offset += restaurants.size
               dataQuery.setOffset(offset)
               Backendless.Data.of(Restaurant::class.java).find(dataQuery, this)
           } else {
               Log.i(TAG, "Total time (ms) - ${System.currentTimeMillis() - startTime}")
           }
       }

       override fun handleFault(fault: BackendlessFault) {
           Log.e(TAG, fault.message)
       }
   }

   Backendless.Data.of(Restaurant::class.java).getObjectCount(object : AsyncCallback<Int> {
       override fun handleResponse(response: Int?) {
           Log.i(TAG, "Total restaurants - $response")
           Backendless.Data.of(Restaurant::class.java).find(dataQuery, callback)
       }

       override fun handleFault(fault: BackendlessFault) {
           Log.e(TAG, fault.message)
       }
   })
}

BOOL firstResponse = YES;
NSInteger offset = 0;

DataQueryBuilder *queryBuilder = [DataQueryBuilder new];
[queryBuilder setPageSizeWithPageSize:100];
[queryBuilder setOffsetWithOffset:offset];

- (void)advancedPaging {
    NSDate *startTime = [NSDate date];
    DataStoreFactory *dataStore = [Backendless.shared.data of:[Restaurant class]];
    
    [dataStore findWithQueryBuilder:queryBuilder responseHandler:^(NSArray *restaurants) {
        [dataStore getObjectCountWithResponseHandler:^(NSInteger totalObjects) {
            if (self->firstResponse) {
                self->firstResponse = NO;
                NSLog(@"Total restaurants - %li", (long)totalObjects);
            }
            NSInteger size = restaurants.count;
            if (size == 0) {
                NSLog(@"Total time (ms) - %g", 1000 * [[NSDate date] timeIntervalSinceDate:startTime]);
            }
            else {
                NSLog(@"Loaded %li restaurants in the current page", (long)size);
                self->offset += size;
            }
            if ([self->queryBuilder getOffset] < totalObjects) { [self->queryBuilder setOffsetWithOffset:self->offset];
                [self advancedPaging];
            }
        } errorHandler:^(Fault *fault) {
            NSLog(@"Error: %@", fault.message);
        }];
    } errorHandler:^(Fault *fault) {
        NSLog(@"Error: %@", fault.message);
    }];
}

var firstResponse = true
var offset = 0

let queryBuilder = DataQueryBuilder()
queryBuilder.setPageSize(pageSize: 100)
queryBuilder.setOffset(offset: offset)

func advancedPaging() {
    let startTime = Date()
    let dataStore = Backendless.shared.data.of(Restaurant.self)
        
    dataStore.find(queryBuilder: queryBuilder, responseHandler: { restaurants in
        dataStore.getObjectCount(responseHandler: { totalObjects in
            if self.firstResponse {
                self.firstResponse = false
                print("Total restaurants - \(totalObjects)")
            }
            let size = restaurants.count
            if size == 0 {
                print("Total time (ms) - \(Int(Date().timeIntervalSince(startTime) * 1000))")
            }
            else {
                print("Loaded \(size) restaurants in the current page")
                self.offset += size
            }
            if self.queryBuilder.getOffset() < totalObjects {
                self.queryBuilder.setOffset(offset: self.offset)
                self.advancedPaging()
            }
        }, errorHandler: { fault in
            print("Error: \(fault.message ?? "")")
        })
    }, errorHandler: { fault in
        print("Error: \(fault.message ?? "")")
    })
}

Backendless.initApp(APP_ID, API_KEY)

const PAGE_SIZE = 100 // max allowed value

const fetchAllRestaurants = async () => {
  let offset = 0
  let lastPageSize = 0
  let itemsCollection = []

  const startTime = Date.now()

  const pageQuery = Backendless.DataQueryBuilder.create()
  pageQuery.setPageSize(PAGE_SIZE)

  do {
    pageQuery.setOffset(offset)

    const items = await Backendless.Data.of('Restaurant').find(pageQuery)

    lastPageSize = items.length

    itemsCollection = itemsCollection.concat(items)

    offset += PAGE_SIZE
  } while (lastPageSize >= PAGE_SIZE)

  console.log(`Total time (ms) - ${Date.now() - startTime}`)

  return itemsCollection
}

const onSuccess = restaurants => {
  console.log(`Loaded ${ restaurants.length } restaurants objects`)

  restaurants.forEach(restaurant => {
    console.log(`Restaurant name = ${ restaurant.name }`)
  })
}

const onError = error => {
  console.error(`Server reported an error: [${error.code}] - ${error.message}`)
}

Promise.resolve()
  .then(fetchAllRestaurants)
  .then(onSuccess)
  .catch(onError)

static void _advancedPagingAsync() {
   int startTime = DateTime.now().millisecondsSinceEpoch;

   int PAGESIZE = 100;
   DataQueryBuilder dataQuery = DataQueryBuilder()
     ..pageSize = PAGESIZE;

   int offset = 0;

   Function getPagedData;
   getPagedData = () {
     Backendless.Data.of("Restaurant").find(dataQuery).then((restaurants) {
       int size = restaurants.length;
       print("Loaded $size restaurants in the current page");

       if (size == PAGESIZE) {
         offset += restaurants.length;
         dataQuery.offset = offset;
         getPagedData();
       } else {
         print("Total time (ms) - ${DateTime.now().millisecondsSinceEpoch - startTime}");
       }
     });
   };

   Backendless.Data.of("Restaurant").getObjectCount().then((count) {
     print("Total restaurants - $count");
     getPagedData();
   });
 }

How to Efficiently Load Large Data Sets in a Mobile App With Data Paging

Leave a Reply Cancel reply

Related Articles