Android RXJava2 Retrofit Handling paging

Question

I'm having issues handling paging with RXJava2. I tried to follow this post from SO but it was dated. I was hoping to get a more up to date answer. I'm using Retrofit and I'm trying to page a third party API till I have all of the results. The response is below but I pretty much want to through all pages and combine all results.

Currently my code is as follows:

public Observable<WatchList> syncWatchlistSection(String accountId) {
    String sessionToken = mLocal.getSessionId();
    int page = 1;

    return getWatchlistFromServerHelper(sessionToken, accountId, page).concatMap(new Function<WatchList, ObservableSource<? extends WatchList>>() {
        @Override
        public ObservableSource<? extends WatchList> apply(WatchList watchList) throws Exception {
            return Observable.fromArray(watchList);
        }
    });

}

And in the same file the helper file similar to the previous SO post.

private Observable<WatchList> getWatchlistFromServerHelper(String sessionToken, String accountId, int page) {
        return mRestAPI.fetchWatchlistOnServer(sessionToken, accountId, page).concatMap(new Function<WatchList, Observable<WatchList>>() {
            @Override
            public Observable<WatchList> apply(WatchList watchList) throws Exception {
                if (watchList.getPage() == watchList.getTotalPages()) {
                    return Observable.just(watchList);
                }
                return Observable.just(watchList).concatWith(getWatchlistFromServerHelper(sessionToken, accountId, page + 1));
            }
        });
    }

I'm calling this via a subscribe on my activity.

mSectionLogic.syncWatchlistSection(String.valueOf(account.getId()))
            .observeOn(AndroidSchedulers.mainThread())
            .subscribeOn(Schedulers.io())
            .subscribe(new Observer<WatchList>() {
              ... so on ...

This works but doesn't handle the paging. The response looks something like:

{
  page: 1,
  totalPages: 5,
  results: []
}

What I've done is it should return when page == totalPages . Except it doesn't combine the previous results and just returns the last page of results.

mRestAPI.fetchWatchlistOnServer is the Retrofit2 Get request that returns Observable<Watchlist> .

What am I doing wrong? How do I combine results with Observable? Preferably not in Lamda notation.

Other resources I looked at: - android rxjava2/retrofit2 chaining calls with pagination token

Answer 1

You need to build the Observable. You can use recursion but if you have large number of pages your calling stack might overflow. See the answer here

Something like

private Observable<WatchList> getWatchlistFromServerHelper(String sessionToken, String accountId, int startPage) {
    int index = startPage;
    BehaviorSubject<Integer> pagecontrol = BehaviorSubject.create(startPage);
    Observable<WatchList> ret = pageControl.asObservable()
                                           .concatMap(new Function<Integer,Observable<WatchList>>()
                                                     {
                                                       Observable<WatchList> apply(Integer page) {
                                                            mRestAPI.fetchWatchlistOnServer(sessionToken, accountId, page)
                                                                    .doOnNext(new Function<watchList -> 
                                                                            (watchList.getPage() == watchList.getTotalPages() ? 
                                                                             Observable.<WatchList>empty().doOnCompleted(()->pageControl.onCompleted()) :
                                                                             pageControl.onNext(page+1)));
                                                        }
                                                     }
                                                     );
    return ret;
}

should work.

Answer 2

My below sample code is working, here you need to make some modifications according to your requirements.

import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.support.v7.widget.LinearLayoutManager;
import android.support.v7.widget.RecyclerView;
import android.view.View;
import android.widget.ProgressBar;

import org.reactivestreams.Publisher;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

import butterknife.BindView;
import butterknife.ButterKnife;
import io.reactivex.Flowable;
import io.reactivex.android.schedulers.AndroidSchedulers;
import io.reactivex.annotations.NonNull;
import io.reactivex.disposables.CompositeDisposable;
import io.reactivex.functions.Consumer;
import io.reactivex.functions.Function;
import io.reactivex.processors.PublishProcessor;

/**
* Created by Akash on 11/11/17.
*/
public class MainActivity extends AppCompatActivity {

private final int VISIBLE_THRESHOLD = 1;
@BindView(R.id.recyclerView)
RecyclerView recyclerView;
@BindView(R.id.progressBar)
ProgressBar progressBar;


private CompositeDisposable mDisposable;
private PublishProcessor<Integer> mPaginator;
private RxPagingAdapter mAdapter;
private boolean isLoading = false;
private int pageNumber = 1;
private int lastVisibleItem, totalItemCount;
private LinearLayoutManager mLayoutManager;


@Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.activity_main);
    ButterKnife.bind(this);

    mDisposable = new CompositeDisposable();
    mPaginator = PublishProcessor.create();
    mLayoutManager = new LinearLayoutManager(this);
    recyclerView.setLayoutManager(mLayoutManager);
    mAdapter = new RxPagingAdapter();
    recyclerView.setAdapter(mAdapter);

    // register scroll listener
    recyclerView.addOnScrollListener(new RecyclerView.OnScrollListener() {
        @Override
        public void onScrolled(RecyclerView recyclerView,
                               int dx, int dy) {
            super.onScrolled(recyclerView, dx, dy);

            totalItemCount = mLayoutManager.getItemCount();
            lastVisibleItem = mLayoutManager
                    .findLastVisibleItemPosition();
            if (!isLoading
                    && totalItemCount <= (lastVisibleItem + VISIBLE_THRESHOLD)) {
                pageNumber++;
                mPaginator.onNext(pageNumber);
                isLoading = true;
            }
        }
    });
    subscribeApi();
}

@Override
protected void onDestroy() {
    super.onDestroy();
    if (!mDisposable.isDisposed())
        mDisposable.dispose();
}

/**
 * Adding to disposable.
 */
private void subscribeApi() {

    mDisposable.add(mPaginator
            .onBackpressureDrop()
            .concatMap(new Function<Integer, Publisher<List<String>>>() {
                @Override
                public Publisher<List<String>> apply(@NonNull Integer page) throws Exception {
                    isLoading = true;
                    progressBar.setVisibility(View.VISIBLE);
                    return apiResponse(page);
                }
            })
            .observeOn(AndroidSchedulers.mainThread())
            .subscribe(new Consumer<List<String>>() {
                @Override
                public void accept(@NonNull List<String> items) throws Exception {
                    mAdapter.addItems(items);
                    mAdapter.notifyDataSetChanged();
                    isLoading = false;
                    progressBar.setVisibility(View.INVISIBLE);
                }
            }));

    mPaginator.onNext(pageNumber);

}

/**
 * This is just sample.
 * You can call api with retrofit interface method which returns Flowable and there you go.
 */
private Flowable<List<String>> apiResponse(final int page) {
    return Flowable.just(true)
            .delay(3, TimeUnit.SECONDS)
            .map(new Function<Boolean, List<String>>() {
                @Override
                public List<String> apply(@NonNull Boolean value) throws Exception {
                    List<String> items = new ArrayList<>();
                    for (int i = 1; i <= 10; i++) {
                        items.add("Item " + (page * 10 + i));
                    }
                    return items;
                }
            });
}
}

Adapter class method

 void addItems(List<String> items) {
    this.itemArrayList.addAll(items);
 }

Source : GitHub