feat(pagination): smart pagination

PocketKit/Sources/PocketKit/Refresh/RefreshCoordinator.swift

@@ -26,7 +26,7 @@ class RefreshCoordinator {
     }
 
     func initialize() {
-        _ = taskScheduler.registerHandler(forTaskWithIdentifier: Self.taskID, using: .main) { [weak self] task in
+        _ = taskScheduler.registerHandler(forTaskWithIdentifier: Self.taskID, using: .global(qos: .background)) { [weak self] task in
             self?.refresh(task)
             self?.submitRequest()
         }

PocketKit/Sources/PocketKit/Root/RootViewModel.swift

@@ -104,6 +104,7 @@ class RootViewModel {
         tracker.addPersistentEntity(UserEntity(guid: session.guid, userID: session.userIdentifier))
         Log.setUserID(session.userIdentifier)
         source.refreshSaves()
+        source.refreshArchive()
     }
 
     private func tearDownSession() {

PocketKit/Sources/Sync/Operations/FetchArchive.swift

@@ -9,21 +9,18 @@ class FetchArchive: SyncOperation {
     private let space: Space
     private let events: SyncEvents
     private let initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>
-    private let maxItems: Int
     private let lastRefresh: LastRefresh
 
     init(
         apollo: ApolloClientProtocol,
         space: Space,
         events: SyncEvents,
         initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>,
-        maxItems: Int,
         lastRefresh: LastRefresh
     ) {
         self.apollo = apollo
         self.space = space
         self.events = events
-        self.maxItems = maxItems
         self.lastRefresh = lastRefresh
         self.initialDownloadState = initialDownloadState
     }
@@ -64,19 +61,19 @@ class FetchArchive: SyncOperation {
     }
 
     private func fetchArchive() async throws {
-        var pagination = PaginationSpec(maxItems: maxItems)
+        var pagination = PaginationSpec(maxItems: SyncConstants.Archive.firstLoadMaxCount, pageSize: SyncConstants.Archive.initalPageSize)
 
         repeat {
             let result = try await fetchPage(pagination)
 
             if case .started = initialDownloadState.value,
                let totalCount = result.data?.user?.savedItems?.totalCount,
                pagination.cursor == nil {
-                initialDownloadState.send(.paginating(totalCount: totalCount))
+                initialDownloadState.send(.paginating(totalCount: totalCount > pagination.maxItems ? pagination.maxItems : totalCount))
             }
 
             try await updateLocalStorage(result: result)
-            pagination = pagination.nextPage(result: result)
+            pagination = pagination.nextPage(result: result, pageSize: SyncConstants.Archive.pageSize)
         } while pagination.shouldFetchNextPage
 
         initialDownloadState.send(.completed)
@@ -86,7 +83,7 @@ class FetchArchive: SyncOperation {
         let query = FetchArchiveQuery(
             pagination: .some(PaginationInput(
                 after: pagination.cursor ?? .none,
-                first: .some(pagination.maxItems)
+                first: .some(pagination.pageSize)
             )),
             filter: .none,
             sort: .some(SavedItemsSort(sortBy: .init(.archivedAt), sortOrder: .init(.desc)))
@@ -134,28 +131,31 @@ class FetchArchive: SyncOperation {
         let cursor: String?
         let shouldFetchNextPage: Bool
         let maxItems: Int
+        let pageSize: Int
 
-        init(maxItems: Int) {
-            self.init(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems)
+        init(maxItems: Int, pageSize: Int) {
+            self.init(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems, pageSize: pageSize)
         }
 
-        private init(cursor: String?, shouldFetchNextPage: Bool, maxItems: Int) {
+        private init(cursor: String?, shouldFetchNextPage: Bool, maxItems: Int, pageSize: Int) {
             self.cursor = cursor
             self.shouldFetchNextPage = shouldFetchNextPage
             self.maxItems = maxItems
+            self.pageSize = pageSize
         }
 
-        func nextPage(result: GraphQLResult<FetchArchiveQuery.Data>) -> PaginationSpec {
+        func nextPage(result: GraphQLResult<FetchArchiveQuery.Data>, pageSize: Int) -> PaginationSpec {
             guard let savedItems = result.data?.user?.savedItems,
                   let itemCount = savedItems.edges?.count,
                   let endCursor = savedItems.pageInfo.endCursor else {
-                      return PaginationSpec(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems)
+                      return PaginationSpec(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems, pageSize: pageSize)
                   }
 
             return PaginationSpec(
                 cursor: endCursor,
                 shouldFetchNextPage: savedItems.pageInfo.hasNextPage && itemCount < maxItems,
-                maxItems: maxItems - itemCount
+                maxItems: maxItems - itemCount,
+                pageSize: pageSize
             )
         }
     }

PocketKit/Sources/Sync/Operations/FetchSaves.swift

@@ -10,7 +10,6 @@ class FetchSaves: SyncOperation {
     private let space: Space
     private let events: SyncEvents
     private let initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>
-    private let maxItems: Int
     private let lastRefresh: LastRefresh
 
     init(
@@ -19,21 +18,19 @@ class FetchSaves: SyncOperation {
         space: Space,
         events: SyncEvents,
         initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>,
-        maxItems: Int,
         lastRefresh: LastRefresh
     ) {
         self.user = user
         self.apollo = apollo
         self.space = space
         self.events = events
-        self.maxItems = maxItems
         self.lastRefresh = lastRefresh
         self.initialDownloadState = initialDownloadState
     }
 
     func execute() async -> SyncOperationResult {
         do {
-            try await fetchList()
+            try await fetchSaves()
             try await fetchTags()
 
             lastRefresh.refreshedSaves()
@@ -67,8 +64,8 @@ class FetchSaves: SyncOperation {
         }
     }
 
-    private func fetchList() async throws {
-        var pagination = PaginationSpec(maxItems: maxItems)
+    private func fetchSaves() async throws {
+        var pagination = PaginationSpec(maxItems: SyncConstants.Saves.firstLoadMaxCount, pageSize: SyncConstants.Saves.initalPageSize)
 
         repeat {
             let result = try await fetchPage(pagination)
@@ -79,11 +76,11 @@ class FetchSaves: SyncOperation {
             if case .started = initialDownloadState.value,
                let totalCount = result.data?.user?.savedItems?.totalCount,
                pagination.cursor == nil {
-                initialDownloadState.send(.paginating(totalCount: totalCount))
+                initialDownloadState.send(.paginating(totalCount: totalCount > pagination.maxItems ? pagination.maxItems : totalCount))
             }
 
             try await updateLocalStorage(result: result)
-            pagination = pagination.nextPage(result: result)
+            pagination = pagination.nextPage(result: result, pageSize: SyncConstants.Saves.pageSize)
         } while pagination.shouldFetchNextPage
 
         initialDownloadState.send(.completed)
@@ -111,7 +108,7 @@ class FetchSaves: SyncOperation {
         let query = FetchSavesQuery(
             pagination: .some(PaginationInput(
                 after: pagination.cursor ?? .none,
-                first: .some(pagination.maxItems)
+                first: .some(pagination.pageSize)
             )),
             savedItemsFilter: .none
         )
@@ -168,28 +165,31 @@ class FetchSaves: SyncOperation {
         let cursor: String?
         let shouldFetchNextPage: Bool
         let maxItems: Int
+        let pageSize: Int
 
-        init(maxItems: Int) {
-            self.init(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems)
+        init(maxItems: Int, pageSize: Int) {
+            self.init(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems, pageSize: pageSize)
         }
 
-        private init(cursor: String?, shouldFetchNextPage: Bool, maxItems: Int) {
+        private init(cursor: String?, shouldFetchNextPage: Bool, maxItems: Int, pageSize: Int) {
             self.cursor = cursor
             self.shouldFetchNextPage = shouldFetchNextPage
             self.maxItems = maxItems
+            self.pageSize = pageSize
         }
 
-        func nextPage(result: GraphQLResult<FetchSavesQuery.Data>) -> PaginationSpec {
+        func nextPage(result: GraphQLResult<FetchSavesQuery.Data>, pageSize: Int) -> PaginationSpec {
             guard let savedItems = result.data?.user?.savedItems,
                   let itemCount = savedItems.edges?.count,
                   let endCursor = savedItems.pageInfo.endCursor else {
-                      return PaginationSpec(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems)
+                      return PaginationSpec(cursor: nil, shouldFetchNextPage: false, maxItems: maxItems, pageSize: pageSize)
                   }
 
             return PaginationSpec(
                 cursor: endCursor,
                 shouldFetchNextPage: savedItems.pageInfo.hasNextPage && itemCount < maxItems,
-                maxItems: maxItems - itemCount
+                maxItems: maxItems - itemCount,
+                pageSize: pageSize
             )
         }
     }

PocketKit/Sources/Sync/Operations/SyncOperationFactory.swift

@@ -12,7 +12,6 @@ protocol SyncOperationFactory {
         space: Space,
         events: SyncEvents,
         initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>,
-        maxItems: Int,
         lastRefresh: LastRefresh
     ) -> SyncOperation
 
@@ -21,7 +20,6 @@ protocol SyncOperationFactory {
         space: Space,
         events: SyncEvents,
         initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>,
-        maxItems: Int,
         lastRefresh: LastRefresh
     ) -> SyncOperation
 
@@ -53,7 +51,6 @@ class OperationFactory: SyncOperationFactory {
         space: Space,
         events: SyncEvents,
         initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>,
-        maxItems: Int,
         lastRefresh: LastRefresh
     ) -> SyncOperation {
         return FetchSaves(
@@ -62,7 +59,6 @@ class OperationFactory: SyncOperationFactory {
             space: space,
             events: events,
             initialDownloadState: initialDownloadState,
-            maxItems: maxItems,
             lastRefresh: lastRefresh
         )
     }
@@ -72,15 +68,13 @@ class OperationFactory: SyncOperationFactory {
         space: Space,
         events: SyncEvents,
         initialDownloadState: CurrentValueSubject<InitialDownloadState, Never>,
-        maxItems: Int,
         lastRefresh: LastRefresh
     ) -> SyncOperation {
         return FetchArchive(
             apollo: apollo,
             space: space,
             events: events,
             initialDownloadState: initialDownloadState,
-            maxItems: maxItems,
             lastRefresh: lastRefresh
         )
     }

PocketKit/Sources/Sync/PocketSource.swift

@@ -204,7 +204,7 @@ public class PocketSource: Source {
 
 // MARK: - Saves/Archive items
 extension PocketSource {
-    public func refreshSaves(maxItems: Int = 400, completion: (() -> Void)? = nil) {
+    public func refreshSaves(maxItems: Int, completion: (() -> Void)? = nil) {
         if lastRefresh.lastRefreshSaves == nil {
             initialSavesDownloadState.send(.started)
         }
@@ -215,14 +215,13 @@ extension PocketSource {
             space: space,
             events: _events,
             initialDownloadState: initialSavesDownloadState,
-            maxItems: maxItems,
             lastRefresh: lastRefresh
         )
 
         enqueue(operation: operation, task: .fetchSaves(maxItems: maxItems), completion: completion)
     }
 
-    public func refreshArchive(maxItems: Int = 400, completion: (() -> Void)? = nil) {
+    public func refreshArchive(maxItems: Int, completion: (() -> Void)? = nil) {
         if lastRefresh.lastRefreshArchive == nil {
             initialArchiveDownloadState.send(.started)
         }
@@ -232,7 +231,6 @@ extension PocketSource {
             space: space,
             events: _events,
             initialDownloadState: initialArchiveDownloadState,
-            maxItems: maxItems,
             lastRefresh: lastRefresh
         )
 
@@ -523,7 +521,6 @@ extension PocketSource {
                     space: space,
                     events: _events,
                     initialDownloadState: initialSavesDownloadState,
-                    maxItems: maxItems,
                     lastRefresh: lastRefresh
                 )
                 enqueue(operation: operation, persistentTask: persistentTask)
@@ -533,7 +530,6 @@ extension PocketSource {
                     space: space,
                     events: _events,
                     initialDownloadState: initialArchiveDownloadState,
-                    maxItems: maxItems,
                     lastRefresh: lastRefresh
                 )
                 enqueue(operation: operation, persistentTask: persistentTask)

PocketKit/Sources/Sync/Slates/SlateService.swift

@@ -21,7 +21,7 @@ class APISlateService: SlateService {
     }
 
     func fetchSlateLineup(_ identifier: String) async throws {
-        let query = GetSlateLineupQuery(lineupID: identifier, maxRecommendations: 5)
+        let query = GetSlateLineupQuery(lineupID: identifier, maxRecommendations: SyncConstants.Home.recomendationsPerSlateFromSlateLineup)
 
         guard let remote = try await apollo.fetch(query: query).data?.getSlateLineup else {
             Log.capture(message: "Error loading slate lineup")
@@ -32,7 +32,7 @@ class APISlateService: SlateService {
     }
 
     func fetchSlate(_ slateID: String) async throws {
-        let query = GetSlateQuery(slateID: slateID, recommendationCount: 25)
+        let query = GetSlateQuery(slateID: slateID, recommendationCount: SyncConstants.Home.recomendationsPerSlateDetail)
 
         guard let remote = try await apollo.fetch(query: query)
             .data?.getSlate.fragments.slateParts else {

PocketKit/Sources/Sync/Source.swift

@@ -95,18 +95,18 @@ public protocol Source {
 
 public extension Source {
     func refreshSaves(completion: (() -> Void)?) {
-        self.refreshSaves(maxItems: 400, completion: completion)
+        self.refreshSaves(maxItems: SyncConstants.Saves.firstLoadMaxCount, completion: completion)
     }
 
     func refreshSaves() {
-        self.refreshSaves(maxItems: 400, completion: nil)
+        self.refreshSaves(maxItems: SyncConstants.Saves.firstLoadMaxCount, completion: nil)
     }
 
     func refreshArchive(completion: (() -> Void)?) {
-        self.refreshArchive(maxItems: 400, completion: completion)
+        self.refreshArchive(maxItems: SyncConstants.Archive.firstLoadMaxCount, completion: completion)
     }
 
     func refreshArchive() {
-        self.refreshArchive(maxItems: 400, completion: nil)
+        self.refreshArchive(maxItems: SyncConstants.Archive.firstLoadMaxCount, completion: nil)
     }
 }

PocketKit/Sources/Sync/SyncConstants.swift

@@ -0,0 +1,40 @@
+//
+//  File.swift
+//  
+//
+//  Created by Daniel Brooks on 3/9/23.
+//
+
+import Foundation
+
+struct SyncConstants {
+    struct Saves {
+        /// How many saves we load when a user logs in. As they save and use pocket they may accumilate more, but we only download the amount of latest saves here to start.
+        static let firstLoadMaxCount = 500
+
+        /// How many saves we should load on the first login request for saves, we use a small value here so the user immediately sees content.
+        static let initalPageSize = 15
+
+        /// How many saves to load per subsequent page until we hit our load count.
+        static let pageSize = 30
+    }
+
+    struct Archive {
+        /// How many archives we load when a user logs in. As they save and use pocket they may accumilate more, but we only download the amount of latest archives here to start.
+        static let firstLoadMaxCount = 500
+
+        /// How many archives we should load on the first login request for archives, we use a small value here so the user immediately sees content.
+        static let initalPageSize = 15
+
+        /// How many archives to load per subsequent page until we hit our load count.
+        static let pageSize = 30
+    }
+
+    struct Home {
+        /// How many recomendations to pull in when we load them via getSlateLineup (ie. Home)
+        static let recomendationsPerSlateFromSlateLineup = 5
+
+        /// How many recomendations to pull in when we load them via getSlate (ie. a detail view)
+        static let recomendationsPerSlateDetail = 25
+    }
+}