From c25afdc203d8e1aca772bd3801707725cd96b645 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 6 Dec 2020 13:21:25 +0100 Subject: [PATCH] Chunked sync --- cps/kobo.py | 86 +++++++++++++++++++++------------------ cps/services/SyncToken.py | 10 ++++- 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index a9c0f936..90d0182e 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -43,6 +43,7 @@ from flask_login import current_user from werkzeug.datastructures import Headers from sqlalchemy import func from sqlalchemy.sql.expression import and_, or_ +from sqlalchemy.orm import load_only from sqlalchemy.exc import StatementError import requests @@ -56,6 +57,8 @@ KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]} KOBO_STOREAPI_URL = "https://storeapi.kobo.com" KOBO_IMAGEHOST_URL = "https://kbimages1-a.akamaihd.net" +SYNC_ITEM_LIMIT = 5 + kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo) kobo_auth.register_url_value_preprocessor(kobo) @@ -142,68 +145,70 @@ def HandleSyncRequest(): new_books_last_modified = sync_token.books_last_modified new_books_last_created = sync_token.books_last_created new_reading_state_last_modified = sync_token.reading_state_last_modified + new_archived_last_modified = datetime.datetime.min sync_results = [] # We reload the book database so that the user get's a fresh view of the library # in case of external changes (e.g: adding a book through Calibre). calibre_db.reconnect_db(config, ub.app_DB_path) - archived_books = ( - ub.session.query(ub.ArchivedBook) - .filter(ub.ArchivedBook.user_id == int(current_user.id)) - .all() - ) - - # We join-in books that have had their Archived bit recently modified in order to either: - # * Restore them to the user's device. - # * Delete them from the user's device. - # (Ideally we would use a join for this logic, however cross-database joins don't look trivial in SqlAlchemy.) - recently_restored_or_archived_books = [] - archived_book_ids = {} - new_archived_last_modified = datetime.datetime.min - for archived_book in archived_books: - if archived_book.last_modified > sync_token.archive_last_modified: - recently_restored_or_archived_books.append(archived_book.book_id) - if archived_book.is_archived: - archived_book_ids[archived_book.book_id] = True - new_archived_last_modified = max( - new_archived_last_modified, archived_book.last_modified) - - # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast. - # It looks like it's treating the db.Books.last_modified field as a string and may fail - # the comparison because of the +00:00 suffix. changed_entries = ( - calibre_db.session.query(db.Books) - .join(db.Data) - .filter(or_(func.datetime(db.Books.last_modified) > sync_token.books_last_modified, - db.Books.id.in_(recently_restored_or_archived_books))) + calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived) + .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id) + .filter(db.Books.last_modified >= sync_token.books_last_modified) + .filter(db.Books.id>sync_token.books_last_id) .filter(db.Data.format.in_(KOBO_FORMATS)) - .all() + # .filter(ub.ArchivedBook.is_archived == 0) + .order_by(db.Books.last_modified) + .order_by(db.Books.id) + .limit(SYNC_ITEM_LIMIT) ) reading_states_in_new_entitlements = [] for book in changed_entries: - kobo_reading_state = get_or_create_reading_state(book.id) + kobo_reading_state = get_or_create_reading_state(book.Books.id) entitlement = { - "BookEntitlement": create_book_entitlement(book, archived=(book.id in archived_book_ids)), - "BookMetadata": get_metadata(book), + "BookEntitlement": create_book_entitlement(book.Books, archived=(book.is_archived == True)), + "BookMetadata": get_metadata(book.Books), } if kobo_reading_state.last_modified > sync_token.reading_state_last_modified: - entitlement["ReadingState"] = get_kobo_reading_state_response(book, kobo_reading_state) + entitlement["ReadingState"] = get_kobo_reading_state_response(book.Books, kobo_reading_state) new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified) - reading_states_in_new_entitlements.append(book.id) + reading_states_in_new_entitlements.append(book.Books.id) - if book.timestamp > sync_token.books_last_created: + if book.Books.timestamp > sync_token.books_last_created: sync_results.append({"NewEntitlement": entitlement}) else: sync_results.append({"ChangedEntitlement": entitlement}) new_books_last_modified = max( - book.last_modified, new_books_last_modified + book.Books.last_modified, new_books_last_modified ) - new_books_last_created = max(book.timestamp, new_books_last_created) + new_books_last_created = max(book.Books.timestamp, new_books_last_created) + max_change = (changed_entries + .from_self() + .filter(ub.ArchivedBook.is_archived) + .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()) + .first() + ) + if max_change: + max_change = max_change.last_modified + else: + max_change = new_archived_last_modified + new_archived_last_modified = max(new_archived_last_modified, max_change) + + # no. of books returned + book_count = changed_entries.count() + + # last entry: + if book_count: + books_last_id = changed_entries.all()[-1].Books.id or -1 + else: + books_last_id = -1 + + # generate reading state data changed_reading_states = ( ub.session.query(ub.KoboReadingState) .filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified, @@ -225,11 +230,12 @@ def HandleSyncRequest(): sync_token.books_last_modified = new_books_last_modified sync_token.archive_last_modified = new_archived_last_modified sync_token.reading_state_last_modified = new_reading_state_last_modified + sync_token.books_last_id = books_last_id - return generate_sync_response(sync_token, sync_results) + return generate_sync_response(sync_token, sync_results, book_count) -def generate_sync_response(sync_token, sync_results): +def generate_sync_response(sync_token, sync_results, set_cont=False): extra_headers = {} if config.config_kobo_proxy: # Merge in sync results from the official Kobo store. @@ -245,6 +251,8 @@ def generate_sync_response(sync_token, sync_results): except Exception as e: log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e)) + if set_cont: + extra_headers["x-kobo-sync"] = "continue" sync_token.to_headers(extra_headers) response = make_response(jsonify(sync_results), extra_headers) diff --git a/cps/services/SyncToken.py b/cps/services/SyncToken.py index f6db960b..4ad5fa2c 100644 --- a/cps/services/SyncToken.py +++ b/cps/services/SyncToken.py @@ -85,6 +85,7 @@ class SyncToken: "archive_last_modified": {"type": "string"}, "reading_state_last_modified": {"type": "string"}, "tags_last_modified": {"type": "string"}, + "books_last_id": {"type": "integer", "optional": True} }, } @@ -96,6 +97,7 @@ class SyncToken: archive_last_modified=datetime.min, reading_state_last_modified=datetime.min, tags_last_modified=datetime.min, + books_last_id=-1 ): self.raw_kobo_store_token = raw_kobo_store_token self.books_last_created = books_last_created @@ -103,6 +105,7 @@ class SyncToken: self.archive_last_modified = archive_last_modified self.reading_state_last_modified = reading_state_last_modified self.tags_last_modified = tags_last_modified + self.books_last_id = books_last_id @staticmethod def from_headers(headers): @@ -137,6 +140,7 @@ class SyncToken: archive_last_modified = get_datetime_from_json(data_json, "archive_last_modified") reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified") tags_last_modified = get_datetime_from_json(data_json, "tags_last_modified") + books_last_id = data_json["books_last_id"] except TypeError: log.error("SyncToken timestamps don't parse to a datetime.") return SyncToken(raw_kobo_store_token=raw_kobo_store_token) @@ -147,7 +151,8 @@ class SyncToken: books_last_modified=books_last_modified, archive_last_modified=archive_last_modified, reading_state_last_modified=reading_state_last_modified, - tags_last_modified=tags_last_modified + tags_last_modified=tags_last_modified, + books_last_id=books_last_id ) def set_kobo_store_header(self, store_headers): @@ -170,7 +175,8 @@ class SyncToken: "books_last_created": to_epoch_timestamp(self.books_last_created), "archive_last_modified": to_epoch_timestamp(self.archive_last_modified), "reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified), - "tags_last_modified": to_epoch_timestamp(self.tags_last_modified) + "tags_last_modified": to_epoch_timestamp(self.tags_last_modified), + "books_last_id":self.books_last_id }, } return b64encode_json(token)