Merge branch 'master' into cover_thumbnail

# Conflicts:
#	test/Calibre-Web TestSummary_Linux.html

SECURITY.md

@@ -24,20 +24,20 @@ To receive fixes for security vulnerabilities it is required to always upgrade t
 | V 0.6.13      | JavaScript could get executed  in the shelf title                                                                  ||
 | V 0.6.13      | Login with the old session cookie after logout. Thanks to @ibarrionuevo                                            ||
 | V 0.6.14      | CSRF was possible. Thanks to @mik317 and Hagai Wechsler (WhiteSource)                                              |CVE-2021-25965|
-| V 0.6.14      | Migrated some routes to POST-requests (CSRF protection). Thanks to @scara31                                        ||
-| V 0.6.15      | Fix for "javascript:" script links in identifier. Thanks to @scara31                                               ||
+| V 0.6.14      | Migrated some routes to POST-requests (CSRF protection). Thanks to @scara31                                        |CVE-2021-4164|
+| V 0.6.15      | Fix for "javascript:" script links in identifier. Thanks to @scara31                                               |CVE-2021-4170|
 | V 0.6.15      | Cross-Site Scripting vulnerability on uploaded cover file names. Thanks to @ibarrionuevo                           ||
 | V 0.6.15      | Creating public shelfs is now denied if user is missing the edit public shelf right. Thanks to @ibarrionuevo       ||
 | V 0.6.15      | Changed error message in case of trying to delete a shelf unauthorized. Thanks to @ibarrionuevo                    ||
-| V 0.6.16      | JavaScript could get executed on authors page. Thanks to @alicaz                                                   ||
-| V 0.6.16      | Localhost can no longer be used to upload covers. Thanks to @scara31                                               ||
-| V 0.6.16      | Another case where public shelfs could be created without permission is prevented. Thanks to @nhiephon             ||
-| V 0.6.16      | It's prevented to get the name of a private shelfs. Thanks to @nhiephon                                            ||
-| V 0.6.17      | The SSRF Protection can no longer be bypassed via an HTTP redirect. Thanks to @416e6e61                            ||
-| V 0.6.17      | The SSRF Protection can no longer be bypassed via 0.0.0.0 and it's ipv6 equivalent. Thanks to @r0hanSH             ||
+| V 0.6.16      | JavaScript could get executed on authors page. Thanks to @alicaz                                                   |CVE-2022-0352|
+| V 0.6.16      | Localhost can no longer be used to upload covers. Thanks to @scara31                                               |CVE-2022-0339|
+| V 0.6.16      | Another case where public shelfs could be created without permission is prevented. Thanks to @nhiephon             |CVE-2022-0273|
+| V 0.6.16      | It's prevented to get the name of a private shelfs. Thanks to @nhiephon                                            |CVE-2022-0405|
+| V 0.6.17      | The SSRF Protection can no longer be bypassed via an HTTP redirect. Thanks to @416e6e61                            |CVE-2022-0767|
+| V 0.6.17      | The SSRF Protection can no longer be bypassed via 0.0.0.0 and it's ipv6 equivalent. Thanks to @r0hanSH             |CVE-2022-0766|
 | V 0.6.18      | Possible SQL Injection is prevented in user table  Thanks to Iman Sharafaldin (Forward Security)                   ||
-| V 0.6.18      | The SSRF protection no longer can be bypassed by IPV6/IPV4 embedding. Thanks to  @416e6e61                         ||
-| V 0.6.18      | The SSRF protection no longer can be bypassed to connect to other servers in the local network. Thanks to @michaellrowley ||
+| V 0.6.18      | The SSRF protection no longer can be bypassed by IPV6/IPV4 embedding. Thanks to  @416e6e61                         |CVE-2022-0939|
+| V 0.6.18      | The SSRF protection no longer can be bypassed to connect to other servers in the local network. Thanks to @michaellrowley |CVE-2022-0990|
 
 
 ## Statement regarding Log4j (CVE-2021-44228 and related)

cps/comic.py

@@ -130,7 +130,9 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
                 series=loaded_metadata.series or "",
                 series_id=loaded_metadata.issue or "",
                 languages=loaded_metadata.language,
-                publisher="")
+                publisher="",
+                pubdate="",
+                identifiers=[])
 
     return BookMeta(
         file_path=tmp_file_path,
@@ -143,4 +145,6 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
         series="",
         series_id="",
         languages="",
-        publisher="")
+        publisher="",
+        pubdate="",
+        identifiers=[])

cps/constants.py

@@ -161,7 +161,7 @@ def selected_roles(dictionary):
 
 # :rtype: BookMeta
 BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '
-                                  'series_id, languages, publisher')
+                                  'series_id, languages, publisher, pubdate, identifiers')
 
 STABLE_VERSION = {'version': '0.6.19 Beta'}
 

cps/db.py

@@ -903,9 +903,20 @@ class CalibreDB:
                     .join(books_languages_link).join(Books)\
                     .filter(self.common_filters(return_all_languages=return_all_languages)) \
                     .group_by(text('books_languages_link.lang_code')).all()
+            tags = list()
             for lang in languages:
-                lang[0].name = isoLanguages.get_language_name(get_locale(), lang[0].lang_code)
-            return sorted(languages, key=lambda x: x[0].name, reverse=reverse_order)
+                tag = Category(isoLanguages.get_language_name(get_locale(), lang[0].lang_code), lang[0].lang_code)
+                tags.append([tag, lang[1]])
+            # Append all books without language to list
+            if not return_all_languages:
+                no_lang_count = (self.session.query(Books)
+                                 .outerjoin(books_languages_link).outerjoin(Languages)
+                                 .filter(Languages.lang_code == None)
+                                 .filter(self.common_filters())
+                                 .count())
+                if no_lang_count:
+                    tags.append([Category(_("None"), "none"), no_lang_count])
+            return sorted(tags, key=lambda x: x[0].name, reverse=reverse_order)
         else:
             if not languages:
                 languages = self.session.query(Languages) \
@@ -977,3 +988,22 @@ def lcase(s):
         _log = logger.create()
         _log.error_or_exception(ex)
         return s.lower()
+
+
+class Category:
+    name = None
+    id = None
+    count = None
+    rating = None
+
+    def __init__(self, name, cat_id, rating=None):
+        self.name = name
+        self.id = cat_id
+        self.rating = rating
+        self.count = 1
+
+'''class Count:
+    count = None
+
+    def __init__(self, count):
+        self.count = count'''

cps/editbooks.py

@@ -984,8 +984,13 @@ def create_book_on_upload(modify_date, meta):
     # combine path and normalize path from Windows systems
     path = os.path.join(author_dir, title_dir).replace('\\', '/')
 
+    try:
+        pubdate = datetime.strptime(meta.pubdate[:10], "%Y-%m-%d")
+    except:
+        pubdate = datetime(101, 1, 1)
+
     # Calibre adds books with utc as timezone
-    db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
+    db_book = db.Books(title, "", sort_authors, datetime.utcnow(), pubdate,
                        '1', datetime.utcnow(), path, meta.cover, db_author, [], "")
 
     modify_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session,
@@ -1018,6 +1023,16 @@ def create_book_on_upload(modify_date, meta):
 
     # flush content, get db_book.id available
     calibre_db.session.flush()
+
+    # Handle identifiers now that db_book.id is available
+    identifier_list = []
+    for type_key, type_value in meta.identifiers:
+        identifier_list.append(db.Identifiers(type_value, type_key, db_book.id))
+    modification, warning = modify_identifiers(identifier_list, db_book.identifiers, calibre_db.session)
+    if warning:
+        flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning")
+    modify_date |= modification
+
     return db_book, input_authors, title_dir, renamed_authors
 
 

cps/epub.py

@@ -63,13 +63,15 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 
     epub_metadata = {}
 
-    for s in ['title', 'description', 'creator', 'language', 'subject']:
+    for s in ['title', 'description', 'creator', 'language', 'subject', 'publisher', 'date']:
         tmp = p.xpath('dc:%s/text()' % s, namespaces=ns)
         if len(tmp) > 0:
             if s == 'creator':
                 epub_metadata[s] = ' & '.join(split_authors(tmp))
             elif s == 'subject':
                 epub_metadata[s] = ', '.join(tmp)
+            elif s == 'date':
+                epub_metadata[s] = tmp[0][:10]
             else:
                 epub_metadata[s] = tmp[0]
         else:
@@ -78,6 +80,12 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     if epub_metadata['subject'] == 'Unknown':
         epub_metadata['subject'] = ''
 
+    if epub_metadata['publisher'] == u'Unknown':
+        epub_metadata['publisher'] = ''
+
+    if epub_metadata['date'] == u'Unknown':
+        epub_metadata['date'] = ''
+
     if epub_metadata['description'] == u'Unknown':
         description = tree.xpath("//*[local-name() = 'description']/text()")
         if len(description) > 0:
@@ -92,6 +100,14 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 
     cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path)
 
+    identifiers = []
+    for node in p.xpath('dc:identifier', namespaces=ns):
+        identifier_name=node.attrib.values()[-1];
+        identifier_value=node.text;
+        if identifier_name in ('uuid','calibre'):
+            continue;
+        identifiers.append( [identifier_name, identifier_value] )
+
     if not epub_metadata['title']:
         title = original_file_name
     else:
@@ -108,7 +124,9 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
         series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
         series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
         languages=epub_metadata['language'],
-        publisher="")
+        publisher=epub_metadata['publisher'].encode('utf-8').decode('utf-8'),
+        pubdate=epub_metadata['date'],
+        identifiers=identifiers)
 
 
 def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):

cps/fb2.py

@@ -77,4 +77,6 @@ def get_fb2_info(tmp_file_path, original_file_extension):
         series="",
         series_id="",
         languages="",
-        publisher="")
+        publisher="",
+        pubdate="",
+        identifiers=[])

cps/metadata_provider/amazon.py

@@ -19,11 +19,13 @@
 import concurrent.futures
 import requests
 from bs4 import BeautifulSoup as BS  # requirement
+from typing import List, Optional
 
 try:
     import cchardet #optional for better speed
 except ImportError:
     pass
+from cps import logger
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 import cps.logger as logger
 
@@ -31,6 +33,9 @@ import cps.logger as logger
 from operator import itemgetter
 log = logger.create()
 
+log = logger.create()
+
+
 class Amazon(Metadata):
     __name__ = "Amazon"
     __id__ = "amazon"
@@ -49,17 +54,21 @@ class Amazon(Metadata):
 
     def search(
         self, query: str, generic_cover: str = "", locale: str = "en"
-    ):
+    ) -> Optional[List[MetaRecord]]:
         #timer=time()
         def inner(link, index) -> [dict, int]:
-            try:
-                with self.session as session:
-                    r = session.get(f"https://www.amazon.com{link}")
+            with self.session as session:
+                try:
+                    r = session.get(f"https://www.amazon.com/{link}")
                     r.raise_for_status()
-                    long_soup = BS(r.text, "lxml")  #~4sec :/
-                    soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
-                    if soup2 is None:
-                        return
+                except Exception as ex:
+                    log.warning(ex)
+                    return
+                long_soup = BS(r.text, "lxml")  #~4sec :/
+                soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
+                if soup2 is None:
+                    return
+                try:
                     match = MetaRecord(
                         title = "",
                         authors = "",
@@ -104,27 +113,29 @@ class Amazon(Metadata):
                     except (AttributeError, TypeError):
                         match.cover = ""
                     return match, index
-            except Exception as e:
-                log.error_or_exception(e)
-                return
+                except Exception as e:
+                    log.error_or_exception(e)
+                    return
 
         val = list()
-        try:
-            if self.active:
+        if self.active:
+            try:
                 results = self.session.get(
-                    f"https://www.amazon.com/s?k={query.replace(' ', '+')}"
-                    f"&i=digital-text&sprefix={query.replace(' ', '+')}"
+                    f"https://www.amazon.com/s?k={query.replace(' ', '+')}&i=digital-text&sprefix={query.replace(' ', '+')}"
                     f"%2Cdigital-text&ref=nb_sb_noss",
                     headers=self.headers)
                 results.raise_for_status()
-                soup = BS(results.text, 'html.parser')
-                links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
-                              soup.findAll("div", attrs={"data-component-type": "s-search-result"})]
-                with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
-                    fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])}
-                    val = list(map(lambda x: x.result(), concurrent.futures.as_completed(fut)))
-            result = list(filter(lambda x: x, val))
-            return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance
-        except requests.exceptions.HTTPError as e:
-            log.error_or_exception(e)
-            return []
+            except requests.exceptions.HTTPError as e:
+                log.error_or_exception(e)
+                return None
+            except Exception as e:
+                log.warning(e)
+                return None
+            soup = BS(results.text, 'html.parser')
+            links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
+                          soup.findAll("div", attrs={"data-component-type": "s-search-result"})]
+            with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+                fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])}
+                val = list(map(lambda x : x.result() ,concurrent.futures.as_completed(fut)))
+        result = list(filter(lambda x: x, val))
+        return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance

cps/metadata_provider/comicvine.py

@@ -21,8 +21,11 @@ from typing import Dict, List, Optional
 from urllib.parse import quote
 
 import requests
+from cps import logger
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
 
 class ComicVine(Metadata):
     __name__ = "ComicVine"
@@ -46,10 +49,15 @@ class ComicVine(Metadata):
             if title_tokens:
                 tokens = [quote(t.encode("utf-8")) for t in title_tokens]
                 query = "%20".join(tokens)
-            result = requests.get(
-                f"{ComicVine.BASE_URL}{query}{ComicVine.QUERY_PARAMS}",
-                headers=ComicVine.HEADERS,
-            )
+            try:
+                result = requests.get(
+                    f"{ComicVine.BASE_URL}{query}{ComicVine.QUERY_PARAMS}",
+                    headers=ComicVine.HEADERS,
+                )
+                result.raise_for_status()
+            except Exception as e:
+                log.warning(e)
+                return None
             for result in result.json()["results"]:
                 match = self._parse_search_result(
                     result=result, generic_cover=generic_cover, locale=locale

cps/metadata_provider/douban.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2022 xlivevil
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+import re
+from concurrent import futures
+from typing import List, Optional
+
+import requests
+from html2text import HTML2Text
+from lxml import etree
+
+from cps import logger
+from cps.services.Metadata import Metadata, MetaRecord, MetaSourceInfo
+
+log = logger.create()
+
+
+def html2text(html: str) -> str:
+
+    h2t = HTML2Text()
+    h2t.body_width = 0
+    h2t.single_line_break = True
+    h2t.emphasis_mark = "*"
+    return h2t.handle(html)
+
+
+class Douban(Metadata):
+    __name__ = "豆瓣"
+    __id__ = "douban"
+    DESCRIPTION = "豆瓣"
+    META_URL = "https://book.douban.com/"
+    SEARCH_URL = "https://www.douban.com/j/search"
+
+    ID_PATTERN = re.compile(r"sid: (?P<id>\d+),")
+    AUTHORS_PATTERN = re.compile(r"作者|译者")
+    PUBLISHER_PATTERN = re.compile(r"出版社")
+    SUBTITLE_PATTERN = re.compile(r"副标题")
+    PUBLISHED_DATE_PATTERN = re.compile(r"出版年")
+    SERIES_PATTERN = re.compile(r"丛书")
+    IDENTIFIERS_PATTERN = re.compile(r"ISBN|统一书号")
+
+    TITTLE_XPATH = "//span[@property='v:itemreviewed']"
+    COVER_XPATH = "//a[@class='nbg']"
+    INFO_XPATH = "//*[@id='info']//span[@class='pl']"
+    TAGS_XPATH = "//a[contains(@class, 'tag')]"
+    DESCRIPTION_XPATH = "//div[@id='link-report']//div[@class='intro']"
+    RATING_XPATH = "//div[@class='rating_self clearfix']/strong"
+
+    session = requests.Session()
+    session.headers = {
+        'user-agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56',
+    }
+
+    def search(
+        self, query: str, generic_cover: str = "", locale: str = "en"
+    ) -> Optional[List[MetaRecord]]:
+        if self.active:
+            log.debug(f"starting search {query} on douban")
+            if title_tokens := list(
+                self.get_title_tokens(query, strip_joiners=False)
+            ):
+                query = "+".join(title_tokens)
+
+            try:
+                r = self.session.get(
+                    self.SEARCH_URL, params={"cat": 1001, "q": query}
+                )
+                r.raise_for_status()
+
+            except Exception as e:
+                log.warning(e)
+                return None
+
+            results = r.json()
+            if results["total"] == 0:
+                return []
+
+            book_id_list = [
+                self.ID_PATTERN.search(item).group("id")
+                for item in results["items"][:10] if self.ID_PATTERN.search(item)
+            ]
+
+            with futures.ThreadPoolExecutor(max_workers=5) as executor:
+
+                fut = [
+                    executor.submit(self._parse_single_book, book_id, generic_cover)
+                    for book_id in book_id_list
+                ]
+                
+                val = [
+                    future.result() 
+                    for future in futures.as_completed(fut) if future.result()
+                ]
+
+        return val
+
+    def _parse_single_book(
+        self, id: str, generic_cover: str = ""
+    ) -> Optional[MetaRecord]:
+        url = f"https://book.douban.com/subject/{id}/"
+
+        try:
+            r = self.session.get(url)
+            r.raise_for_status()
+        except Exception as e:
+            log.warning(e)
+            return None
+
+        match = MetaRecord(
+            id=id,
+            title="",
+            authors=[],
+            url=url,
+            source=MetaSourceInfo(
+                id=self.__id__,
+                description=self.DESCRIPTION,
+                link=self.META_URL,
+            ),
+        )
+
+        html = etree.HTML(r.content.decode("utf8"))
+
+        match.title = html.xpath(self.TITTLE_XPATH)[0].text
+        match.cover = html.xpath(self.COVER_XPATH)[0].attrib["href"] or generic_cover
+        try:
+            rating_num = float(html.xpath(self.RATING_XPATH)[0].text.strip())
+        except Exception:
+            rating_num = 0
+        match.rating = int(-1 * rating_num // 2 * -1) if rating_num else 0
+
+        tag_elements = html.xpath(self.TAGS_XPATH)
+        if len(tag_elements):
+            match.tags = [tag_element.text for tag_element in tag_elements]
+
+        description_element = html.xpath(self.DESCRIPTION_XPATH)
+        if len(description_element):
+            match.description = html2text(etree.tostring(
+                description_element[-1], encoding="utf8").decode("utf8"))
+
+        info = html.xpath(self.INFO_XPATH)
+
+        for element in info:
+            text = element.text
+            if self.AUTHORS_PATTERN.search(text):
+                next = element.getnext()
+                while next is not None and next.tag != "br":
+                    match.authors.append(next.text)
+                    next = next.getnext()
+            elif self.PUBLISHER_PATTERN.search(text):
+                match.publisher = element.tail.strip()
+            elif self.SUBTITLE_PATTERN.search(text):
+                match.title = f'{match.title}:' + element.tail.strip()
+            elif self.PUBLISHED_DATE_PATTERN.search(text):
+                match.publishedDate = self._clean_date(element.tail.strip())
+            elif self.SUBTITLE_PATTERN.search(text):
+                match.series = element.getnext().text
+            elif i_type := self.IDENTIFIERS_PATTERN.search(text):
+                match.identifiers[i_type.group()] = element.tail.strip()
+
+        return match
+
+    
+    def _clean_date(self, date: str) -> str:
+        """
+        Clean up the date string to be in the format YYYY-MM-DD
+
+        Examples of possible patterns:
+            '2014-7-16', '1988年4月', '1995-04', '2021-8', '2020-12-1', '1996年',
+            '1972', '2004/11/01', '1959年3月北京第1版第1印'
+        """
+        year = date[:4]
+        moon = "01"
+        day = "01"
+
+        if len(date) > 5:
+            digit = []
+            ls = []
+            for i in range(5, len(date)):
+                if date[i].isdigit():
+                    digit.append(date[i])
+                elif digit:
+                    ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
+                    digit = []
+            if digit:
+                ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
+
+            moon = ls[0]
+            if len(ls)>1:
+                day = ls[1]     
+
+        return f"{year}-{moon}-{day}"

cps/metadata_provider/google.py

@@ -22,9 +22,12 @@ from urllib.parse import quote
 
 import requests
 
+from cps import logger
 from cps.isoLanguages import get_lang3, get_language_name
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
 
 class Google(Metadata):
     __name__ = "Google"
@@ -45,7 +48,12 @@ class Google(Metadata):
             if title_tokens:
                 tokens = [quote(t.encode("utf-8")) for t in title_tokens]
                 query = "+".join(tokens)
-            results = requests.get(Google.SEARCH_URL + query)
+            try:
+                results = requests.get(Google.SEARCH_URL + query)
+                results.raise_for_status()
+            except Exception as e:
+                log.warning(e)
+                return None
             for result in results.json().get("items", []):
                 val.append(
                     self._parse_search_result(

cps/metadata_provider/lubimyczytac.py

@@ -27,9 +27,12 @@ from html2text import HTML2Text
 from lxml.html import HtmlElement, fromstring, tostring
 from markdown2 import Markdown
 
+from cps import logger
 from cps.isoLanguages import get_language_name
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
 SYMBOLS_TO_TRANSLATE = (
     "öÖüÜóÓőŐúÚéÉáÁűŰíÍąĄćĆęĘłŁńŃóÓśŚźŹżŻ",
     "oOuUoOoOuUeEaAuUiIaAcCeElLnNoOsSzZzZ",
@@ -112,20 +115,23 @@ class LubimyCzytac(Metadata):
         self, query: str, generic_cover: str = "", locale: str = "en"
     ) -> Optional[List[MetaRecord]]:
         if self.active:
-            result = requests.get(self._prepare_query(title=query))
-            if result.text:
-                root = fromstring(result.text)
-                lc_parser = LubimyCzytacParser(root=root, metadata=self)
-                matches = lc_parser.parse_search_results()
-                if matches:
-                    with ThreadPool(processes=10) as pool:
-                        final_matches = pool.starmap(
-                            lc_parser.parse_single_book,
-                            [(match, generic_cover, locale) for match in matches],
-                        )
-                    return final_matches
-                return matches
-            return []
+            try:
+                result = requests.get(self._prepare_query(title=query))
+                result.raise_for_status()
+            except Exception as e:
+                log.warning(e)
+                return None
+            root = fromstring(result.text)
+            lc_parser = LubimyCzytacParser(root=root, metadata=self)
+            matches = lc_parser.parse_search_results()
+            if matches:
+                with ThreadPool(processes=10) as pool:
+                    final_matches = pool.starmap(
+                        lc_parser.parse_single_book,
+                        [(match, generic_cover, locale) for match in matches],
+                    )
+                return final_matches
+            return matches
 
     def _prepare_query(self, title: str) -> str:
         query = ""
@@ -202,7 +208,12 @@ class LubimyCzytacParser:
     def parse_single_book(
         self, match: MetaRecord, generic_cover: str, locale: str
     ) -> MetaRecord:
-        response = requests.get(match.url)
+        try:
+            response = requests.get(match.url)
+            response.raise_for_status()
+        except Exception as e:
+            log.warning(e)
+            return None
         self.root = fromstring(response.text)
         match.cover = self._parse_cover(generic_cover=generic_cover)
         match.description = self._parse_description()

cps/metadata_provider/scholar.py

@@ -28,8 +28,12 @@ try:
 except FakeUserAgentError:
     raise ImportError("No module named 'scholarly'")
 
+from cps import logger
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
+
 class scholar(Metadata):
     __name__ = "Google Scholar"
     __id__ = "googlescholar"
@@ -44,7 +48,11 @@ class scholar(Metadata):
             if title_tokens:
                 tokens = [quote(t.encode("utf-8")) for t in title_tokens]
                 query = " ".join(tokens)
-            scholar_gen = itertools.islice(scholarly.search_pubs(query), 10)
+            try:
+                scholar_gen = itertools.islice(scholarly.search_pubs(query), 10)
+            except Exception as e:
+                log.warning(e)
+                return None
             for result in scholar_gen:
                 match = self._parse_search_result(
                     result=result, generic_cover="", locale=locale

cps/search_metadata.py

@@ -57,7 +57,7 @@ for f in modules:
         try:
             importlib.import_module("cps.metadata_provider." + a)
             new_list.append(a)
-        except ImportError as e:
+        except (ImportError, IndentationError, SyntaxError) as e:
             log.error("Import error for metadata source: {} - {}".format(a, e))
             pass
 
@@ -138,6 +138,6 @@ def metadata_search():
                 if active.get(c.__id__, True)
             }
             for future in concurrent.futures.as_completed(meta):
-                data.extend([asdict(x) for x in future.result()])
+                data.extend([asdict(x) for x in future.result() if x])
     # log.info({'Time elapsed {}'.format(current_milli_time()-start)})
     return Response(json.dumps(data), mimetype="application/json")

cps/static/js/edit_books.js

@@ -33,7 +33,7 @@ $(".datepicker").datepicker({
     if (results) {
         pubDate = new Date(results[1], parseInt(results[2], 10) - 1, results[3]) || new Date(this.value);
         $(this).next('input')
-            .val(pubDate.toLocaleDateString(language))
+            .val(pubDate.toLocaleDateString(language.replaceAll("_","-")))
             .removeClass("hidden");
     }
 }).trigger("change");

cps/static/js/get_meta.js

@@ -92,14 +92,19 @@ $(function () {
                 data: {"query": keyword},
                 dataType: "json",
                 success: function success(data) {
-                    $("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
-                    data.forEach(function(book) {
-                        var $book = $(templates.bookResult(book));
-                        $book.find("img").on("click", function () {
-                            populateForm(book);
+                    if (data.length) {
+                        $("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
+                        data.forEach(function(book) {
+                            var $book = $(templates.bookResult(book));
+                            $book.find("img").on("click", function () {
+                                populateForm(book);
+                            });
+                            $("#book-list").append($book);
                         });
-                        $("#book-list").append($book);
-                    });
+                    }
+                    else {
+                        $("#meta-info").html("<p class=\"text-danger\">" + msg.no_result + "!</p>" + $("#meta-info")[0].innerHTML)
+                    }
                 },
                 error: function error() {
                     $("#meta-info").html("<p class=\"text-danger\">" + msg.search_error + "!</p>" + $("#meta-info")[0].innerHTML);

cps/templates/languages.html

@@ -1,35 +0,0 @@
-{% extends "layout.html" %}
-{% block body %}
-  <h1>{{title}}</h1>
-  <div class="filterheader hidden-xs">
-    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-    <div id="asc" data-order="{{ order }}" data-id="{{ data }}" class="btn btn-primary {% if order == 1 %} active{% endif%}"><span class="glyphicon glyphicon-sort-by-alphabet"></span></div>
-    <div id="desc" data-id="{{ data }}" class="btn btn-primary{% if order == 0 %} active{% endif%}"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></div>
-    {% if charlist|length %}
-    <div id="all" class="active btn btn-primary {% if charlist|length > 9 %}hidden-sm{% endif %}">{{_('All')}}</div>
-    {% endif %}
-    <div class="btn-group character {% if charlist|length > 9 %}hidden-sm{% endif %}" role="group">
-      {% for char in charlist%}
-      <div class="btn btn-primary char">{{char}}</div>
-      {% endfor %}
-    </div>
-  </div>
-  <div class="container">
-      <div div id="list" class="col-xs-12 col-sm-6">
-  {% for lang in languages %}
-      {% if loop.index0 == (loop.length/2)|int and loop.length > 20 %}
-      </div>
-      <div id="second" class="col-xs-12 col-sm-6">
-    {% endif %}
-    <div class="row" data-id="{{lang[0].name}}">
-        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{lang[1]}}</span></div>
-        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{{url_for('web.books_list', book_id=lang[0].lang_code, data=data, sort_param='stored')}}">{{lang[0].name}}</a></div>
-    </div>
-  {% endfor %}
-      </div>
-  </div>
-{% endblock %}
-{% block js %}
-<script src="{{ url_for('static', filename='js/filter_list.js') }}"></script>
-{% endblock %}
-

cps/templates/list.html

@@ -14,7 +14,7 @@
       {% endif %}
       <div class="btn-group character {% if charlist|length > 9 %}hidden-sm{% endif %}" role="group">
         {% for char in charlist%}
-        <div class="btn btn-primary char">{{char.char}}</div>
+        <div class="btn btn-primary char">{{char[0]}}</div>
         {% endfor %}
       </div>
 
@@ -29,8 +29,8 @@
         </div>
         <div id="second" class="col-xs-12 col-sm-6">
       {% endif %}
-      <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry.name %}{{entry.name}}{% else %}{{entry[0].name}}{% endif %}{% endif %}">
-        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
+      <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry[0].format %}{{entry[0].format}}{% else %}{% if entry[0].rating %}{{entry[0].rating}}{% else %}{{entry[0].name}}{% endif %}{% endif %}{% endif %}">
+        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry[1]}}</span></div>
         <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort_param='stored', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort_param='stored', book_id=entry[0].id )}}{% endif %}">
           {% if entry.name %}
           <div class="rating">

cps/uploader.py

@@ -107,52 +107,10 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension):
         series="",
         series_id="",
         languages="",
-        publisher="")
-
-
-def parse_xmp(pdf_file):
-    """
-    Parse XMP Metadata and prepare for BookMeta object
-    """
-    try:
-        xmp_info = pdf_file.getXmpMetadata()
-    except Exception as ex:
-        log.debug('Can not read XMP metadata {}'.format(ex))
-        return None
-
-    if xmp_info:
-        try:
-            xmp_author = xmp_info.dc_creator # list
-        except AttributeError:
-            xmp_author = ['']
-
-        if xmp_info.dc_title:
-            xmp_title = xmp_info.dc_title['x-default']
-        else:
-            xmp_title = ''
-
-        if xmp_info.dc_description:
-            xmp_description = xmp_info.dc_description['x-default']
-        else:
-            xmp_description = ''
-
-        languages = []
-        try:
-            for i in xmp_info.dc_language:
-                #calibre-web currently only takes one language.
-                languages.append(isoLanguages.get_lang3(i))
-        except AttributeError:
-            languages.append('')
-
-        xmp_tags = ', '.join(xmp_info.dc_subject)
-        xmp_publisher = ', '.join(xmp_info.dc_publisher)
-
-        return {'author': xmp_author,
-                    'title': xmp_title,
-                    'subject': xmp_description,
-                    'tags': xmp_tags, 'languages': languages,
-                    'publisher': xmp_publisher
-                    }
+        publisher="",
+        pubdate="",
+        identifiers=[]
+        )
 
 
 def parse_xmp(pdf_file):
@@ -251,7 +209,9 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
         series="",
         series_id="",
         languages=','.join(languages),
-        publisher=publisher)
+        publisher=publisher,
+        pubdate="",
+        identifiers=[])
 
 
 def pdf_preview(tmp_file_path, tmp_dir):

cps/web.py

@@ -307,10 +307,20 @@ def get_matching_tags():
     return json_dumps
 
 
-def generate_char_list(data_colum, db_link):
-    return (calibre_db.session.query(func.upper(func.substr(data_colum, 1, 1)).label('char'))
+def generate_char_list(entries): # data_colum, db_link):
+    char_list = list()
+    for entry in entries:
+        upper_char = entry[0].name[0].upper()
+        if upper_char not in char_list:
+            char_list.append(upper_char)
+    return char_list
+
+
+def query_char_list(data_colum, db_link):
+    results = (calibre_db.session.query(func.upper(func.substr(data_colum, 1, 1)).label('char'))
             .join(db_link).join(db.Books).filter(calibre_db.common_filters())
             .group_by(func.upper(func.substr(data_colum, 1, 1))).all())
+    return results
 
 
 def get_sort_function(sort_param, data):
@@ -526,50 +536,92 @@ def render_author_books(page, author_id, order):
 
 
 def render_publisher_books(page, book_id, order):
-    publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
-    if publisher:
+    if book_id == '-1':
         entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
-                                                                db.Books.publishers.any(db.Publishers.id == book_id),
+                                                                db.Publishers.name == None,
                                                                 [db.Series.name, order[0][0], db.Books.series_index],
                                                                 True, config.config_read_column,
+                                                                db.books_publishers_link,
+                                                                db.Books.id == db.books_publishers_link.c.book,
+                                                                db.Publishers,
+                                                                db.books_series_link,
+                                                                db.Books.id == db.books_series_link.c.book,
+                                                                db.Series)
+        publisher = _("None")
+    else:
+        publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
+        if publisher:
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                    db.Books,
+                                                                    db.Books.publishers.any(
+                                                                        db.Publishers.id == book_id),
+                                                                    [db.Series.name, order[0][0],
+                                                                     db.Books.series_index],
+                                                                    True, config.config_read_column,
+                                                                    db.books_series_link,
+                                                                    db.Books.id == db.books_series_link.c.book,
+                                                                    db.Series)
+            publisher = publisher.name
+        else:
+            abort(404)
+
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
+                                 title=_(u"Publisher: %(name)s", name=publisher),
+                                 page="publisher",
+                                 order=order[1])
+
+
+def render_series_books(page, book_id, order):
+    if book_id == '-1':
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Series.name == None,
+                                                                [order[0][0]],
+                                                                True, config.config_read_column,
                                                                 db.books_series_link,
                                                                 db.Books.id == db.books_series_link.c.book,
                                                                 db.Series)
-        return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
-                                     title=_(u"Publisher: %(name)s", name=publisher.name),
-                                     page="publisher",
-                                     order=order[1])
+        series_name = _("None")
     else:
-        abort(404)
-
-
-def render_series_books(page, book_id, order):
-    name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
-    if name:
-        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
-                                                                db.Books,
-                                                                db.Books.series.any(db.Series.id == book_id),
-                                                                [order[0][0]],
-                                                                True, config.config_read_column)
-        return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
-                                     title=_(u"Series: %(serie)s", serie=name.name), page="series", order=order[1])
-    else:
-        abort(404)
+        series_name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
+        if series_name:
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                    db.Books,
+                                                                    db.Books.series.any(db.Series.id == book_id),
+                                                                    [order[0][0]],
+                                                                    True, config.config_read_column)
+            series_name = series_name.name
+        else:
+            abort(404)
+    return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
+                                 title=_(u"Series: %(serie)s", serie=series_name), page="series", order=order[1])
 
 
 def render_ratings_books(page, book_id, order):
-    name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
-    entries, random, pagination = calibre_db.fill_indexpage(page, 0,
-                                                            db.Books,
-                                                            db.Books.ratings.any(db.Ratings.id == book_id),
-                                                            [order[0][0]],
-                                                            True, config.config_read_column)
-    if name and name.rating <= 10:
+    if book_id == '-1':
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Books.ratings == None,
+                                                                [order[0][0]],
+                                                                True, config.config_read_column,
+                                                                db.books_series_link,
+                                                                db.Books.id == db.books_series_link.c.book,
+                                                                db.Series)
+        title = _(u"Rating: None")
+        rating = -1
+    else:
+        name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Books.ratings.any(db.Ratings.id == book_id),
+                                                                [order[0][0]],
+                                                                True, config.config_read_column)
+        title = _(u"Rating: %(rating)s stars", rating=int(name.rating / 2))
+        rating = name.rating
+    if title and rating <= 10:
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
-                                     title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)),
-                                     page="ratings",
-                                     order=order[1])
+                                     title=title, page="ratings", order=order[1])
     else:
         abort(404)
 
@@ -591,33 +643,61 @@ def render_formats_books(page, book_id, order):
 
 
 def render_category_books(page, book_id, order):
-    name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
-    if name:
+    if book_id == '-1':
         entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
-                                                                db.Books.tags.any(db.Tags.id == book_id),
+                                                                db.Tags.name == None,
                                                                 [order[0][0], db.Series.name, db.Books.series_index],
                                                                 True, config.config_read_column,
+                                                                db.books_tags_link,
+                                                                db.Books.id == db.books_tags_link.c.book,
+                                                                db.Tags,
                                                                 db.books_series_link,
                                                                 db.Books.id == db.books_series_link.c.book,
                                                                 db.Series)
-        return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
-                                     title=_(u"Category: %(name)s", name=name.name), page="category", order=order[1])
+        tagsname = _("None")
     else:
-        abort(404)
+        tagsname = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
+        if tagsname:
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                    db.Books,
+                                                                    db.Books.tags.any(db.Tags.id == book_id),
+                                                                    [order[0][0], db.Series.name,
+                                                                     db.Books.series_index],
+                                                                    True, config.config_read_column,
+                                                                    db.books_series_link,
+                                                                    db.Books.id == db.books_series_link.c.book,
+                                                                    db.Series)
+            tagsname = tagsname.name
+        else:
+            abort(404)
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
+                                 title=_(u"Category: %(name)s", name=tagsname), page="category", order=order[1])
 
 
 def render_language_books(page, name, order):
     try:
-        lang_name = isoLanguages.get_language_name(get_locale(), name)
+        if name.lower() != "none":
+            lang_name = isoLanguages.get_language_name(get_locale(), name)
+        else:
+            lang_name = _("None")
     except KeyError:
         abort(404)
-
-    entries, random, pagination = calibre_db.fill_indexpage(page, 0,
-                                                            db.Books,
-                                                            db.Books.languages.any(db.Languages.lang_code == name),
-                                                            [order[0][0]],
-                                                            True, config.config_read_column)
+    if name == "none":
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Languages.lang_code == None,
+                                                                [order[0][0]],
+                                                                True, config.config_read_column,
+                                                                db.books_languages_link,
+                                                                db.Books.id == db.books_languages_link.c.book,
+                                                                db.Languages)
+    else:
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Books.languages.any(db.Languages.lang_code == name),
+                                                                [order[0][0]],
+                                                                True, config.config_read_column)
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
                                  title=_(u"Language: %(name)s", name=lang_name), page="language", order=order[1])
 
@@ -880,7 +960,7 @@ def author_list():
         entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
             .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_authors_link.author')).order_by(order).all()
-        char_list = generate_char_list(db.Authors.sort, db.books_authors_link)
+        char_list = query_char_list(db.Authors.sort, db.books_authors_link)
         # If not creating a copy, readonly databases can not display authornames with "|" in it as changing the name
         # starts a change session
         author_copy = copy.deepcopy(entries)
@@ -926,7 +1006,15 @@ def publisher_list():
         entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
             .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_publishers_link.publisher')).order_by(order).all()
-        char_list = generate_char_list(db.Publishers.name, db.books_publishers_link)
+        no_publisher_count = (calibre_db.session.query(db.Books)
+                           .outerjoin(db.books_publishers_link).outerjoin(db.Publishers)
+                           .filter(db.Publishers.name == None)
+                           .filter(calibre_db.common_filters())
+                           .count())
+        if no_publisher_count:
+            entries.append([db.Category(_("None"), "-1"), no_publisher_count])
+        entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
+        char_list = generate_char_list(entries)
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
                                      title=_(u"Publishers"), page="publisherlist", data="publisher", order=order_no)
     else:
@@ -943,11 +1031,19 @@ def series_list():
         else:
             order = db.Series.sort.asc()
             order_no = 1
-        char_list = generate_char_list(db.Series.sort, db.books_series_link)
+        char_list = query_char_list(db.Series.sort, db.books_series_link)
         if current_user.get_view_property('series', 'series_view') == 'list':
             entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
                 .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(text('books_series_link.series')).order_by(order).all()
+            no_series_count = (calibre_db.session.query(db.Books)
+                            .outerjoin(db.books_series_link).outerjoin(db.Series)
+                            .filter(db.Series.name == None)
+                            .filter(calibre_db.common_filters())
+                            .count())
+            if no_series_count:
+                entries.append([db.Category(_("None"), "-1"), no_series_count])
+            entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
             return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
                                          title=_(u"Series"), page="serieslist", data="series", order=order_no)
         else:
@@ -976,6 +1072,13 @@ def ratings_list():
                                            (db.Ratings.rating / 2).label('name')) \
             .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_ratings_link.rating')).order_by(order).all()
+        no_rating_count = (calibre_db.session.query(db.Books)
+                           .outerjoin(db.books_ratings_link).outerjoin(db.Ratings)
+                           .filter(db.Ratings.rating == None)
+                           .filter(calibre_db.common_filters())
+                           .count())
+        entries.append([db.Category(_("None"), "-1", -1), no_rating_count])
+        entries = sorted(entries, key=lambda x: x[0].rating, reverse=not order_no)
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"Ratings list"), page="ratingslist", data="ratings", order=order_no)
     else:
@@ -997,6 +1100,12 @@ def formats_list():
                                            db.Data.format.label('format')) \
             .join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(db.Data.format).order_by(order).all()
+        no_format_count = (calibre_db.session.query(db.Books).outerjoin(db.Data)
+                           .filter(db.Data.format == None)
+                           .filter(calibre_db.common_filters())
+                           .count())
+        if no_format_count:
+            entries.append([db.Category(_("None"), "-1"), no_format_count])
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"File formats list"), page="formatslist", data="formats", order=order_no)
     else:
@@ -1008,15 +1117,10 @@ def formats_list():
 def language_overview():
     if current_user.check_visibility(constants.SIDEBAR_LANGUAGE) and current_user.filter_language() == u"all":
         order_no = 0 if current_user.get_view_property('language', 'dir') == 'desc' else 1
-        char_list = list()
         languages = calibre_db.speaking_language(reverse_order=not order_no, with_count=True)
-        for lang in languages:
-            upper_lang = lang[0].name[0].upper()
-            if upper_lang not in char_list:
-                char_list.append(upper_lang)
-        return render_title_template('languages.html', languages=languages,
-                                     charlist=char_list, title=_(u"Languages"), page="langlist",
-                                     data="language", order=order_no)
+        char_list = generate_char_list(languages)
+        return render_title_template('list.html', entries=languages, folder='web.books_list', charlist=char_list,
+                                     title=_(u"Languages"), page="langlist", data="language", order=order_no)
     else:
         abort(404)
 
@@ -1034,7 +1138,15 @@ def category_list():
         entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
             .join(db.books_tags_link).join(db.Books).order_by(order).filter(calibre_db.common_filters()) \
             .group_by(text('books_tags_link.tag')).all()
-        char_list = generate_char_list(db.Tags.name, db.books_tags_link)
+        no_tag_count = (calibre_db.session.query(db.Books)
+                         .outerjoin(db.books_tags_link).outerjoin(db.Tags)
+                        .filter(db.Tags.name == None)
+                         .filter(calibre_db.common_filters())
+                         .count())
+        if no_tag_count:
+            entries.append([db.Category(_("None"), "-1"), no_tag_count])
+        entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
+        char_list = generate_char_list(entries)
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
                                      title=_(u"Categories"), page="catlist", data="category", order=order_no)
     else:

requirements.txt

@@ -2,7 +2,7 @@ APScheduler>=3.6.3,<3.10.0
 werkzeug<2.1.0
 Babel>=1.3,<3.0
 Flask-Babel>=0.11.1,<2.1.0
-Flask-Login>=0.3.2,<0.5.1
+Flask-Login>=0.3.2,<0.6.1
 Flask-Principal>=0.3.2,<0.5.1
 backports_abc>=0.4
 Flask>=1.0.2,<2.1.0

setup.cfg

@@ -42,7 +42,7 @@ install_requires =
 	werkzeug<2.1.0
 	Babel>=1.3,<3.0
 	Flask-Babel>=0.11.1,<2.1.0
-	Flask-Login>=0.3.2,<0.5.1
+	Flask-Login>=0.3.2,<0.6.1
 	Flask-Principal>=0.3.2,<0.5.1
 	backports_abc>=0.4
 	Flask>=1.0.2,<2.1.0