499a66dfb0
Fix duplicate user and email (now case insensitive) Output of calibre on stderr is now logged (full traceback in debug-log, otherwise, only errormessage) Natural sorting for comic reader Fix for long running tasks
512 lines
22 KiB
Python
512 lines
22 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
|
|
# Copyright (C) 2018-2019 OzzieIsaacs, bodybybuddha, janeczku
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import division, print_function, unicode_literals
|
|
import sys
|
|
import os
|
|
import re
|
|
import smtplib
|
|
import socket
|
|
import time
|
|
import threading
|
|
from datetime import datetime, timedelta
|
|
|
|
try:
|
|
from StringIO import StringIO
|
|
from email.MIMEBase import MIMEBase
|
|
from email.MIMEMultipart import MIMEMultipart
|
|
from email.MIMEText import MIMEText
|
|
except ImportError:
|
|
from io import StringIO
|
|
from email.mime.base import MIMEBase
|
|
from email.mime.multipart import MIMEMultipart
|
|
from email.mime.text import MIMEText
|
|
|
|
from email import encoders
|
|
from email.utils import formatdate
|
|
from email.utils import make_msgid
|
|
from email.generator import Generator
|
|
from flask_babel import gettext as _
|
|
|
|
from . import logger, config, db, gdriveutils
|
|
from .subproc_wrapper import process_open
|
|
|
|
|
|
log = logger.create()
|
|
|
|
chunksize = 8192
|
|
# task 'status' consts
|
|
STAT_WAITING = 0
|
|
STAT_FAIL = 1
|
|
STAT_STARTED = 2
|
|
STAT_FINISH_SUCCESS = 3
|
|
#taskType consts
|
|
TASK_EMAIL = 1
|
|
TASK_CONVERT = 2
|
|
TASK_UPLOAD = 3
|
|
TASK_CONVERT_ANY = 4
|
|
|
|
RET_FAIL = 0
|
|
RET_SUCCESS = 1
|
|
|
|
|
|
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
|
|
# it in MIME Base64 encoded to
|
|
def get_attachment(bookpath, filename):
|
|
"""Get file as MIMEBase message"""
|
|
calibrepath = config.config_calibre_dir
|
|
if config.config_use_google_drive:
|
|
df = gdriveutils.getFileFromEbooksFolder(bookpath, filename)
|
|
if df:
|
|
datafile = os.path.join(calibrepath, bookpath, filename)
|
|
if not os.path.exists(os.path.join(calibrepath, bookpath)):
|
|
os.makedirs(os.path.join(calibrepath, bookpath))
|
|
df.GetContentFile(datafile)
|
|
else:
|
|
return None
|
|
file_ = open(datafile, 'rb')
|
|
data = file_.read()
|
|
file_.close()
|
|
os.remove(datafile)
|
|
else:
|
|
try:
|
|
file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
|
|
data = file_.read()
|
|
file_.close()
|
|
except IOError as e:
|
|
log.exception(e) # traceback.print_exc()
|
|
log.error(u'The requested file could not be read. Maybe wrong permissions?')
|
|
return None
|
|
|
|
attachment = MIMEBase('application', 'octet-stream')
|
|
attachment.set_payload(data)
|
|
encoders.encode_base64(attachment)
|
|
attachment.add_header('Content-Disposition', 'attachment',
|
|
filename=filename)
|
|
return attachment
|
|
|
|
|
|
# Class for sending email with ability to get current progress
|
|
class emailbase():
|
|
|
|
transferSize = 0
|
|
progress = 0
|
|
|
|
def data(self, msg):
|
|
self.transferSize = len(msg)
|
|
(code, resp) = smtplib.SMTP.data(self, msg)
|
|
self.progress = 0
|
|
return (code, resp)
|
|
|
|
def send(self, strg):
|
|
"""Send `strg' to the server."""
|
|
log.debug('send: %r', strg[:300])
|
|
if hasattr(self, 'sock') and self.sock:
|
|
try:
|
|
if self.transferSize:
|
|
lock=threading.Lock()
|
|
lock.acquire()
|
|
self.transferSize = len(strg)
|
|
lock.release()
|
|
for i in range(0, self.transferSize, chunksize):
|
|
if isinstance(strg, bytes):
|
|
self.sock.send((strg[i:i+chunksize]))
|
|
else:
|
|
self.sock.send((strg[i:i + chunksize]).encode('utf-8'))
|
|
lock.acquire()
|
|
self.progress = i
|
|
lock.release()
|
|
else:
|
|
self.sock.sendall(strg.encode('utf-8'))
|
|
except socket.error:
|
|
self.close()
|
|
raise smtplib.SMTPServerDisconnected('Server not connected')
|
|
else:
|
|
raise smtplib.SMTPServerDisconnected('please run connect() first')
|
|
|
|
@classmethod
|
|
def _print_debug(self, *args):
|
|
log.debug(args)
|
|
|
|
def getTransferStatus(self):
|
|
if self.transferSize:
|
|
lock2 = threading.Lock()
|
|
lock2.acquire()
|
|
value = int((float(self.progress) / float(self.transferSize))*100)
|
|
lock2.release()
|
|
return str(value) + ' %'
|
|
else:
|
|
return "100 %"
|
|
|
|
|
|
# Class for sending email with ability to get current progress, derived from emailbase class
|
|
class email(emailbase, smtplib.SMTP):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
smtplib.SMTP.__init__(self, *args, **kwargs)
|
|
|
|
|
|
# Class for sending ssl encrypted email with ability to get current progress, , derived from emailbase class
|
|
class email_SSL(emailbase, smtplib.SMTP_SSL):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
|
|
|
|
|
|
#Class for all worker tasks in the background
|
|
class WorkerThread(threading.Thread):
|
|
|
|
def __init__(self):
|
|
self._stopevent = threading.Event()
|
|
threading.Thread.__init__(self)
|
|
self.status = 0
|
|
self.current = 0
|
|
self.last = 0
|
|
self.queue = list()
|
|
self.UIqueue = list()
|
|
self.asyncSMTP=None
|
|
self.id = 0
|
|
|
|
# Main thread loop starting the different tasks
|
|
def run(self):
|
|
while not self._stopevent.isSet():
|
|
doLock = threading.Lock()
|
|
doLock.acquire()
|
|
if self.current != self.last:
|
|
doLock.release()
|
|
if self.queue[self.current]['taskType'] == TASK_EMAIL:
|
|
self._send_raw_email()
|
|
if self.queue[self.current]['taskType'] == TASK_CONVERT:
|
|
self._convert_any_format()
|
|
if self.queue[self.current]['taskType'] == TASK_CONVERT_ANY:
|
|
self._convert_any_format()
|
|
# TASK_UPLOAD is handled implicitly
|
|
self.current += 1
|
|
else:
|
|
doLock.release()
|
|
time.sleep(1)
|
|
|
|
def stop(self):
|
|
self._stopevent.set()
|
|
|
|
def get_send_status(self):
|
|
if self.asyncSMTP:
|
|
return self.asyncSMTP.getTransferStatus()
|
|
else:
|
|
return "0 %"
|
|
|
|
def _delete_completed_tasks(self):
|
|
for index, task in reversed(list(enumerate(self.UIqueue))):
|
|
if task['progress'] == "100 %":
|
|
# delete tasks
|
|
self.queue.pop(index)
|
|
self.UIqueue.pop(index)
|
|
# if we are deleting entries before the current index, adjust the index
|
|
self.current -= 1
|
|
self.last = len(self.queue)
|
|
|
|
def get_taskstatus(self):
|
|
if self.current < len(self.queue):
|
|
if self.UIqueue[self.current]['stat'] == STAT_STARTED:
|
|
if self.queue[self.current]['taskType'] == TASK_EMAIL:
|
|
self.UIqueue[self.current]['progress'] = self.get_send_status()
|
|
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
|
|
self.UIqueue[self.current]['rt'] = self.UIqueue[self.current]['formRuntime'].days*24*60 \
|
|
+ self.UIqueue[self.current]['formRuntime'].seconds \
|
|
+ self.UIqueue[self.current]['formRuntime'].microseconds
|
|
return self.UIqueue
|
|
|
|
def _convert_any_format(self):
|
|
# convert book, and upload in case of google drive
|
|
self.UIqueue[self.current]['stat'] = STAT_STARTED
|
|
self.queue[self.current]['starttime'] = datetime.now()
|
|
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
|
|
curr_task = self.queue[self.current]['taskType']
|
|
filename = self._convert_ebook_format()
|
|
if filename:
|
|
if config.config_use_google_drive:
|
|
gdriveutils.updateGdriveCalibreFromLocal()
|
|
if curr_task == TASK_CONVERT:
|
|
self.add_email(self.queue[self.current]['settings']['subject'], self.queue[self.current]['path'],
|
|
filename, self.queue[self.current]['settings'], self.queue[self.current]['kindle'],
|
|
self.UIqueue[self.current]['user'], self.queue[self.current]['title'],
|
|
self.queue[self.current]['settings']['body'])
|
|
|
|
def _convert_ebook_format(self):
|
|
error_message = None
|
|
file_path = self.queue[self.current]['file_path']
|
|
bookid = self.queue[self.current]['bookid']
|
|
format_old_ext = u'.' + self.queue[self.current]['settings']['old_book_format'].lower()
|
|
format_new_ext = u'.' + self.queue[self.current]['settings']['new_book_format'].lower()
|
|
|
|
# check to see if destination format already exists -
|
|
# if it does - mark the conversion task as complete and return a success
|
|
# this will allow send to kindle workflow to continue to work
|
|
if os.path.isfile(file_path + format_new_ext):
|
|
log.info("Book id %d already converted to %s", bookid, format_new_ext)
|
|
cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first()
|
|
self.queue[self.current]['path'] = file_path
|
|
self.queue[self.current]['title'] = cur_book.title
|
|
self._handleSuccess()
|
|
return file_path + format_new_ext
|
|
else:
|
|
log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", bookid, format_new_ext)
|
|
|
|
# check if converter-executable is existing
|
|
if not os.path.exists(config.config_converterpath):
|
|
# ToDo Text is not translated
|
|
self._handleError(u"Convertertool %s not found" % config.config_converterpath)
|
|
return
|
|
|
|
try:
|
|
# check which converter to use kindlegen is "1"
|
|
if format_old_ext == '.epub' and format_new_ext == '.mobi':
|
|
if config.config_ebookconverter == 1:
|
|
'''if os.name == 'nt':
|
|
command = config.config_converterpath + u' "' + file_path + u'.epub"'
|
|
if sys.version_info < (3, 0):
|
|
command = command.encode(sys.getfilesystemencoding())
|
|
else:'''
|
|
command = [config.config_converterpath, file_path + u'.epub']
|
|
quotes = [1]
|
|
if config.config_ebookconverter == 2:
|
|
# Linux py2.7 encode as list without quotes no empty element for parameters
|
|
# linux py3.x no encode and as list without quotes no empty element for parameters
|
|
# windows py2.7 encode as string with quotes empty element for parameters is okay
|
|
# windows py 3.x no encode and as string with quotes empty element for parameters is okay
|
|
# separate handling for windows and linux
|
|
quotes = [1,2]
|
|
'''if os.name == 'nt':
|
|
command = config.config_converterpath + u' "' + file_path + format_old_ext + u'" "' + \
|
|
file_path + format_new_ext + u'" ' + config.config_calibre
|
|
if sys.version_info < (3, 0):
|
|
command = command.encode(sys.getfilesystemencoding())
|
|
else:'''
|
|
command = [config.config_converterpath, (file_path + format_old_ext),
|
|
(file_path + format_new_ext)]
|
|
index = 3
|
|
if config.config_calibre:
|
|
parameters = config.config_calibre.split(" ")
|
|
for param in parameters:
|
|
command.append(param)
|
|
quotes.append(index)
|
|
index += 1
|
|
p = process_open(command, quotes)
|
|
# p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
|
|
except OSError as e:
|
|
self._handleError(_(u"Ebook-converter failed: %(error)s", error=e))
|
|
return
|
|
|
|
if config.config_ebookconverter == 1:
|
|
nextline = p.communicate()[0]
|
|
# Format of error message (kindlegen translates its output texts):
|
|
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
|
|
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
|
|
# If error occoures, store error message for logfile
|
|
if conv_error:
|
|
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
|
|
error=conv_error.group(1), message=conv_error.group(2).strip())
|
|
log.debug("convert_kindlegen: %s", nextline)
|
|
else:
|
|
while p.poll() is None:
|
|
nextline = p.stdout.readline()
|
|
if os.name == 'nt' and sys.version_info < (3, 0):
|
|
nextline = nextline.decode('windows-1252')
|
|
elif os.name == 'posix' and sys.version_info < (3, 0):
|
|
nextline = nextline.decode('utf-8')
|
|
log.debug(nextline.strip('\r\n'))
|
|
# parse progress string from calibre-converter
|
|
progress = re.search("(\d+)%\s.*", nextline)
|
|
if progress:
|
|
self.UIqueue[self.current]['progress'] = progress.group(1) + ' %'
|
|
|
|
# process returncode
|
|
check = p.returncode
|
|
calibre_traceback = p.stderr.readlines()
|
|
for ele in calibre_traceback:
|
|
log.debug(ele.strip('\n'))
|
|
if not ele.startswith('Traceback') and not ele.startswith(' File'):
|
|
error_message = "Calibre failed with error: %s" % ele.strip('\n')
|
|
|
|
# kindlegen returncodes
|
|
# 0 = Info(prcgen):I1036: Mobi file built successfully
|
|
# 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
|
|
# 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors!
|
|
if (check < 2 and config.config_ebookconverter == 1) or \
|
|
(check == 0 and config.config_ebookconverter == 2):
|
|
cur_book = db.session.query(db.Books).filter(db.Books.id == bookid).first()
|
|
if os.path.isfile(file_path + format_new_ext):
|
|
new_format = db.Data(name=cur_book.data[0].name,
|
|
book_format=self.queue[self.current]['settings']['new_book_format'].upper(),
|
|
book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext))
|
|
cur_book.data.append(new_format)
|
|
db.session.commit()
|
|
self.queue[self.current]['path'] = cur_book.path
|
|
self.queue[self.current]['title'] = cur_book.title
|
|
if config.config_use_google_drive:
|
|
os.remove(file_path + format_old_ext)
|
|
self._handleSuccess()
|
|
return file_path + format_new_ext
|
|
else:
|
|
error_message = format_new_ext.upper() + ' format not found on disk'
|
|
log.info("ebook converter failed with error while converting book")
|
|
if not error_message:
|
|
error_message = 'Ebook converter failed with unknown error'
|
|
self._handleError(error_message)
|
|
return
|
|
|
|
|
|
def add_convert(self, file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
|
|
addLock = threading.Lock()
|
|
addLock.acquire()
|
|
if self.last >= 20:
|
|
self._delete_completed_tasks()
|
|
# progress, runtime, and status = 0
|
|
self.id += 1
|
|
task = TASK_CONVERT_ANY
|
|
if kindle_mail:
|
|
task = TASK_CONVERT
|
|
self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle': kindle_mail,
|
|
'taskType': task, 'settings':settings})
|
|
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'taskMess': taskMessage,
|
|
'runtime': '0 s', 'stat': STAT_WAITING,'id': self.id, 'taskType': task } )
|
|
|
|
self.last=len(self.queue)
|
|
addLock.release()
|
|
|
|
def add_email(self, subject, filepath, attachment, settings, recipient, user_name, taskMessage,
|
|
text):
|
|
# if more than 20 entries in the list, clean the list
|
|
addLock = threading.Lock()
|
|
addLock.acquire()
|
|
if self.last >= 20:
|
|
self._delete_completed_tasks()
|
|
# progress, runtime, and status = 0
|
|
self.id += 1
|
|
self.queue.append({'subject':subject, 'attachment':attachment, 'filepath':filepath,
|
|
'settings':settings, 'recipent':recipient, 'starttime': 0,
|
|
'taskType': TASK_EMAIL, 'text':text})
|
|
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'taskMess': taskMessage,
|
|
'runtime': '0 s', 'stat': STAT_WAITING,'id': self.id, 'taskType': TASK_EMAIL })
|
|
self.last=len(self.queue)
|
|
addLock.release()
|
|
|
|
def add_upload(self, user_name, taskMessage):
|
|
# if more than 20 entries in the list, clean the list
|
|
addLock = threading.Lock()
|
|
addLock.acquire()
|
|
if self.last >= 20:
|
|
self._delete_completed_tasks()
|
|
# progress=100%, runtime=0, and status finished
|
|
self.id += 1
|
|
self.queue.append({'starttime': datetime.now(), 'taskType': TASK_UPLOAD})
|
|
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': "100 %", 'taskMess': taskMessage,
|
|
'runtime': '0 s', 'stat': STAT_FINISH_SUCCESS,'id': self.id, 'taskType': TASK_UPLOAD})
|
|
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
|
|
self.last=len(self.queue)
|
|
addLock.release()
|
|
|
|
|
|
def _send_raw_email(self):
|
|
self.queue[self.current]['starttime'] = datetime.now()
|
|
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
|
|
self.UIqueue[self.current]['stat'] = STAT_STARTED
|
|
obj=self.queue[self.current]
|
|
# create MIME message
|
|
msg = MIMEMultipart()
|
|
msg['Subject'] = self.queue[self.current]['subject']
|
|
msg['Message-Id'] = make_msgid('calibre-web')
|
|
msg['Date'] = formatdate(localtime=True)
|
|
text = self.queue[self.current]['text']
|
|
msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
|
|
if obj['attachment']:
|
|
result = get_attachment(obj['filepath'], obj['attachment'])
|
|
if result:
|
|
msg.attach(result)
|
|
else:
|
|
self._handleError(u"Attachment not found")
|
|
return
|
|
|
|
msg['From'] = obj['settings']["mail_from"]
|
|
msg['To'] = obj['recipent']
|
|
|
|
use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
|
|
try:
|
|
# convert MIME message to string
|
|
fp = StringIO()
|
|
gen = Generator(fp, mangle_from_=False)
|
|
gen.flatten(msg)
|
|
msg = fp.getvalue()
|
|
|
|
# send email
|
|
timeout = 600 # set timeout to 5mins
|
|
|
|
# redirect output to logfile on python2 pn python3 debugoutput is caught with overwritten
|
|
# _print_debug function
|
|
if sys.version_info < (3, 0):
|
|
org_smtpstderr = smtplib.stderr
|
|
smtplib.stderr = logger.StderrLogger('worker.smtp')
|
|
|
|
if use_ssl == 2:
|
|
self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
|
|
else:
|
|
self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
|
|
|
|
# link to logginglevel
|
|
if logger.is_debug_enabled():
|
|
self.asyncSMTP.set_debuglevel(1)
|
|
if use_ssl == 1:
|
|
self.asyncSMTP.starttls()
|
|
if obj['settings']["mail_password"]:
|
|
self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
|
|
self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], msg)
|
|
self.asyncSMTP.quit()
|
|
self._handleSuccess()
|
|
|
|
if sys.version_info < (3, 0):
|
|
smtplib.stderr = org_smtpstderr
|
|
|
|
except (MemoryError) as e:
|
|
self._handleError(u'Error sending email: ' + e.message)
|
|
return None
|
|
except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
|
|
if hasattr(e, "smtp_error"):
|
|
text = e.smtp_error.decode('utf-8').replace("\n",'. ')
|
|
elif hasattr(e, "message"):
|
|
text = e.message
|
|
else:
|
|
text = ''
|
|
self._handleError(u'Error sending email: ' + text)
|
|
return None
|
|
except (socket.error) as e:
|
|
self._handleError(u'Error sending email: ' + e.strerror)
|
|
return None
|
|
|
|
def _handleError(self, error_message):
|
|
log.error(error_message)
|
|
self.UIqueue[self.current]['stat'] = STAT_FAIL
|
|
self.UIqueue[self.current]['progress'] = "100 %"
|
|
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
|
|
self.UIqueue[self.current]['message'] = error_message
|
|
|
|
def _handleSuccess(self):
|
|
self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS
|
|
self.UIqueue[self.current]['progress'] = "100 %"
|
|
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
|