Performance improvements

libgutenberg/DublinCore.py

@@ -148,6 +148,10 @@ def marc(self):
 RE_MARC_SUBFIELD = re.compile(r"\$[a-z]")
 RE_MARC_SPSEP = re.compile(r"[\n ](,|:)([A-Za-z0-9])")
 
+# Translated format strings
+TRANS_AND_JOINER = _(' and ')
+TRANS_TITLE_BY_AUTHORS = _('{title} by {authors}')
+TRANS_TITLE_BY_AUTHORS_ET_AL = _('{title} by {authors} et al.')
 
 class DublinCore(object):
     """ Hold DublinCore attributes.
@@ -272,7 +276,7 @@ def strunk(list_):
         """
         if len(list_) > 2:
             list_ = (', '.join(list_[:-1]) + ',', list_[-1])
-        return _(' and ').join(list_)
+        return TRANS_AND_JOINER.join(list_)
 
 
     @staticmethod
@@ -327,11 +331,11 @@ def cutoff(title, size):
 
         for tail in (self.strunk(fullnames), self.strunk(surnames)):
             if len(tail) + title_len < size:
-                return _('{title} by {authors}').format(title = title, authors = tail)
+                return TRANS_TITLE_BY_AUTHORS.format(title = title, authors = tail)
 
         for tail in (fullnames[0], surnames[0]):
             if len(tail) + title_len < size:
-                return _('{title} by {authors} et al.').format(title = title, authors = tail)
+                return TRANS_TITLE_BY_AUTHORS_ET_AL.format(title = title, authors = tail)
 
         return cutoff(title, size)
 

libgutenberg/GutenbergDatabaseDublinCore.py

@@ -19,11 +19,13 @@
 import os
 import datetime
 
+import psycopg2
+
 from . import DublinCore
 from . import GutenbergGlobals as gg
 from .GutenbergGlobals import Struct, PG_URL
 from .Logger import debug, info, warning, error
-from .GutenbergDatabase import xl, DatabaseError, IntegrityError
+from .GutenbergDatabase import DatabaseError, IntegrityError
 
 RE_FIRST_AZ = re.compile (r"^[a-z]")
 
@@ -60,8 +62,8 @@ def load_from_database(self, ebook):
         """
 
         conn = self.pool.connect()
-        c  = conn.cursor()
-        c2 = conn.cursor()
+        c = conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
+        c2 = conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
 
         # id, copyright and release date
 
@@ -71,8 +73,7 @@ def load_from_database(self, ebook):
 select copyrighted, release_date, downloads from books where pk = %(ebook)s""",
                    {'ebook': id_})
 
-        for row in c.fetchall():
-            row = xl(c, row)
+        for row in c:
             self.release_date = row.release_date
             self.rights = ('Copyrighted. Read the copyright notice inside this book for details.'
                            if row.copyrighted
@@ -92,9 +93,7 @@ def load_from_database(self, ebook):
 WHERE mn_books_authors.fk_books = %(ebook)s
 ORDER BY heading, role, author""", {'ebook': id_})
 
-        for row in c.fetchall():
-            row = xl(c, row)
-
+        for row in c:
             author = Struct()
             author.id             = row.pk
             author.name           = row.author
@@ -117,17 +116,15 @@ def load_from_database(self, ebook):
 
             c2.execute("SELECT alias, alias_heading from aliases where fk_authors = %d"
                         % row.pk)
-            for row2 in c2.fetchall():
-                row2 = xl(c2, row2)
+            for row2 in c2:
                 alias = Struct()
                 alias.alias = row2.alias
                 alias.heading = row2.alias_heading
                 author.aliases.append(alias)
 
             c2.execute("""
 SELECT description, url from author_urls where fk_authors = %d""" % row.pk)
-            for row2 in c2.fetchall():
-                row2 = xl(c2, row2)
+            for row2 in c2:
                 webpage = Struct()
                 webpage.description = row2.description
                 webpage.url = row2.url
@@ -146,9 +143,7 @@ def load_from_database(self, ebook):
    and attributes.fk_attriblist = attriblist.pk
  order by attriblist.name""", {'ebook': id_})
 
-        for row in c.fetchall():
-            row = xl(c, row)
-
+        for row in c:
             marc = Struct()
             marc.code = row.name.split(' ')[0]
             marc.text = self.strip_marc_subfields(row.text)
@@ -173,10 +168,12 @@ def load_from_database(self, ebook):
         rows = c.fetchall()
 
         if not rows:
-            rows.append(('en', 'English' ) )
+            row = Struct()
+            row.pk = 'en'
+            row.lang = 'English'
+            rows.append(row)
 
         for row in rows:
-            row = xl(c, row)
             language = Struct()
             language.id = row.pk
             language.language = row.lang
@@ -190,8 +187,7 @@ def load_from_database(self, ebook):
   where subjects.pk = mn_books_subjects.fk_subjects
     and mn_books_subjects.fk_books = %(ebook)s""", {'ebook': id_})
 
-        for row in c.fetchall():
-            row = xl(c, row)
+        for row in c:
             subject = Struct()
             subject.id = row.pk
             subject.subject = row.subject
@@ -205,8 +201,7 @@ def load_from_database(self, ebook):
   where bookshelves.pk = mn_books_bookshelves.fk_bookshelves
     and mn_books_bookshelves.fk_books = %(ebook)s""", {'ebook': id_})
 
-        for row in c.fetchall():
-            row = xl(c, row)
+        for row in c:
             bookshelf = Struct()
             bookshelf.id = row.pk
             bookshelf.bookshelf = row.bookshelf
@@ -220,8 +215,7 @@ def load_from_database(self, ebook):
   where loccs.pk = mn_books_loccs.fk_loccs
     and mn_books_loccs.fk_books = %(ebook)s""", {'ebook': id_})
 
-        for row in c.fetchall():
-            row = xl(c, row)
+        for row in c:
             locc = Struct()
             locc.id = row.pk
             locc.locc = row.locc
@@ -237,10 +231,12 @@ def load_from_database(self, ebook):
         rows = c.fetchall()
 
         if not rows:
-            rows.append(('Text', 'Text') )
+            row = Struct()
+            row.dcmitype = 'Text'
+            row.description = 'Text'
+            rows.append(row)
 
         for row in rows:
-            row = xl(c, row)
             self.categories.append(row.dcmitype)
             dcmitype = Struct()
             dcmitype.id = row.dcmitype
@@ -264,7 +260,7 @@ def load_files_from_database(self, id_):
         self.generated_files = []
 
         conn = self.pool.connect()
-        c  = conn.cursor()
+        c = conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor)
 
         # files (not strictly DublinCore but useful)
 
@@ -280,9 +276,7 @@ def load_files_from_database(self, id_):
 order by filetypes.sortorder, encodings.sortorder, fk_filetypes,
          fk_encodings, fk_compressions, filename""",  {'ebook': id_})
 
-        for row in c.fetchall():
-            row = xl(c, row)
-
+        for row in c:
             file_ = Struct()
             fn = row.filename
             file_.archive_path = fn

libgutenberg/GutenbergGlobals.py

@@ -308,9 +308,7 @@ def archive_dir(ebook):
     ebook = str(ebook)
     if len(ebook) == 1:
         return '0/' + ebook
-    a = []
-    for c in ebook:
-        a.append(c)
+    a = [c for c in ebook]
     a[-1] = ebook
     return "/".join(a)