From 3e002b47889e01a65ea809e5586bdb2796f632f2 Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Thu, 20 Aug 2026 10:48:52 -0700 Subject: [PATCH 1/4] Make psycopg2 do the object conversion --- libgutenberg/GutenbergDatabaseDublinCore.py | 34 +++++++++------------ 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/libgutenberg/GutenbergDatabaseDublinCore.py b/libgutenberg/GutenbergDatabaseDublinCore.py index 8063cc7..a790cc2 100644 --- a/libgutenberg/GutenbergDatabaseDublinCore.py +++ b/libgutenberg/GutenbergDatabaseDublinCore.py @@ -19,11 +19,13 @@ import os import datetime +import psycopg2 + from . import DublinCore from . import GutenbergGlobals as gg from .GutenbergGlobals import Struct, PG_URL from .Logger import debug, info, warning, error -from .GutenbergDatabase import xl, DatabaseError, IntegrityError +from .GutenbergDatabase import DatabaseError, IntegrityError RE_FIRST_AZ = re.compile (r"^[a-z]") @@ -60,8 +62,8 @@ def load_from_database(self, ebook): """ conn = self.pool.connect() - c = conn.cursor() - c2 = conn.cursor() + c = conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) + c2 = conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) # id, copyright and release date @@ -72,7 +74,6 @@ def load_from_database(self, ebook): {'ebook': id_}) for row in c.fetchall(): - row = xl(c, row) self.release_date = row.release_date self.rights = ('Copyrighted. Read the copyright notice inside this book for details.' if row.copyrighted @@ -93,8 +94,6 @@ def load_from_database(self, ebook): ORDER BY heading, role, author""", {'ebook': id_}) for row in c.fetchall(): - row = xl(c, row) - author = Struct() author.id = row.pk author.name = row.author @@ -118,7 +117,6 @@ def load_from_database(self, ebook): c2.execute("SELECT alias, alias_heading from aliases where fk_authors = %d" % row.pk) for row2 in c2.fetchall(): - row2 = xl(c2, row2) alias = Struct() alias.alias = row2.alias alias.heading = row2.alias_heading @@ -127,7 +125,6 @@ def load_from_database(self, ebook): c2.execute(""" SELECT description, url from author_urls where fk_authors = %d""" % row.pk) for row2 in c2.fetchall(): - row2 = xl(c2, row2) webpage = Struct() webpage.description = row2.description webpage.url = row2.url @@ -147,8 +144,6 @@ def load_from_database(self, ebook): order by attriblist.name""", {'ebook': id_}) for row in c.fetchall(): - row = xl(c, row) - marc = Struct() marc.code = row.name.split(' ')[0] marc.text = self.strip_marc_subfields(row.text) @@ -173,10 +168,12 @@ def load_from_database(self, ebook): rows = c.fetchall() if not rows: - rows.append(('en', 'English' ) ) + row = Struct() + row.pk = 'en' + row.lang = 'English' + rows.append(row) for row in rows: - row = xl(c, row) language = Struct() language.id = row.pk language.language = row.lang @@ -191,7 +188,6 @@ def load_from_database(self, ebook): and mn_books_subjects.fk_books = %(ebook)s""", {'ebook': id_}) for row in c.fetchall(): - row = xl(c, row) subject = Struct() subject.id = row.pk subject.subject = row.subject @@ -206,7 +202,6 @@ def load_from_database(self, ebook): and mn_books_bookshelves.fk_books = %(ebook)s""", {'ebook': id_}) for row in c.fetchall(): - row = xl(c, row) bookshelf = Struct() bookshelf.id = row.pk bookshelf.bookshelf = row.bookshelf @@ -221,7 +216,6 @@ def load_from_database(self, ebook): and mn_books_loccs.fk_books = %(ebook)s""", {'ebook': id_}) for row in c.fetchall(): - row = xl(c, row) locc = Struct() locc.id = row.pk locc.locc = row.locc @@ -237,10 +231,12 @@ def load_from_database(self, ebook): rows = c.fetchall() if not rows: - rows.append(('Text', 'Text') ) + row = Struct() + row.dcmitype = 'Text' + row.description = 'Text' + rows.append(row) for row in rows: - row = xl(c, row) self.categories.append(row.dcmitype) dcmitype = Struct() dcmitype.id = row.dcmitype @@ -264,7 +260,7 @@ def load_files_from_database(self, id_): self.generated_files = [] conn = self.pool.connect() - c = conn.cursor() + c = conn.cursor(cursor_factory=psycopg2.extras.NamedTupleCursor) # files (not strictly DublinCore but useful) @@ -281,8 +277,6 @@ def load_files_from_database(self, id_): fk_encodings, fk_compressions, filename""", {'ebook': id_}) for row in c.fetchall(): - row = xl(c, row) - file_ = Struct() fn = row.filename file_.archive_path = fn From 0b164b454b69823204684a91ff41dfecf5d9490f Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Thu, 20 Aug 2026 17:00:04 -0700 Subject: [PATCH 2/4] Use natural cursor iterator --- libgutenberg/GutenbergDatabaseDublinCore.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libgutenberg/GutenbergDatabaseDublinCore.py b/libgutenberg/GutenbergDatabaseDublinCore.py index a790cc2..c5aadb1 100644 --- a/libgutenberg/GutenbergDatabaseDublinCore.py +++ b/libgutenberg/GutenbergDatabaseDublinCore.py @@ -73,7 +73,7 @@ def load_from_database(self, ebook): select copyrighted, release_date, downloads from books where pk = %(ebook)s""", {'ebook': id_}) - for row in c.fetchall(): + for row in c: self.release_date = row.release_date self.rights = ('Copyrighted. Read the copyright notice inside this book for details.' if row.copyrighted @@ -93,7 +93,7 @@ def load_from_database(self, ebook): WHERE mn_books_authors.fk_books = %(ebook)s ORDER BY heading, role, author""", {'ebook': id_}) - for row in c.fetchall(): + for row in c: author = Struct() author.id = row.pk author.name = row.author @@ -116,7 +116,7 @@ def load_from_database(self, ebook): c2.execute("SELECT alias, alias_heading from aliases where fk_authors = %d" % row.pk) - for row2 in c2.fetchall(): + for row2 in c2: alias = Struct() alias.alias = row2.alias alias.heading = row2.alias_heading @@ -124,7 +124,7 @@ def load_from_database(self, ebook): c2.execute(""" SELECT description, url from author_urls where fk_authors = %d""" % row.pk) - for row2 in c2.fetchall(): + for row2 in c2: webpage = Struct() webpage.description = row2.description webpage.url = row2.url @@ -143,7 +143,7 @@ def load_from_database(self, ebook): and attributes.fk_attriblist = attriblist.pk order by attriblist.name""", {'ebook': id_}) - for row in c.fetchall(): + for row in c: marc = Struct() marc.code = row.name.split(' ')[0] marc.text = self.strip_marc_subfields(row.text) @@ -187,7 +187,7 @@ def load_from_database(self, ebook): where subjects.pk = mn_books_subjects.fk_subjects and mn_books_subjects.fk_books = %(ebook)s""", {'ebook': id_}) - for row in c.fetchall(): + for row in c: subject = Struct() subject.id = row.pk subject.subject = row.subject @@ -201,7 +201,7 @@ def load_from_database(self, ebook): where bookshelves.pk = mn_books_bookshelves.fk_bookshelves and mn_books_bookshelves.fk_books = %(ebook)s""", {'ebook': id_}) - for row in c.fetchall(): + for row in c: bookshelf = Struct() bookshelf.id = row.pk bookshelf.bookshelf = row.bookshelf @@ -215,7 +215,7 @@ def load_from_database(self, ebook): where loccs.pk = mn_books_loccs.fk_loccs and mn_books_loccs.fk_books = %(ebook)s""", {'ebook': id_}) - for row in c.fetchall(): + for row in c: locc = Struct() locc.id = row.pk locc.locc = row.locc @@ -276,7 +276,7 @@ def load_files_from_database(self, id_): order by filetypes.sortorder, encodings.sortorder, fk_filetypes, fk_encodings, fk_compressions, filename""", {'ebook': id_}) - for row in c.fetchall(): + for row in c: file_ = Struct() fn = row.filename file_.archive_path = fn From 246b7a7f98f25daa1fe895b48684aff857041360 Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Thu, 20 Aug 2026 17:48:24 -0700 Subject: [PATCH 3/4] Build initial archive_dir list by comprehension --- libgutenberg/GutenbergGlobals.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libgutenberg/GutenbergGlobals.py b/libgutenberg/GutenbergGlobals.py index 63b23f9..c0567b8 100644 --- a/libgutenberg/GutenbergGlobals.py +++ b/libgutenberg/GutenbergGlobals.py @@ -308,9 +308,7 @@ def archive_dir(ebook): ebook = str(ebook) if len(ebook) == 1: return '0/' + ebook - a = [] - for c in ebook: - a.append(c) + a = [c for c in ebook] a[-1] = ebook return "/".join(a) From ad285fbddedccac11f09fe7c71e8f9d1c85d4ebe Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Thu, 20 Aug 2026 17:48:54 -0700 Subject: [PATCH 4/4] Only translate static format strings once --- libgutenberg/DublinCore.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libgutenberg/DublinCore.py b/libgutenberg/DublinCore.py index cb2df24..22694c8 100644 --- a/libgutenberg/DublinCore.py +++ b/libgutenberg/DublinCore.py @@ -148,6 +148,10 @@ def marc(self): RE_MARC_SUBFIELD = re.compile(r"\$[a-z]") RE_MARC_SPSEP = re.compile(r"[\n ](,|:)([A-Za-z0-9])") +# Translated format strings +TRANS_AND_JOINER = _(' and ') +TRANS_TITLE_BY_AUTHORS = _('{title} by {authors}') +TRANS_TITLE_BY_AUTHORS_ET_AL = _('{title} by {authors} et al.') class DublinCore(object): """ Hold DublinCore attributes. @@ -272,7 +276,7 @@ def strunk(list_): """ if len(list_) > 2: list_ = (', '.join(list_[:-1]) + ',', list_[-1]) - return _(' and ').join(list_) + return TRANS_AND_JOINER.join(list_) @staticmethod @@ -327,11 +331,11 @@ def cutoff(title, size): for tail in (self.strunk(fullnames), self.strunk(surnames)): if len(tail) + title_len < size: - return _('{title} by {authors}').format(title = title, authors = tail) + return TRANS_TITLE_BY_AUTHORS.format(title = title, authors = tail) for tail in (fullnames[0], surnames[0]): if len(tail) + title_len < size: - return _('{title} by {authors} et al.').format(title = title, authors = tail) + return TRANS_TITLE_BY_AUTHORS_ET_AL.format(title = title, authors = tail) return cutoff(title, size)