diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..2a4cc32b --- /dev/null +++ b/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = sqlite:///.appdata/database/database2.sqlite + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..5d5b6a0a --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,90 @@ +from logging.config import fileConfig + +from sqlalchemy import create_engine +from sqlalchemy import pool +from sqlalchemy.orm import configure_mappers + +from alembic import context +from alembic.autogenerate import rewriter +from alembic.operations import ops + +from bookworm.database import * + + + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +writer = rewriter.Rewriter() + +@writer.rewrites(ops.MigrationScript) +def add_imports(context, revision, op): + op.imports.add("import bookworm") + return [op] + + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = create_engine(config.get_main_option("sqlalchemy.url")) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + process_revision_directives=writer + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..fbc4b07d --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/28099038d8d6_initial_migration.py b/alembic/versions/28099038d8d6_initial_migration.py new file mode 100644 index 00000000..bfe5e83c --- /dev/null +++ b/alembic/versions/28099038d8d6_initial_migration.py @@ -0,0 +1,146 @@ +"""Initial Migration + +Revision ID: 28099038d8d6 +Revises: +Create Date: 2024-09-23 00:10:33.591108 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import bookworm + +# revision identifiers, used by Alembic. +revision: str = '28099038d8d6' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('book', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=512), nullable=False), + sa.Column('uri', bookworm.database.models.DocumentUriDBType(length=1024), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_book_uri'), 'book', ['uri'], unique=True) + op.create_table('document_position_info', + sa.Column('last_page', sa.Integer(), nullable=True), + sa.Column('last_position', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=512), nullable=False), + sa.Column('uri', bookworm.database.models.DocumentUriDBType(length=1024), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_document_position_info_uri'), 'document_position_info', ['uri'], unique=True) + op.create_table('note_tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=512), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_note_tag_title'), 'note_tag', ['title'], unique=True) + op.create_table('pinned_document', + sa.Column('last_opened_on', sa.DateTime(), nullable=True), + sa.Column('is_pinned', sa.Boolean(), nullable=True), + sa.Column('pinning_order', sa.Integer(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=512), nullable=False), + sa.Column('uri', bookworm.database.models.DocumentUriDBType(length=1024), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_pinned_document_uri'), 'pinned_document', ['uri'], unique=True) + op.create_table('quote_tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=512), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_quote_tag_title'), 'quote_tag', ['title'], unique=True) + op.create_table('recent_document', + sa.Column('last_opened_on', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=512), nullable=False), + sa.Column('uri', bookworm.database.models.DocumentUriDBType(length=1024), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_recent_document_uri'), 'recent_document', ['uri'], unique=True) + op.create_table('bookmark', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=255), nullable=False), + sa.Column('page_number', sa.Integer(), nullable=False), + sa.Column('position', sa.Integer(), nullable=False), + sa.Column('section_title', sa.String(length=1024), nullable=False), + sa.Column('section_identifier', sa.String(length=1024), nullable=False), + sa.Column('date_created', sa.DateTime(), nullable=True), + sa.Column('date_updated', sa.DateTime(), nullable=True), + sa.Column('book_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['book_id'], ['book.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('note', + sa.Column('content', sa.Text(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=255), nullable=False), + sa.Column('page_number', sa.Integer(), nullable=False), + sa.Column('position', sa.Integer(), nullable=False), + sa.Column('section_title', sa.String(length=1024), nullable=False), + sa.Column('section_identifier', sa.String(length=1024), nullable=False), + sa.Column('date_created', sa.DateTime(), nullable=True), + sa.Column('date_updated', sa.DateTime(), nullable=True), + sa.Column('book_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['book_id'], ['book.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('quote', + sa.Column('start_pos', sa.Integer(), nullable=False), + sa.Column('end_pos', sa.Integer(), nullable=False), + sa.Column('content', sa.Text(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=255), nullable=False), + sa.Column('page_number', sa.Integer(), nullable=False), + sa.Column('position', sa.Integer(), nullable=False), + sa.Column('section_title', sa.String(length=1024), nullable=False), + sa.Column('section_identifier', sa.String(length=1024), nullable=False), + sa.Column('date_created', sa.DateTime(), nullable=True), + sa.Column('date_updated', sa.DateTime(), nullable=True), + sa.Column('book_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['book_id'], ['book.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('notes_tags', + sa.Column('note_id', sa.Integer(), nullable=True), + sa.Column('note_tag_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['note_id'], ['note.id'], ), + sa.ForeignKeyConstraint(['note_tag_id'], ['note_tag.id'], ) + ) + op.create_table('quotes_tags', + sa.Column('quote_id', sa.Integer(), nullable=True), + sa.Column('quote_tag_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['quote_id'], ['quote.id'], ), + sa.ForeignKeyConstraint(['quote_tag_id'], ['quote_tag.id'], ) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('quotes_tags') + op.drop_table('notes_tags') + op.drop_table('quote') + op.drop_table('note') + op.drop_table('bookmark') + op.drop_index(op.f('ix_recent_document_uri'), table_name='recent_document') + op.drop_table('recent_document') + op.drop_index(op.f('ix_quote_tag_title'), table_name='quote_tag') + op.drop_table('quote_tag') + op.drop_index(op.f('ix_pinned_document_uri'), table_name='pinned_document') + op.drop_table('pinned_document') + op.drop_index(op.f('ix_note_tag_title'), table_name='note_tag') + op.drop_table('note_tag') + op.drop_index(op.f('ix_document_position_info_uri'), table_name='document_position_info') + op.drop_table('document_position_info') + op.drop_index(op.f('ix_book_uri'), table_name='book') + op.drop_table('book') + # ### end Alembic commands ### diff --git a/alembic/versions/85028f013e6d_zero_migration_revision.py b/alembic/versions/85028f013e6d_zero_migration_revision.py new file mode 100644 index 00000000..be60ebd2 --- /dev/null +++ b/alembic/versions/85028f013e6d_zero_migration_revision.py @@ -0,0 +1,30 @@ +"""Zero Migration revision + +Revision ID: 85028f013e6d +Revises: 28099038d8d6 +Create Date: 2024-09-30 14:48:49.349114 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import bookworm + +# revision identifiers, used by Alembic. +revision: str = '85028f013e6d' +down_revision: Union[str, None] = '28099038d8d6' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/bookworm/annotation/__init__.py b/bookworm/annotation/__init__.py index f03d7524..9c953f59 100644 --- a/bookworm/annotation/__init__.py +++ b/bookworm/annotation/__init__.py @@ -3,6 +3,7 @@ import wx from bookworm import config, speech +from bookworm.database.models import Note, Quote from bookworm.logger import logger from bookworm.resources import sounds from bookworm.service import BookwormService @@ -20,7 +21,6 @@ AnnotationSettingsPanel, AnnotationsMenuIds, ) -from .annotation_models import Note, Quote from .annotator import Bookmarker, NoteTaker, Quoter log = logger.getChild(__name__) diff --git a/bookworm/annotation/annotation_models.py b/bookworm/annotation/annotation_models.py deleted file mode 100644 index 2daaadb2..00000000 --- a/bookworm/annotation/annotation_models.py +++ /dev/null @@ -1,115 +0,0 @@ -# coding: utf-8 - -""" -Database models for Annotations. -""" - -from datetime import datetime - -import sqlalchemy as sa -from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm import deferred, relationship, synonym - -from bookworm.database import GetOrCreateMixin, db - - -class TaggedMixin: - """Provides a generic many-to-many relationship - to a dynamically generated tags table using - the `table-per-related` pattern. - - .. admonition::: the dynamically generated table is shared by this model - class and all it's subclasses. - - Adapted from oy-cms. - Copyright (c) 2018 Musharraf Omer - """ - - @staticmethod - def _prepare_association_table(table_name, remote1, remote2): - return sa.Table( - table_name, - db.Model.metadata, - sa.Column(f"{remote1}_id", sa.Integer, sa.ForeignKey(f"{remote1}.id")), - sa.Column(f"{remote2}_id", sa.Integer, sa.ForeignKey(f"{remote2}.id")), - ) - - @declared_attr - def tags(cls): - if not hasattr(cls, "Tag"): - # Create the Tag model - tag_attrs = { - "id": sa.Column(sa.Integer, primary_key=True), - "title": db.string(512, nullable=False, unique=True, index=True), - "items": relationship( - cls, - secondary=lambda: cls.__tags_association_table__, - backref="related_tags", - ), - } - cls.Tag = type( - f"{cls.__name__}Tag", (GetOrCreateMixin, db.Model), tag_attrs - ) - # The many-to-many association table - cls.__tags_association_table__ = cls._prepare_association_table( - table_name=f"{cls.__tablename__}s_tags", - remote1=cls.__tablename__, - remote2=cls.Tag.__tablename__, - ) - return association_proxy( - "related_tags", - "title", - creator=lambda t: cls.Tag.get_or_create(title=t.lower()), - ) - - -class AnnotationBase(db.Model): - __abstract__ = True - title = db.string(255, nullable=False) - page_number = db.integer(nullable=False) - position = db.integer(nullable=False, default=0) - section_title = db.string(1024, nullable=False) - section_identifier = db.string(1024, nullable=False) - date_created = sa.Column(sa.DateTime, default=datetime.utcnow) - date_updated = sa.Column(sa.DateTime, onupdate=datetime.utcnow) - - @declared_attr - def text_column(cls): - return synonym("title") - - @declared_attr - def book_id(cls): - return sa.Column(sa.Integer, sa.ForeignKey("book.id"), nullable=False) - - @declared_attr - def book(cls): - reverse_name = f"{cls.__name__.lower()}s" - return relationship("Book", foreign_keys=[cls.book_id], backref=reverse_name) - - -class Bookmark(AnnotationBase): - """Represents a user-defined bookmark.""" - - -class TaggedContent(AnnotationBase, TaggedMixin): - __abstract__ = True - - @declared_attr - def content(cls): - return deferred(db.text(nullable=False)) - - @declared_attr - def text_column(cls): - return synonym("content") - - -class Note(TaggedContent): - """Represents user comments (notes).""" - - -class Quote(TaggedContent): - """Represents a highlight (quote) from the book.""" - - start_pos = db.integer(nullable=False) - end_pos = db.integer(nullable=False) diff --git a/bookworm/annotation/annotator.py b/bookworm/annotation/annotator.py index 988625d2..f468a950 100644 --- a/bookworm/annotation/annotator.py +++ b/bookworm/annotation/annotator.py @@ -6,11 +6,9 @@ import sqlalchemy as sa from bookworm import config -from bookworm.database.models import Book +from bookworm.database.models import Book, Bookmark, Note, Quote from bookworm.logger import logger -from .annotation_models import Bookmark, Note, Quote - log = logger.getChild(__name__) # The bakery caches query objects to avoid recompiling them into strings in every call diff --git a/bookworm/bootstrap.py b/bookworm/bootstrap.py index 16f9858c..b8a0465d 100644 --- a/bookworm/bootstrap.py +++ b/bookworm/bootstrap.py @@ -1,5 +1,6 @@ # coding: utf-8 +import logging.config import multiprocessing import os import platform diff --git a/bookworm/database/__init__.py b/bookworm/database/__init__.py index 579e8e89..d6913fb8 100644 --- a/bookworm/database/__init__.py +++ b/bookworm/database/__init__.py @@ -1,30 +1,65 @@ # coding: utf-8 """ -Persistent stoarage using SQLlite3 +Persistent storage using SQLlite3 """ - import os +from pathlib import Path import sqlite3 +import sys -import db_magic as db +from alembic import command +from alembic.config import Config +from alembic.migration import MigrationContext +from sqlalchemy import create_engine, text +from sqlalchemy.orm import scoped_session, sessionmaker +from bookworm import app from bookworm.logger import logger +from bookworm import paths from bookworm.paths import db_path as get_db_path -from .models import ( - Book, - DocumentPositionInfo, - GetOrCreateMixin, - PinnedDocument, - RecentDocument, -) -from .schema import upgrade_database_schema +from .models import * log = logger.getChild(__name__) - -def init_database(): +def get_db_url() -> str: db_path = os.path.join(get_db_path(), "database.sqlite") - db.Model.setup_database(f"sqlite:///{db_path}", create=True) - upgrade_database_schema(db.Model.session) + return f"sqlite:///{db_path}" + +def init_database(engine = None, url: str = None) -> bool: + if not url: + url = get_db_url() + if not engine: + engine = create_engine(get_db_url()) + log.debug(f"Using url {url} ") + with engine.connect() as conn: + context = MigrationContext.configure(conn) + rev = context.get_current_revision() + # let's check for the book table + # Should it be too ambiguous, we'd have to revisit what tables should be checked to determine whether the DB is at the baseline point + cursor = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table';")) + tables = [row[0] for row in cursor.fetchall()] + is_baseline = tables != None and "book" in tables and rev == None + log.info(f"Current revision is {rev}") + log.info("Running database migrations and setup") + cfg_file = "" + script_location = "alembic" + if app.is_frozen: + cfg_file = sys._MEIPASS + script_location = paths.app_path("alembic") + + cfg = Config(Path(cfg_file, "alembic.ini")) + cfg.set_main_option('script_location', str(script_location)) + cfg.set_main_option("sqlalchemy.url", url) + if rev == None: + if is_baseline: + log.info("No revision was found, but the database appears to be at the baseline required to begin tracking.") + log.info("Stamping alembic revision") + command.stamp(cfg, "28099038d8d6") + command.upgrade(cfg, "head") + Base.session = scoped_session( + sessionmaker(engine, autocommit=False, autoflush=False) + ) + return True + diff --git a/bookworm/database/models.py b/bookworm/database/models.py index 3a61721c..e7fb8501 100644 --- a/bookworm/database/models.py +++ b/bookworm/database/models.py @@ -5,11 +5,14 @@ """ from datetime import datetime +import re -import db_magic as db import sqlalchemy as sa from sqlalchemy import types +from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import deferred, relationship, synonym, class_mapper, mapper, Query, scoped_session, declarative_base from bookworm.document.uri import DocumentUri from bookworm.logger import logger @@ -40,9 +43,40 @@ def get_or_create(cls, **kwargs): return cls(**kwargs) -class DocumentBase(db.Model, GetOrCreateMixin): +class _QueryProperty(object): + """Convenience property to query a model.""" + + def __get__(self, obj, type): + try: + mapper = class_mapper(type) + if mapper: + return Query(mapper, session=type.session()) + except UnmappedClassError: + return None + +class Model: + id = sa.Column(sa.Integer, primary_key=True) + query = _QueryProperty() + + @declared_attr + def __tablename__(cls) -> str: + """Taken from db_magic""" + """Convert CamelCase class name to underscores_between_words + table name.""" + name = cls.__name__ + return name[0].lower() + re.sub( + r"([A-Z])", lambda m: "_" + m.group(0).lower(), name[1:] + ) + + +Base = declarative_base(cls=Model) + + +class DocumentBase(Base, GetOrCreateMixin): __abstract__ = True - title = db.string(512, nullable=False) + __tablename__ = "document_base" + id = sa.Column(sa.Integer, primary_key=True) + title = sa.Column(sa.String(512), nullable=False) uri = sa.Column(DocumentUriDBType(1024), nullable=False, unique=True, index=True) @classmethod @@ -55,14 +89,17 @@ def get_or_create(cls, *args, **kwargs): class Book(DocumentBase): + __tablename__ = "book" + @property def identifier(self): return self.uri.to_uri_string() class DocumentPositionInfo(DocumentBase): - last_page = db.integer(default=0) - last_position = db.integer(default=0) + __tablename__ = "document_position_info" + last_page = sa.Column(sa.Integer, default=0) + last_position = sa.Column(sa.Integer, default=0) def get_last_position(self): return (self.last_page, self.last_position) @@ -74,7 +111,8 @@ def save_position(self, page, pos): class RecentDocument(DocumentBase): - last_opened_on = db.date_time(default=datetime.utcnow, onupdate=datetime.utcnow) + __tablename__ = "recent_document" + last_opened_on = sa.Column(sa.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) def record_open(self): self.last_opened_on = datetime.utcnow() @@ -93,9 +131,10 @@ def clear_all(cls): class PinnedDocument(DocumentBase): - last_opened_on = db.date_time(default=datetime.utcnow, onupdate=datetime.utcnow) - is_pinned = db.boolean(default=False) - pinning_order = db.integer(default=0) + __tablename__ = "pinned_document" + last_opened_on = sa.Column(sa.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + is_pinned = sa.Column(sa.Boolean, default=False) + pinning_order = sa.Column(sa.Integer, default=0) @classmethod def get_pinned(cls, limit=50): @@ -125,3 +164,111 @@ def clear_all(cls): for item in cls.query.all(): session.delete(item) session.commit() + +# annotation models + +class TaggedMixin: + """Provides a generic many-to-many relationship + to a dynamically generated tags table using + the `table-per-related` pattern. + + .. admonition::: the dynamically generated table is shared by this model + class and all it's subclasses. + + Adapted from oy-cms. + Copyright (c) 2018 Musharraf Omer + """ + + @staticmethod + def _prepare_association_table(table_name, remote1, remote2): + return sa.Table( + table_name, + Base.metadata, + sa.Column(f"{remote1}_id", sa.Integer, sa.ForeignKey(f"{remote1}.id")), + sa.Column(f"{remote2}_id", sa.Integer, sa.ForeignKey(f"{remote2}.id")), + ) + + @declared_attr + def tags(cls): + if not hasattr(cls, "Tag"): + # Create the Tag model + tag_attrs = { + "id": sa.Column(sa.Integer, primary_key=True), + "title": sa.Column(sa.String(512), nullable=False, unique=True, index=True), + "items": relationship( + cls, + secondary=lambda: cls.__tags_association_table__, + backref="related_tags", + ), + } + cls.Tag = type( + f"{cls.__name__}Tag", (GetOrCreateMixin, Base), tag_attrs + ) + # The many-to-many association table + cls.__tags_association_table__ = cls._prepare_association_table( + table_name=f"{cls.__tablename__}s_tags", + remote1=cls.__tablename__, + remote2=cls.Tag.__tablename__, + ) + return association_proxy( + "related_tags", + "title", + creator=lambda t: cls.Tag.get_or_create(title=t.lower()), + ) + + +class AnnotationBase(Base): + __abstract__ = True + __tablename__ = "annotation_base" + id = sa.Column(sa.Integer, primary_key=True) + title = sa.Column(sa.String(255), nullable=False) + page_number = sa.Column(sa.Integer, nullable=False) + position = sa.Column(sa.Integer, nullable=False, default=0) + section_title = sa.Column(sa.String(1024), nullable=False) + section_identifier = sa.Column(sa.String(1024), nullable=False) + date_created = sa.Column(sa.DateTime, default=datetime.utcnow) + date_updated = sa.Column(sa.DateTime, onupdate=datetime.utcnow) + + @declared_attr + def text_column(cls): + return synonym("title") + + @declared_attr + def book_id(cls): + return sa.Column(sa.Integer, sa.ForeignKey("book.id"), nullable=False) + + @declared_attr + def book(cls): + reverse_name = f"{cls.__name__.lower()}s" + return relationship("Book", foreign_keys=[cls.book_id], backref=reverse_name) + + +class Bookmark(AnnotationBase): + """Represents a user-defined bookmark.""" + __tablename__ = "bookmark" + + +class TaggedContent(AnnotationBase, TaggedMixin): + __abstract__ = True + __tablename__ = "tagged_content" + + @declared_attr + def content(cls): + return deferred(sa.Column(sa.Text, nullable=False)) + + @declared_attr + def text_column(cls): + return synonym("content") + + +class Note(TaggedContent): + """Represents user comments (notes).""" + __tablename__ = "note" + + +class Quote(TaggedContent): + """Represents a highlight (quote) from the book.""" + __tablename__ = "quote" + + start_pos = sa.Column(sa.Integer, nullable=False) + end_pos = sa.Column(sa.Integer, nullable=False) diff --git a/bookworm/database/schema.py b/bookworm/database/schema.py deleted file mode 100644 index 4a8c4007..00000000 --- a/bookworm/database/schema.py +++ /dev/null @@ -1,28 +0,0 @@ -# coding: utf-8 - -""" -Contains database schema upgrade rutines for bookworm -""" - -from __future__ import annotations - -from contextlib import suppress - -import sqlalchemy as sa -from db_magic.schema_upgrades import perform_upgrade - -import bookworm.typehints as t -from bookworm.logger import logger - -log = logger.getChild(__name__) -CURRENT_SCHEMA_VERSION = 0 - - -def get_upgrades() -> t.Dict[int, t.Tuple[t.Callable]]: - return {} - - -def upgrade_database_schema(session): - perform_upgrade( - session, upgrades=get_upgrades(), schema_version=CURRENT_SCHEMA_VERSION - ) diff --git a/bookworm/paths.py b/bookworm/paths.py index 613266dc..01f1484a 100644 --- a/bookworm/paths.py +++ b/bookworm/paths.py @@ -113,3 +113,8 @@ def home_data_path(): @merge_paths def fonts_path(): return app_path("resources", "fonts") + + +@merge_paths +def libs_path() -> str: + return app_path() \ No newline at end of file diff --git a/bookworm/platforms/win32/controls/richedit.py b/bookworm/platforms/win32/controls/richedit.py index f879ebd4..dab2f02d 100644 --- a/bookworm/platforms/win32/controls/richedit.py +++ b/bookworm/platforms/win32/controls/richedit.py @@ -14,7 +14,7 @@ from bookworm import app, config from bookworm.gui.text_ctrl_mixin import ContentViewCtrlMixin, ContentViewCtrlPanel from bookworm.logger import logger -from bookworm.paths import app_path +from bookworm.paths import app_path, libs_path from .wnd_proc_hook import WndProcHookMixin @@ -22,7 +22,7 @@ if app.is_frozen: - BKWRICHEDITOPTS_DLL = app_path("BkwRicheditOpts.dll") + BKWRICHEDITOPTS_DLL = libs_path("BkwRicheditOpts.dll") else: BKWRICHEDITOPTS_DLL = ( Path.cwd() diff --git a/bookworm/platforms/win32/speech_engines/espeak/_espeak.py b/bookworm/platforms/win32/speech_engines/espeak/_espeak.py index 7f621d90..232895c3 100644 --- a/bookworm/platforms/win32/speech_engines/espeak/_espeak.py +++ b/bookworm/platforms/win32/speech_engines/espeak/_espeak.py @@ -19,7 +19,7 @@ from bookworm import app from bookworm.runtime import IS_RUNNING_FROM_SOURCE -from bookworm.paths import app_path +from bookworm.paths import app_path, libs_path from bookworm.logger import logger from bookworm.platforms.win32 import nvwave @@ -470,7 +470,7 @@ def get_espeak_directory(): if IS_RUNNING_FROM_SOURCE: espeak_ng_data_dir = Path.cwd().joinpath("scripts", "dlls", "espeak-ng") else: - espeak_ng_data_dir = app_path() + espeak_ng_data_dir = libs_path() return os.fspath(espeak_ng_data_dir) @@ -482,6 +482,6 @@ def get_espeak_dll_path(): "scripts", "dlls", "espeak-ng", app.arch, "espeak-ng.dll" ) else: - espeak_ng_dll = app_path("espeak-ng.dll") + espeak_ng_dll = libs_path("espeak-ng.dll") return os.fspath(espeak_ng_dll) diff --git a/requirements-app.txt b/requirements-app.txt index 23d0f563..e55d2053 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -1,9 +1,9 @@ # Our in-house packages -bp-db-magic==0.2 # Our fork of the configobj (py3k compatible version). https://github.com/blindpandas/configobj/tarball/v5.1.0.dev0 accessible-output2==0.17 +alembic==1.13.2 apsw==3.46.0.0 attrs==23.2.0 babel==2.15.0 @@ -54,7 +54,7 @@ rapidfuzz==3.9.3 regex==2024.5.15 requests==2.32.3 selectolax==0.3.21 -SQLAlchemy==1.3.24 +SQLAlchemy > 2 trafilatura==1.10.0 tzlocal==4.2 ujson==5.10.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index bc392232..3f8bea44 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,8 +6,8 @@ isort==5.13.2 invoke==2.2.0 jinja2==3.1.4 packaging==24.0 -pyinstaller==6.7.0 -pyinstaller-hooks-contrib==2024.6 +pyinstaller==6.10.0 +pyinstaller-hooks-contrib==2024.8 pytest==8.2.1 pytest-xdist==3.6.1 wheel==0.43.0 diff --git a/scripts/builder/Bookworm.spec b/scripts/builder/Bookworm.spec index 91f70fd3..f62dc7bc 100644 --- a/scripts/builder/Bookworm.spec +++ b/scripts/builder/Bookworm.spec @@ -2,6 +2,8 @@ from pathlib import Path +import site + from PyInstaller.utils.hooks import collect_data_files, collect_submodules @@ -21,7 +23,17 @@ BOOKWORM_RESOURCES = collect_data_files( "*.md", ], ) +# alembic +root = Path("../../") +# pyxpdff_data searches for a file named default.xpdf in the site-packages directory +# We need to also include this as a data file +# TODO: Find a way to move this operation under pyxpdf_data DATA_FILES = [ + (f"{root / 'alembic/env.py'}", 'alembic'), + (f"{root / 'alembic/versions/*'}", 'alembic/versions'), + (f"{root / 'alembic.ini'}", '.'), +] +DATA_FILES += [ ( src, Path(dst).relative_to("bookworm"), @@ -40,6 +52,7 @@ HIDDEN_SUBMODULES = [ "justext", ] HIDDEN_IMPORTS = [ + "alembic", "numpy", "cv2", "pkg_resources.py2_warn", diff --git a/tasks.py b/tasks.py index 67720aa8..597e257c 100644 --- a/tasks.py +++ b/tasks.py @@ -719,6 +719,7 @@ def make_version_info_file(c): @make_env def freeze(c): """Freeze the app using pyinstaller.""" + from bookworm import app print("Freezing the application...")