Moving cache to SQLAlchemy ORM

acd_cli.py

@@ -1579,7 +1579,7 @@ def main():
 
     if args.func not in nocache_actions:
         try:
-            cache = db.NodeCache(CACHE_PATH, SETTINGS_PATH, args.check)
+            cache = db.NodeCache(CACHE_PATH, SETTINGS_PATH)
         except:
             raise
             sys.exit(INIT_FAILED_RETVAL)

acdcli/cache/db.py

@@ -1,132 +1,59 @@
-import configparser
 import logging
-import os
-import re
-import sqlite3
-from threading import local
+import configparser
+from sqlalchemy import create_engine
+from sqlalchemy.pool import StaticPool
+from sqlalchemy.orm import sessionmaker
 
 from acdcli.utils.conf import get_conf
 
-from .cursors import *
-from .format import FormatterMixin
-from .query import QueryMixin
 from .schema import SchemaMixin
+from .query import QueryMixin
+from .format import FormatterMixin
 from .sync import SyncMixin
+from .templates.nodes import Nodes
+from .keyvaluestorage import KeyValueStorage
 
 logger = logging.getLogger(__name__)
 
-_ROOT_ID_SQL = 'SELECT id FROM nodes WHERE name IS NULL AND type == "folder" ORDER BY created'
-
-
+_DB_DEFAULT = 'nodes.db'
 _SETTINGS_FILENAME = 'cache.ini'
 
-_def_conf = configparser.ConfigParser()
-_def_conf['sqlite'] = dict(filename='nodes.db', busy_timeout=30000, journal_mode='wal')
-_def_conf['blacklist'] = dict(folders= [])
-
-
-
 class IntegrityError(Exception):
     def __init__(self, msg):
         self.msg = msg
 
     def __str__(self):
         return repr(self.msg)
 
-
-def _create_conn(path: str) -> sqlite3.Connection:
-    c = sqlite3.connect(path)
-    c.row_factory = sqlite3.Row # allow dict-like access on rows with col name
-    return c
-
-
-def _regex_match(pattern: str, cell: str) -> bool:
-    if cell is None:
-        return False
-    return re.match(pattern, cell, re.IGNORECASE) is not None
-
-
-class NodeCache(SchemaMixin, QueryMixin, SyncMixin, FormatterMixin):
+class NodeCache(SchemaMixin, QueryMixin, FormatterMixin, SyncMixin):
     IntegrityCheckType = dict(full=0, quick=1, none=2)
-    """types of SQLite integrity checks"""
-
-    def __init__(self, cache_path: str='', settings_path='', check=IntegrityCheckType['full']):
-        self._conf = get_conf(settings_path, _SETTINGS_FILENAME, _def_conf)
-
-        self.db_path = os.path.join(cache_path, self._conf['sqlite']['filename'])
-        self.tl = local()
 
-        self.integrity_check(check)
+    def __init__(self, cache_path: str='', settings_path=''):
+        self.init_config(cache_path, settings_path)
+        if self._conf["database"]["url"].startswith("sqlite"):
+            self._engine = create_engine(self._conf["database"]["url"],
+                                        connect_args={'check_same_thread': False},
+                                        poolclass=StaticPool)
+        else:
+            self._engine = create_engine(self._conf["database"]["url"], poolclass=StaticPool)
         self.init()
 
-        self._conn.create_function('REGEXP', _regex_match.__code__.co_argcount, _regex_match)
-
-        with cursor(self._conn) as c:
-            c.execute(_ROOT_ID_SQL)
-            row = c.fetchone()
-            if not row:
-                self.root_id = ''
-                return
-            first_id = row['id']
-
-            if c.fetchone():
-                raise IntegrityError('Could not uniquely identify root node.')
+        self._DBSession = sessionmaker(bind=self._engine)
+        self._session = self._DBSession()
 
-            self.root_id = first_id
+        self.KeyValueStorage = KeyValueStorage(self._session)
 
-        self._execute_pragma('busy_timeout', self._conf['sqlite']['busy_timeout'])
-        self._execute_pragma('journal_mode', self._conf['sqlite']['journal_mode'])
-
-    @property
-    def _conn(self) -> sqlite3.Connection:
-        if not hasattr(self.tl, '_conn'):
-            self.tl._conn = _create_conn(self.db_path)
-        return self.tl._conn
-
-    def _execute_pragma(self, key, value) -> str:
-        with cursor(self._conn) as c:
-            c.execute('PRAGMA %s=%s;' % (key, value))
-            r = c.fetchone()
-        if r:
-            logger.debug('Set %s to %s. Result: %s.' % (key, value, r[0]))
-            return r[0]
-
-    def remove_db_file(self) -> bool:
-        """Removes database file."""
-        self._conn.close()
-
-        import os
-        import random
-        import string
-        import tempfile
-
-        tmp_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(16))
-        tmp_name = os.path.join(tempfile.gettempdir(), tmp_name)
-
-        try:
-            os.rename(self.db_path, tmp_name)
-        except OSError:
-            logger.critical('Error renaming/removing database file "%s".' % self.db_path)
-            return False
+        rootNodes = self._session.query(Nodes).filter(Nodes.name == None).all()
+        if len(rootNodes) > 1:
+            raise IntegrityError('Could not uniquely identify root node.')
+        elif len(rootNodes) == 0:
+            self.root_id = ''
         else:
-            try:
-                os.remove(tmp_name)
-            except OSError:
-                logger.info('Database file was moved, but not deleted.')
-        return True
+            self.root_id = rootNodes[0].id
 
-    def integrity_check(self, type_: IntegrityCheckType):
-        """Performs a `self-integrity check
-        <https://www.sqlite.org/pragma.html#pragma_integrity_check>`_ on the database."""
+    def init_config(self, cache_path, settings_path):
+        _def_conf = configparser.ConfigParser()
+        _def_conf['database'] = dict(url='sqlite:///' + cache_path + '/' + _DB_DEFAULT)
+        _def_conf['blacklist'] = dict(folders=[])
 
-        with cursor(self._conn) as c:
-            if type_ == NodeCache.IntegrityCheckType['full']:
-                r = c.execute('PRAGMA integrity_check;')
-            elif type_ == NodeCache.IntegrityCheckType['quick']:
-                r = c.execute('PRAGMA quick_check;')
-            else:
-                return
-            r = c.fetchone()
-            if not r or r[0] != 'ok':
-                logger.warn('Sqlite database integrity check failed. '
-                            'You may need to clear the cache if you encounter any errors.')
+        self._conf = get_conf(settings_path, _SETTINGS_FILENAME, _def_conf)

acdcli/cache/format.py

@@ -7,7 +7,7 @@
 import sys
 import datetime
 
-from .cursors import cursor
+from .templates.nodes import Status
 
 try:
     colors = filter(None, os.environ.get('LS_COLORS', '').split(':'))
@@ -36,7 +36,7 @@ def init(color=ColorMode['auto']):
         global get_adfixes, color_path, color_status, seq_tpl, nor_fmt
         get_adfixes = lambda _: ('', '')
         color_path = lambda x: x
-        color_status = lambda x: x[0]
+        color_status = lambda x: x.value[0]
         seq_tpl = '%s'
         nor_fmt = '%s'
 
@@ -64,11 +64,11 @@ def color_path(path: str) -> str:
 
 def color_status(status):
     """Creates a colored one-character status abbreviation."""
-    if status == 'AVAILABLE':
-        return seq_tpl % '32' + status[0] + res  # green
-    elif status == 'TRASH':
-        return seq_tpl % '31' + status[0] + res  # red
-    return status[0]
+    if status == Status.AVAILABLE:
+        return seq_tpl % '32' + status.value[0] + res  # green
+    elif status == Status.TRASH:
+        return seq_tpl % '31' + status.value[0] + res  # red
+    return status.value[0]
 
 
 def date_str(time_: datetime.datetime) -> str:

acdcli/cache/keyvaluestorage.py

@@ -0,0 +1,21 @@
+from .templates.metadata import Metadata
+
+
+class KeyValueStorage(object):
+    def __init__(self, session):
+        self._session = session
+
+    def __getitem__(self, key: str):
+        return self._session.query(Metadata.value).filter(Metadata.key == key).scalar()
+
+    def __setitem__(self, key: str, value: str):
+        self._session.merge(Metadata(key=key, value=value))
+        self._session.commit()
+
+    def get(self, key: str, default: str=None):
+        r = self._session.query(Metadata).filter(Metadata.key == key).first()
+        return r.value if r else default
+
+    def update(self, dict_: dict):
+        for key in dict_.keys():
+            self.__setitem__(key, dict_[key])