From 35d6010298a48bbc92171a3e192c82a7f351d0fc Mon Sep 17 00:00:00 2001 From: Manuel Cortez Date: Sat, 26 Jun 2021 15:16:50 -0500 Subject: [PATCH] Save tweets cache by taking into account persist_size --- src/sessions/base.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/sessions/base.py b/src/sessions/base.py index 816b658b..e4e2999d 100644 --- a/src/sessions/base.py +++ b/src/sessions/base.py @@ -1,9 +1,5 @@ # -*- coding: utf-8 -*- """ A base class to be derived in possible new sessions for TWBlue and services.""" -from __future__ import absolute_import -from __future__ import unicode_literals -from builtins import str -from builtins import object import os import paths import output @@ -13,11 +9,9 @@ import logging import config_utils import sqlitedict import application -import os from . import session_exceptions as Exceptions log = logging.getLogger("sessionmanager.session") - class baseSession(object): """ toDo: Decorators does not seem to be working when using them in an inherited class.""" @@ -74,22 +68,39 @@ class baseSession(object): def authorise(self): pass + def get_sized_buffer(self, buffer, size, reversed=False): + """ Returns a list with the amount of items specified by size.""" + if isinstance(buffer, list) and size != -1 and len(buffer) > size: + log.debug("Requesting {} items from a list of {} items. Reversed mode: {}".format(size, len(buffer), reversed)) + if reversed == False: + return buffer[size:] + else: + return buffer[:size] + else: + return buffer + def save_persistent_data(self): - """ Save the data to a persistant sqlite backed file. .""" + """ Save the data to a persistent sqlite backed file. .""" dbname=os.path.join(paths.config_path(), str(self.session_id), "cache.db") + log.debug("Saving storage information...") # persist_size set to 0 means not saving data actually. if self.settings["general"]["persist_size"] == 0: if os.path.exists(dbname): os.remove(dbname) return - # Let's check if we need to create a new SqliteDict object or we just need to call to commit in self.db. + # Let's check if we need to create a new SqliteDict object (when loading db in memory) or we just need to call to commit in self (if reading from disk).db. + # If we read from disk, we cannot modify the buffer size here as we could damage the app's integrity. + # We will modify buffer's size (managed by persist_size) upon loading the db into memory in app startup. if self.settings["general"]["load_cache_in_memory"]: + log.debug("Opening database to dump memory contents...") db=sqlitedict.SqliteDict(dbname, 'c') for k in self.db.keys(): - db[k] = self.db[k] + db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"]) db.close() + log.debug("Data has been saved in the database.") else: try: + log.debug("Syncing new data to disk...") self.db.commit() except: output.speak(_("An exception occurred while saving the {app} database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the {app} developers.").format(app=application.name),True) @@ -98,6 +109,7 @@ class baseSession(object): def load_persistent_data(self): """Import data from a database file from user config.""" + log.debug("Loading storage data...") dbname=os.path.join(paths.config_path(), str(self.session_id), "cache.db") # If persist_size is set to 0, we should remove the db file as we are no longer going to save anything. if self.settings["general"]["persist_size"] == 0: @@ -107,16 +119,24 @@ class baseSession(object): return # try to load the db file. try: + log.debug("Opening database...") db=sqlitedict.SqliteDict(os.path.join(paths.config_path(), dbname), 'c') # If load_cache_in_memory is set to true, we will load the whole database into memory for faster access. # This is going to be faster when retrieving specific objects, at the cost of more memory. # Setting this to False will read the objects from database as they are needed, which might be slower for bigger datasets. if self.settings["general"]["load_cache_in_memory"]: + log.debug("Loading database contents into memory...") for k in db.keys(): self.db[k] = db[k] db.close() + log.debug("Contents were loaded successfully.") else: + log.debug("Instantiating database from disk.") self.db = db + # We must make sure we won't load more than the amount of buffer specified. + log.debug("Checking if we will load all content...") + for k in self.db.keys(): + self.db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"]) if self.db.get("cursors") == None: cursors = dict(direct_messages=-1) self.db["cursors"] = cursors