Save tweets cache by taking into account persist_size

src/sessions/base.py

@@ -1,9 +1,5 @@
 # -*- coding: utf-8 -*-
 """ A base class to be derived in possible new sessions for TWBlue and services."""
-from __future__ import absolute_import
-from __future__ import unicode_literals
-from builtins import str
-from builtins import object
 import os
 import paths
 import output
@@ -13,11 +9,9 @@ import logging
 import config_utils
 import sqlitedict
 import application
-import os
 from . import session_exceptions as Exceptions
 log = logging.getLogger("sessionmanager.session")
 
-
 class baseSession(object):
     """ toDo: Decorators does not seem to be working when using them in an inherited class."""
 
@@ -74,22 +68,39 @@ class baseSession(object):
     def authorise(self):
         pass
 
+    def get_sized_buffer(self, buffer, size, reversed=False):
+        """ Returns a list with the amount of items specified by size."""
+        if isinstance(buffer, list) and size != -1 and len(buffer) > size:
+            log.debug("Requesting {} items from a list of {} items. Reversed mode: {}".format(size, len(buffer), reversed))
+            if reversed == False:
+                return buffer[size:]
+            else:
+                return buffer[:size]
+        else:
+            return buffer
+
     def save_persistent_data(self):
-        """ Save the data to a persistant sqlite backed file. ."""
+        """ Save the data to a persistent sqlite backed file. ."""
         dbname=os.path.join(paths.config_path(), str(self.session_id), "cache.db")
+        log.debug("Saving storage information...")
         # persist_size set to 0 means not saving data actually.
         if self.settings["general"]["persist_size"] == 0:
             if os.path.exists(dbname):
                 os.remove(dbname)
             return
-        # Let's check if we need to create a new SqliteDict object or we just need to call to commit in self.db.
+        # Let's check if we need to create a new SqliteDict object (when loading db in memory) or we just need to call to commit in self (if reading from disk).db.
+        # If we read from disk, we cannot modify the buffer size here as we could damage the app's integrity.
+        # We will modify buffer's size (managed by persist_size) upon loading the db into memory in app startup.
         if self.settings["general"]["load_cache_in_memory"]:
+            log.debug("Opening database to dump memory contents...")
             db=sqlitedict.SqliteDict(dbname, 'c')
             for k in self.db.keys():
-                db[k] = self.db[k]
+                db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
             db.close()
+            log.debug("Data has been saved in the database.")
         else:
             try:
+                log.debug("Syncing new data to disk...")
                 self.db.commit()
             except:
                 output.speak(_("An exception occurred while saving the {app} database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the {app} developers.").format(app=application.name),True)
@@ -98,6 +109,7 @@ class baseSession(object):
 
     def load_persistent_data(self):
         """Import data from a database file from user config."""
+        log.debug("Loading storage data...")
         dbname=os.path.join(paths.config_path(), str(self.session_id), "cache.db")
         # If persist_size is set to 0, we should remove the db file as we are no longer going to save anything.
         if self.settings["general"]["persist_size"] == 0:
@@ -107,16 +119,24 @@ class baseSession(object):
             return
         # try to load the db file.
         try:
+            log.debug("Opening database...")
             db=sqlitedict.SqliteDict(os.path.join(paths.config_path(), dbname), 'c')
             # If load_cache_in_memory is set to true, we will load the whole database into memory for faster access.
             # This is going to be faster when retrieving specific objects, at the cost of more memory.
             # Setting this to False will read the objects from database as they are needed, which might be slower for bigger datasets.
             if self.settings["general"]["load_cache_in_memory"]:
+                log.debug("Loading database contents into memory...")
                 for k in db.keys():
                     self.db[k] = db[k]
                 db.close()
+                log.debug("Contents were loaded successfully.")
             else:
+                log.debug("Instantiating database from disk.")
                 self.db = db
+                # We must make sure we won't load more than the amount of buffer specified.
+                log.debug("Checking if we will load all content...")
+                for k in self.db.keys():
+                    self.db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
             if self.db.get("cursors") == None:
                 cursors = dict(direct_messages=-1)
                 self.db["cursors"] = cursors