Make TWBlue to respect persist_size when loading the cache database. Added tests for both reading from disk and loading in memory caches

src/run_tests.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+import unittest
+
+testmodules = ["test.test_cache"]
+
+suite = unittest.TestSuite()
+
+for t in testmodules:
+    try:
+        # If the module defines a suite() function, call it to get the suite.
+        mod = __import__(t, globals(), locals(), ['suite'])
+        suitefn = getattr(mod, 'suite')
+        suite.addTest(suitefn())
+    except (ImportError, AttributeError):
+        # else, just load all the test cases from the module.
+        suite.addTest(unittest.defaultTestLoader.loadTestsFromName(t))
+
+unittest.TextTestRunner(verbosity=2).run(suite)

src/sessions/base.py

@@ -72,10 +72,10 @@ class baseSession(object):
         """ Returns a list with the amount of items specified by size."""
         if isinstance(buffer, list) and size != -1 and len(buffer) > size:
             log.debug("Requesting {} items from a list of {} items. Reversed mode: {}".format(size, len(buffer), reversed))
-            if reversed == False:
+            if reversed == True:
-                return buffer[size:]
-            else:
                 return buffer[:size]
+            else:
+                return buffer[len(buffer)-size:]
         else:
             return buffer
 
@@ -91,16 +91,19 @@ class baseSession(object):
         # Let's check if we need to create a new SqliteDict object (when loading db in memory) or we just need to call to commit in self (if reading from disk).db.
         # If we read from disk, we cannot modify the buffer size here as we could damage the app's integrity.
         # We will modify buffer's size (managed by persist_size) upon loading the db into memory in app startup.
-        if self.settings["general"]["load_cache_in_memory"]:
+        if self.settings["general"]["load_cache_in_memory"] and isinstance(self.db, dict):
             log.debug("Opening database to dump memory contents...")
             db=sqlitedict.SqliteDict(dbname, 'c')
             for k in self.db.keys():
-                db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
+                sized_buff = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
+                db[k] = sized_buff
+            db.commit(blocking=True)
             db.close()
             log.debug("Data has been saved in the database.")
         else:
             try:
                 log.debug("Syncing new data to disk...")
+                if hasattr(self.db, "commit"):
                     self.db.commit()
             except:
                 output.speak(_("An exception occurred while saving the {app} database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the {app} developers.").format(app=application.name),True)
@@ -128,6 +131,7 @@ class baseSession(object):
                 log.debug("Loading database contents into memory...")
                 for k in db.keys():
                     self.db[k] = db[k]
+                db.commit(blocking=True)
                 db.close()
                 log.debug("Contents were loaded successfully.")
             else:
@@ -136,7 +140,8 @@ class baseSession(object):
                 # We must make sure we won't load more than the amount of buffer specified.
                 log.debug("Checking if we will load all content...")
                 for k in self.db.keys():
-                    self.db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
+                    sized_buffer = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
+                    self.db[k] = sized_buffer
             if self.db.get("cursors") == None:
                 cursors = dict(direct_messages=-1)
                 self.db["cursors"] = cursors

src/test/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

src/test/test_cache.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+""" Test case to check some of the scenarios we might face when storing tweets in cache, both loading into memory or rreading from disk. """
+import unittest
+import os
+import paths
+import sqlitedict
+import shutil
+# The base session module requires sound as a dependency, and this needs libVLC to be locatable.
+os.environ['PYTHON_VLC_MODULE_PATH']=os.path.abspath(os.path.join(paths.app_path(), "..", "windows-dependencies", "x86"))
+os.environ['PYTHON_VLC_LIB_PATH']=os.path.abspath(os.path.join(paths.app_path(), "..", "windows-dependencies", "x86", "libvlc.dll"))
+from sessions import base
+
+class cacheTestCase(unittest.TestCase):
+
+    def setUp(self):
+        """ Configures a fake session to check caching objects here. """
+        self.session = base.baseSession("testing")
+        if os.path.exists(os.path.join(paths.config_path(), "testing")) == False:
+            os.mkdir(os.path.join(paths.config_path(), "testing"))
+        self.session.get_configuration()
+
+    def tearDown(self):
+        """ Removes the previously configured session. """
+        session_folder = os.path.join(paths.config_path(), "testing")
+        if os.path.exists(session_folder):
+            shutil.rmtree(session_folder)
+
+    def generate_dataset(self):
+        """ Generates a sample dataset"""
+        dataset = dict(home_timeline=["message" for i in range(10000)], mentions_timeline=["mention" for i in range(20000)])
+        return dataset
+
+    ### Testing database being read from disk.
+
+    def test_cache_in_disk_unlimited_size(self):
+        """ Tests cache database being read from disk, storing the whole datasets. """
+        dataset = self.generate_dataset()
+        self.session.settings["general"]["load_cache_in_memory"] = False
+        self.session.settings["general"]["persist_size"] = -1
+        self.session.load_persistent_data()
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        self.session.save_persistent_data()
+        self.assertIsInstance(self.session.db, sqlitedict.SqliteDict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(len(self.session.db.get("home_timeline")), 10000)
+        self.assertEquals(len(self.session.db.get("mentions_timeline")), 20000)
+        self.session.db.close()
+
+
+    def test_cache_in_disk_limited_dataset(self):
+        """ Tests wether the cache stores only the amount of items we ask it to store. """
+        dataset = self.generate_dataset()
+        self.session.settings["general"]["load_cache_in_memory"] = False
+        self.session.settings["general"]["persist_size"] = 100
+        self.session.load_persistent_data()
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        # We need to save and load the db again because we cannot modify buffers' size while the database is opened.
+        # As TWBlue reads directly from db when reading from disk, an attempt to modify buffers size while Blue is reading the db
+        # Might cause an out of sync error between the GUI lists and the database.
+        # So we perform the changes to buffer size when loading data during app startup if the DB is read from disk.
+        self.session.save_persistent_data()
+        self.session.load_persistent_data()
+        self.assertIsInstance(self.session.db, sqlitedict.SqliteDict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(len(self.session.db.get("home_timeline")), 100)
+        self.assertEquals(len(self.session.db.get("mentions_timeline")), 100)
+        self.session.db.close()
+
+    def test_cache_in_disk_limited_dataset_unreversed(self):
+        """Test if the cache is saved properly in unreversed buffers, when newest items are at the end of the list. """
+        dataset = dict(home_timeline=[i for i in range(20)], mentions_timeline=[i for i in range(20)])
+        self.session.settings["general"]["load_cache_in_memory"] = False
+        self.session.settings["general"]["persist_size"] = 10
+        self.session.load_persistent_data()
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        # We need to save and load the db again because we cannot modify buffers' size while the database is opened.
+        # As TWBlue reads directly from db when reading from disk, an attempt to modify buffers size while Blue is reading the db
+        # Might cause an out of sync error between the GUI lists and the database.
+        # So we perform the changes to buffer size when loading data during app startup if the DB is read from disk.
+        self.session.save_persistent_data()
+        self.session.load_persistent_data()
+        self.assertIsInstance(self.session.db, sqlitedict.SqliteDict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(self.session.db.get("home_timeline")[0], 10)
+        self.assertEquals(self.session.db.get("mentions_timeline")[0], 10)
+        self.assertEquals(self.session.db.get("home_timeline")[-1], 19)
+        self.assertEquals(self.session.db.get("mentions_timeline")[-1], 19)
+        self.session.db.close()
+
+    def test_cache_in_disk_limited_dataset_reversed(self):
+        """Test if the cache is saved properly in reversed buffers, when newest items are at the start of the list. """
+        dataset = dict(home_timeline=[i for i in range(19, -1, -1)], mentions_timeline=[i for i in range(19, -1, -1)])
+        self.session.settings["general"]["load_cache_in_memory"] = False
+        self.session.settings["general"]["persist_size"] = 10
+        self.session.settings["general"]["reverse_timelines"] = True
+        self.session.load_persistent_data()
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        # We need to save and load the db again because we cannot modify buffers' size while the database is opened.
+        # As TWBlue reads directly from db when reading from disk, an attempt to modify buffers size while Blue is reading the db
+        # Might cause an out of sync error between the GUI lists and the database.
+        # So we perform the changes to buffer size when loading data during app startup if the DB is read from disk.
+        self.session.save_persistent_data()
+        self.session.load_persistent_data()
+        self.assertIsInstance(self.session.db, sqlitedict.SqliteDict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(self.session.db.get("home_timeline")[0], 19)
+        self.assertEquals(self.session.db.get("mentions_timeline")[0], 19)
+        self.assertEquals(self.session.db.get("home_timeline")[-1], 10)
+        self.assertEquals(self.session.db.get("mentions_timeline")[-1], 10)
+        self.session.db.close()
+
+    ### Testing database being loaded into memory. Those tests should give the same results than before
+    ### but as we have different code depending whether we load db into memory or read it from disk,
+    ### We need to test this anyways.
+    def test_cache_in_memory_unlimited_size(self):
+        """ Tests cache database being loaded in memory, storing the whole datasets. """
+        dataset = self.generate_dataset()
+        self.session.settings["general"]["load_cache_in_memory"] = True
+        self.session.settings["general"]["persist_size"] = -1
+        self.session.load_persistent_data()
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        self.session.save_persistent_data()
+        self.session.load_persistent_data()
+        self.assertIsInstance(self.session.db, dict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(len(self.session.db.get("home_timeline")), 10000)
+        self.assertEquals(len(self.session.db.get("mentions_timeline")), 20000)
+
+    def test_cache_in_memory_limited_dataset(self):
+        """ Tests wether the cache stores only the amount of items we ask it to store, when loaded in memory. """
+        dataset = self.generate_dataset()
+        self.session.settings["general"]["load_cache_in_memory"] = True
+        self.session.settings["general"]["persist_size"] = 100
+        self.session.load_persistent_data()
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        self.session.save_persistent_data()
+        self.session.load_persistent_data()
+        self.assertIsInstance(self.session.db, dict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(len(self.session.db.get("home_timeline")), 100)
+        self.assertEquals(len(self.session.db.get("mentions_timeline")), 100)
+
+    def test_cache_in_memory_limited_dataset_unreversed(self):
+        """Test if the cache is saved properly when loaded in memory in unreversed buffers, when newest items are at the end of the list. """
+        dataset = dict(home_timeline=[i for i in range(20)], mentions_timeline=[i for i in range(20)])
+        self.session.settings["general"]["load_cache_in_memory"] = True
+        self.session.settings["general"]["persist_size"] = 10
+        self.session.load_persistent_data()
+        self.assertTrue(len(self.session.db)==1)
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        self.session.save_persistent_data()
+        self.session.load_persistent_data()
+        self.assertIsInstance(self.session.db, dict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(self.session.db.get("home_timeline")[0], 10)
+        self.assertEquals(self.session.db.get("mentions_timeline")[0], 10)
+        self.assertEquals(self.session.db.get("home_timeline")[-1], 19)
+        self.assertEquals(self.session.db.get("mentions_timeline")[-1], 19)
+
+    def test_cache_in_memory_limited_dataset_reversed(self):
+        """Test if the cache is saved properly in reversed buffers, when newest items are at the start of the list. This test if for db read into memory. """
+        dataset = dict(home_timeline=[i for i in range(19, -1, -1)], mentions_timeline=[i for i in range(19, -1, -1)])
+        self.session.settings["general"]["load_cache_in_memory"] = True
+        self.session.settings["general"]["persist_size"] = 10
+        self.session.settings["general"]["reverse_timelines"] = True
+        self.session.load_persistent_data()
+        self.session.db["home_timeline"] = dataset["home_timeline"]
+        self.session.db["mentions_timeline"] = dataset["mentions_timeline"]
+        self.session.save_persistent_data()
+        self.session.load_persistent_data()
+        self.assertIsInstance(self.session.db, dict)
+        self.assertTrue(self.session.db.get("home_timeline") != None)
+        self.assertTrue(self.session.db.get("mentions_timeline") != None)
+        self.assertEquals(self.session.db.get("home_timeline")[0], 19)
+        self.assertEquals(self.session.db.get("mentions_timeline")[0], 19)
+        self.assertEquals(self.session.db.get("home_timeline")[-1], 10)
+        self.assertEquals(self.session.db.get("mentions_timeline")[-1], 10)
+
+if __name__ == "__main__":
+    unittest.main()