diff --git a/src/run_tests.py b/src/run_tests.py new file mode 100644 index 00000000..41842347 --- /dev/null +++ b/src/run_tests.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +import unittest + +testmodules = ["test.test_cache"] + +suite = unittest.TestSuite() + +for t in testmodules: + try: + # If the module defines a suite() function, call it to get the suite. + mod = __import__(t, globals(), locals(), ['suite']) + suitefn = getattr(mod, 'suite') + suite.addTest(suitefn()) + except (ImportError, AttributeError): + # else, just load all the test cases from the module. + suite.addTest(unittest.defaultTestLoader.loadTestsFromName(t)) + +unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file diff --git a/src/sessions/base.py b/src/sessions/base.py index e4e2999d..63ee5142 100644 --- a/src/sessions/base.py +++ b/src/sessions/base.py @@ -72,10 +72,10 @@ class baseSession(object): """ Returns a list with the amount of items specified by size.""" if isinstance(buffer, list) and size != -1 and len(buffer) > size: log.debug("Requesting {} items from a list of {} items. Reversed mode: {}".format(size, len(buffer), reversed)) - if reversed == False: - return buffer[size:] - else: + if reversed == True: return buffer[:size] + else: + return buffer[len(buffer)-size:] else: return buffer @@ -91,17 +91,20 @@ class baseSession(object): # Let's check if we need to create a new SqliteDict object (when loading db in memory) or we just need to call to commit in self (if reading from disk).db. # If we read from disk, we cannot modify the buffer size here as we could damage the app's integrity. # We will modify buffer's size (managed by persist_size) upon loading the db into memory in app startup. - if self.settings["general"]["load_cache_in_memory"]: + if self.settings["general"]["load_cache_in_memory"] and isinstance(self.db, dict): log.debug("Opening database to dump memory contents...") db=sqlitedict.SqliteDict(dbname, 'c') for k in self.db.keys(): - db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"]) + sized_buff = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"]) + db[k] = sized_buff + db.commit(blocking=True) db.close() log.debug("Data has been saved in the database.") else: try: log.debug("Syncing new data to disk...") - self.db.commit() + if hasattr(self.db, "commit"): + self.db.commit() except: output.speak(_("An exception occurred while saving the {app} database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the {app} developers.").format(app=application.name),True) log.exception("Exception while saving {}".format(dbname)) @@ -128,6 +131,7 @@ class baseSession(object): log.debug("Loading database contents into memory...") for k in db.keys(): self.db[k] = db[k] + db.commit(blocking=True) db.close() log.debug("Contents were loaded successfully.") else: @@ -136,7 +140,8 @@ class baseSession(object): # We must make sure we won't load more than the amount of buffer specified. log.debug("Checking if we will load all content...") for k in self.db.keys(): - self.db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"]) + sized_buffer = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"]) + self.db[k] = sized_buffer if self.db.get("cursors") == None: cursors = dict(direct_messages=-1) self.db["cursors"] = cursors diff --git a/src/test/__init__.py b/src/test/__init__.py new file mode 100644 index 00000000..40a96afc --- /dev/null +++ b/src/test/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/test/test_cache.py b/src/test/test_cache.py new file mode 100644 index 00000000..af5e1c4d --- /dev/null +++ b/src/test/test_cache.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +""" Test case to check some of the scenarios we might face when storing tweets in cache, both loading into memory or rreading from disk. """ +import unittest +import os +import paths +import sqlitedict +import shutil +# The base session module requires sound as a dependency, and this needs libVLC to be locatable. +os.environ['PYTHON_VLC_MODULE_PATH']=os.path.abspath(os.path.join(paths.app_path(), "..", "windows-dependencies", "x86")) +os.environ['PYTHON_VLC_LIB_PATH']=os.path.abspath(os.path.join(paths.app_path(), "..", "windows-dependencies", "x86", "libvlc.dll")) +from sessions import base + +class cacheTestCase(unittest.TestCase): + + def setUp(self): + """ Configures a fake session to check caching objects here. """ + self.session = base.baseSession("testing") + if os.path.exists(os.path.join(paths.config_path(), "testing")) == False: + os.mkdir(os.path.join(paths.config_path(), "testing")) + self.session.get_configuration() + + def tearDown(self): + """ Removes the previously configured session. """ + session_folder = os.path.join(paths.config_path(), "testing") + if os.path.exists(session_folder): + shutil.rmtree(session_folder) + + def generate_dataset(self): + """ Generates a sample dataset""" + dataset = dict(home_timeline=["message" for i in range(10000)], mentions_timeline=["mention" for i in range(20000)]) + return dataset + + ### Testing database being read from disk. + + def test_cache_in_disk_unlimited_size(self): + """ Tests cache database being read from disk, storing the whole datasets. """ + dataset = self.generate_dataset() + self.session.settings["general"]["load_cache_in_memory"] = False + self.session.settings["general"]["persist_size"] = -1 + self.session.load_persistent_data() + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + self.session.save_persistent_data() + self.assertIsInstance(self.session.db, sqlitedict.SqliteDict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(len(self.session.db.get("home_timeline")), 10000) + self.assertEquals(len(self.session.db.get("mentions_timeline")), 20000) + self.session.db.close() + + + def test_cache_in_disk_limited_dataset(self): + """ Tests wether the cache stores only the amount of items we ask it to store. """ + dataset = self.generate_dataset() + self.session.settings["general"]["load_cache_in_memory"] = False + self.session.settings["general"]["persist_size"] = 100 + self.session.load_persistent_data() + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + # We need to save and load the db again because we cannot modify buffers' size while the database is opened. + # As TWBlue reads directly from db when reading from disk, an attempt to modify buffers size while Blue is reading the db + # Might cause an out of sync error between the GUI lists and the database. + # So we perform the changes to buffer size when loading data during app startup if the DB is read from disk. + self.session.save_persistent_data() + self.session.load_persistent_data() + self.assertIsInstance(self.session.db, sqlitedict.SqliteDict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(len(self.session.db.get("home_timeline")), 100) + self.assertEquals(len(self.session.db.get("mentions_timeline")), 100) + self.session.db.close() + + def test_cache_in_disk_limited_dataset_unreversed(self): + """Test if the cache is saved properly in unreversed buffers, when newest items are at the end of the list. """ + dataset = dict(home_timeline=[i for i in range(20)], mentions_timeline=[i for i in range(20)]) + self.session.settings["general"]["load_cache_in_memory"] = False + self.session.settings["general"]["persist_size"] = 10 + self.session.load_persistent_data() + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + # We need to save and load the db again because we cannot modify buffers' size while the database is opened. + # As TWBlue reads directly from db when reading from disk, an attempt to modify buffers size while Blue is reading the db + # Might cause an out of sync error between the GUI lists and the database. + # So we perform the changes to buffer size when loading data during app startup if the DB is read from disk. + self.session.save_persistent_data() + self.session.load_persistent_data() + self.assertIsInstance(self.session.db, sqlitedict.SqliteDict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(self.session.db.get("home_timeline")[0], 10) + self.assertEquals(self.session.db.get("mentions_timeline")[0], 10) + self.assertEquals(self.session.db.get("home_timeline")[-1], 19) + self.assertEquals(self.session.db.get("mentions_timeline")[-1], 19) + self.session.db.close() + + def test_cache_in_disk_limited_dataset_reversed(self): + """Test if the cache is saved properly in reversed buffers, when newest items are at the start of the list. """ + dataset = dict(home_timeline=[i for i in range(19, -1, -1)], mentions_timeline=[i for i in range(19, -1, -1)]) + self.session.settings["general"]["load_cache_in_memory"] = False + self.session.settings["general"]["persist_size"] = 10 + self.session.settings["general"]["reverse_timelines"] = True + self.session.load_persistent_data() + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + # We need to save and load the db again because we cannot modify buffers' size while the database is opened. + # As TWBlue reads directly from db when reading from disk, an attempt to modify buffers size while Blue is reading the db + # Might cause an out of sync error between the GUI lists and the database. + # So we perform the changes to buffer size when loading data during app startup if the DB is read from disk. + self.session.save_persistent_data() + self.session.load_persistent_data() + self.assertIsInstance(self.session.db, sqlitedict.SqliteDict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(self.session.db.get("home_timeline")[0], 19) + self.assertEquals(self.session.db.get("mentions_timeline")[0], 19) + self.assertEquals(self.session.db.get("home_timeline")[-1], 10) + self.assertEquals(self.session.db.get("mentions_timeline")[-1], 10) + self.session.db.close() + + ### Testing database being loaded into memory. Those tests should give the same results than before + ### but as we have different code depending whether we load db into memory or read it from disk, + ### We need to test this anyways. + def test_cache_in_memory_unlimited_size(self): + """ Tests cache database being loaded in memory, storing the whole datasets. """ + dataset = self.generate_dataset() + self.session.settings["general"]["load_cache_in_memory"] = True + self.session.settings["general"]["persist_size"] = -1 + self.session.load_persistent_data() + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + self.session.save_persistent_data() + self.session.load_persistent_data() + self.assertIsInstance(self.session.db, dict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(len(self.session.db.get("home_timeline")), 10000) + self.assertEquals(len(self.session.db.get("mentions_timeline")), 20000) + + def test_cache_in_memory_limited_dataset(self): + """ Tests wether the cache stores only the amount of items we ask it to store, when loaded in memory. """ + dataset = self.generate_dataset() + self.session.settings["general"]["load_cache_in_memory"] = True + self.session.settings["general"]["persist_size"] = 100 + self.session.load_persistent_data() + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + self.session.save_persistent_data() + self.session.load_persistent_data() + self.assertIsInstance(self.session.db, dict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(len(self.session.db.get("home_timeline")), 100) + self.assertEquals(len(self.session.db.get("mentions_timeline")), 100) + + def test_cache_in_memory_limited_dataset_unreversed(self): + """Test if the cache is saved properly when loaded in memory in unreversed buffers, when newest items are at the end of the list. """ + dataset = dict(home_timeline=[i for i in range(20)], mentions_timeline=[i for i in range(20)]) + self.session.settings["general"]["load_cache_in_memory"] = True + self.session.settings["general"]["persist_size"] = 10 + self.session.load_persistent_data() + self.assertTrue(len(self.session.db)==1) + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + self.session.save_persistent_data() + self.session.load_persistent_data() + self.assertIsInstance(self.session.db, dict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(self.session.db.get("home_timeline")[0], 10) + self.assertEquals(self.session.db.get("mentions_timeline")[0], 10) + self.assertEquals(self.session.db.get("home_timeline")[-1], 19) + self.assertEquals(self.session.db.get("mentions_timeline")[-1], 19) + + def test_cache_in_memory_limited_dataset_reversed(self): + """Test if the cache is saved properly in reversed buffers, when newest items are at the start of the list. This test if for db read into memory. """ + dataset = dict(home_timeline=[i for i in range(19, -1, -1)], mentions_timeline=[i for i in range(19, -1, -1)]) + self.session.settings["general"]["load_cache_in_memory"] = True + self.session.settings["general"]["persist_size"] = 10 + self.session.settings["general"]["reverse_timelines"] = True + self.session.load_persistent_data() + self.session.db["home_timeline"] = dataset["home_timeline"] + self.session.db["mentions_timeline"] = dataset["mentions_timeline"] + self.session.save_persistent_data() + self.session.load_persistent_data() + self.assertIsInstance(self.session.db, dict) + self.assertTrue(self.session.db.get("home_timeline") != None) + self.assertTrue(self.session.db.get("mentions_timeline") != None) + self.assertEquals(self.session.db.get("home_timeline")[0], 19) + self.assertEquals(self.session.db.get("mentions_timeline")[0], 19) + self.assertEquals(self.session.db.get("home_timeline")[-1], 10) + self.assertEquals(self.session.db.get("mentions_timeline")[-1], 10) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file