mcvsoftware/twblue
1
0
Fork 0
mirror of https://github.com/MCV-Software/TWBlue.git synced 2024-11-26 12:53:12 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

Save tweets cache by taking into account persist_size

Browse Source
This commit is contained in:
Manuel Cortez 2021-06-26 15:16:50 -05:00
parent 5712dd735b
commit 35d6010298
1 changed files with 29 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/sessions/base.py
View File

@ -1,9 +1,5 @@
# -*- coding: utf-8 -*-
""" A base class to be derived in possible new sessions for TWBlue and services."""
from __future__ import absolute_import
from __future__ import unicode_literals
from builtins import str
from builtins import object
import os
import paths
import output
@ -13,11 +9,9 @@ import logging
import config_utils
import sqlitedict
import application
import os
from . import session_exceptions as Exceptions
log = logging.getLogger("sessionmanager.session")
class baseSession(object):
""" toDo: Decorators does not seem to be working when using them in an inherited class."""
@ -74,22 +68,39 @@ class baseSession(object):
def authorise(self):
pass
def get_sized_buffer(self, buffer, size, reversed=False):
""" Returns a list with the amount of items specified by size."""
if isinstance(buffer, list) and size != -1 and len(buffer) > size:
log.debug("Requesting {} items from a list of {} items. Reversed mode: {}".format(size, len(buffer), reversed))
if reversed == False:
return buffer[size:]
else:
return buffer[:size]
else:
return buffer
def save_persistent_data(self):
""" Save the data to a persistant sqlite backed file. ."""
""" Save the data to a persistent sqlite backed file. ."""
dbname=os.path.join(paths.config_path(), str(self.session_id), "cache.db")
log.debug("Saving storage information...")
# persist_size set to 0 means not saving data actually.
if self.settings["general"]["persist_size"] == 0:
if os.path.exists(dbname):
os.remove(dbname)
return
# Let's check if we need to create a new SqliteDict object or we just need to call to commit in self.db.
# Let's check if we need to create a new SqliteDict object (when loading db in memory) or we just need to call to commit in self (if reading from disk).db.
# If we read from disk, we cannot modify the buffer size here as we could damage the app's integrity.
# We will modify buffer's size (managed by persist_size) upon loading the db into memory in app startup.
if self.settings["general"]["load_cache_in_memory"]:
log.debug("Opening database to dump memory contents...")
db=sqlitedict.SqliteDict(dbname, 'c')
for k in self.db.keys():
db[k] = self.db[k]
db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
db.close()
log.debug("Data has been saved in the database.")
else:
try:
log.debug("Syncing new data to disk...")
self.db.commit()
except:
output.speak(_("An exception occurred while saving the {app} database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the {app} developers.").format(app=application.name),True)
@ -98,6 +109,7 @@ class baseSession(object):
def load_persistent_data(self):
"""Import data from a database file from user config."""
log.debug("Loading storage data...")
dbname=os.path.join(paths.config_path(), str(self.session_id), "cache.db")
# If persist_size is set to 0, we should remove the db file as we are no longer going to save anything.
if self.settings["general"]["persist_size"] == 0:
@ -107,16 +119,24 @@ class baseSession(object):
return
# try to load the db file.
try:
log.debug("Opening database...")
db=sqlitedict.SqliteDict(os.path.join(paths.config_path(), dbname), 'c')
# If load_cache_in_memory is set to true, we will load the whole database into memory for faster access.
# This is going to be faster when retrieving specific objects, at the cost of more memory.
# Setting this to False will read the objects from database as they are needed, which might be slower for bigger datasets.
if self.settings["general"]["load_cache_in_memory"]:
log.debug("Loading database contents into memory...")
for k in db.keys():
self.db[k] = db[k]
db.close()
log.debug("Contents were loaded successfully.")
else:
log.debug("Instantiating database from disk.")
self.db = db
# We must make sure we won't load more than the amount of buffer specified.
log.debug("Checking if we will load all content...")
for k in self.db.keys():
self.db[k] = self.get_sized_buffer(self.db[k], self.settings["general"]["persist_size"], self.settings["general"]["reverse_timelines"])
if self.db.get("cursors") == None:
cursors = dict(direct_messages=-1)
self.db["cursors"] = cursors