From 382acf7c8cf9d8aa69ce24d2cf30b4ae1d1c9b26 Mon Sep 17 00:00:00 2001 From: Manuel Cortez Date: Wed, 23 Jun 2021 13:40:21 -0500 Subject: [PATCH] Use slitedict to attempt to reduce memory usage when caching tweets --- requirements.txt | 1 + src/controller/mainController.py | 4 ++-- src/sessions/base.py | 39 ++++++++++++++++++-------------- src/sessions/twitter/session.py | 26 ++++++++++++++------- 4 files changed, 43 insertions(+), 27 deletions(-) diff --git a/requirements.txt b/requirements.txt index ae1296b8..9943eb0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,7 @@ cx_freeze tweepy twitter-text-parser pyenchant +sqlitedict git+https://github.com/accessibleapps/libloader git+https://github.com/accessibleapps/platform_utils git+https://github.com/accessibleapps/accessible_output2 diff --git a/src/controller/mainController.py b/src/controller/mainController.py index ed8bf128..1c11c404 100644 --- a/src/controller/mainController.py +++ b/src/controller/mainController.py @@ -254,8 +254,8 @@ class Controller(object): # Connection checker executed each minute. self.checker_function = RepeatingTimer(60, self.check_connection) # self.checker_function.start() - self.save_db = RepeatingTimer(300, self.save_data_in_db) - self.save_db.start() +# self.save_db = RepeatingTimer(300, self.save_data_in_db) +# self.save_db.start() log.debug("Setting updates to buffers every %d seconds..." % (60*config.app["app-settings"]["update_period"],)) self.update_buffers_function = RepeatingTimer(60*config.app["app-settings"]["update_period"], self.update_buffers) self.update_buffers_function.start() diff --git a/src/sessions/base.py b/src/sessions/base.py index 597cacfc..6f3ecef9 100644 --- a/src/sessions/base.py +++ b/src/sessions/base.py @@ -11,12 +11,13 @@ import time import sound import logging import config_utils -import shelve +import sqlitedict import application import os from . import session_exceptions as Exceptions log = logging.getLogger("sessionmanager.session") + class baseSession(object): """ toDo: Decorators does not seem to be working when using them in an inherited class.""" @@ -81,18 +82,19 @@ class baseSession(object): os.remove(shelfname+".dat") return try: - if not os.path.exists(shelfname+".dat"): - output.speak("Generating database, this might take a while.",True) - shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c') - for key, value in list(self.db.items()): - if type(key) != str and type(key) != str: - output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True) - log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!") - if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]: - shelf[key]=value[self.settings["general"]["persist_size"]:] - else: - shelf[key]=value - shelf.close() + self.db.commit() +# if not os.path.exists(shelfname+".dat"): +# output.speak("Generating database, this might take a while.",True) +# shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c') +# for key, value in list(self.db.items()): +# if type(key) != str and type(key) != str: +# output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True) +# log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!") +# if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]: +# shelf[key]=value[self.settings["general"]["persist_size"]:] +# else: +# shelf[key]=value +# shelf.close() except: output.speak("An exception occurred while shelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True) log.exception("Exception while shelving" + shelfname) @@ -106,10 +108,13 @@ class baseSession(object): os.remove(shelfname+".dat") return try: - shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c') - for key,value in list(shelf.items()): - self.db[key]=value - shelf.close() + self.db=sqlitedict.SqliteDict(os.path.join(paths.config_path(), shelfname), 'c') + if self.db.get("cursors") == None: + cursors = dict(direct_messages=-1) + self.db["cursors"] = cursors +# for key,value in list(shelf.items()): +# self.db[key]=value +# shelf.close() except: output.speak("An exception occurred while deshelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True) log.exception("Exception while deshelving" + shelfname) diff --git a/src/sessions/twitter/session.py b/src/sessions/twitter/session.py index 1cdbece0..01801319 100644 --- a/src/sessions/twitter/session.py +++ b/src/sessions/twitter/session.py @@ -38,6 +38,7 @@ class Session(base.baseSession): self.db[name] = [] if ("users" in self.db) == False: self.db["users"] = {} + objects = self.db[name] if ignore_older and len(self.db[name]) > 0: if self.settings["general"]["reverse_timelines"] == False: last_id = self.db[name][0].id @@ -52,12 +53,13 @@ class Session(base.baseSession): i = self.check_quoted_status(i) i = self.check_long_tweet(i) if i == False: continue - if self.settings["general"]["reverse_timelines"] == False: self.db[name].append(i) - else: self.db[name].insert(0, i) + if self.settings["general"]["reverse_timelines"] == False: objects.append(i) + else: objects.insert(0, i) num = num+1 if hasattr(i, "user"): if (i.user.id in self.db["users"]) == False: self.db["users"][i.user.id] = i.user + self.db[name] = objects return num def order_people(self, name, data): @@ -68,11 +70,13 @@ class Session(base.baseSession): num = 0 if (name in self.db) == False: self.db[name] = [] + objects = self.db[name] for i in data: if utils.find_item(i.id, self.db[name]) == None: - if self.settings["general"]["reverse_timelines"] == False: self.db[name].append(i) - else: self.db[name].insert(0, i) + if self.settings["general"]["reverse_timelines"] == False: objects.append(i) + else: objects.insert(0, i) num = num+1 + self.db[name] = objects return num def order_direct_messages(self, data): @@ -83,19 +87,25 @@ class Session(base.baseSession): sent = 0 if ("direct_messages" in self.db) == False: self.db["direct_messages"] = [] + if ("sent_direct_messages" in self.db) == False: + self.db["sent_direct_messages"] = [] + objects = self.db["direct_messages"] + sent_objects = self.db["sent_direct_messages"] for i in data: # Twitter returns sender_id as str, which must be converted to int in order to match to our user_id object. if int(i.message_create["sender_id"]) == self.db["user_id"]: if "sent_direct_messages" in self.db and utils.find_item(i.id, self.db["sent_direct_messages"]) == None: - if self.settings["general"]["reverse_timelines"] == False: self.db["sent_direct_messages"].append(i) - else: self.db["sent_direct_messages"].insert(0, i) + if self.settings["general"]["reverse_timelines"] == False: sent_objects.append(i) + else: sent_objects.insert(0, i) sent = sent+1 else: if utils.find_item(i.id, self.db["direct_messages"]) == None: - if self.settings["general"]["reverse_timelines"] == False: self.db["direct_messages"].append(i) - else: self.db["direct_messages"].insert(0, i) + if self.settings["general"]["reverse_timelines"] == False: objects.append(i) + else: objects.insert(0, i) incoming = incoming+1 pub.sendMessage("sent-dms-updated", total=sent, account=self.db["user_name"]) + self.db["direct_messages"] = objects + self.db["sent_direct_messages"] = sent_objects return incoming def __init__(self, *args, **kwargs):