mcvsoftware/twblue
1
0
Fork 0
mirror of https://github.com/MCV-Software/TWBlue.git synced 2025-02-19 16:33:10 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use slitedict to attempt to reduce memory usage when caching tweets

Browse Source
This commit is contained in:
Manuel Cortez 2021-06-23 13:40:21 -05:00
parent 39e1fb017c
commit 382acf7c8c
4 changed files with 43 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -31,6 +31,7 @@ cx_freeze
tweepy tweepy
twitter-text-parser twitter-text-parser
pyenchant pyenchant
sqlitedict
git+https://github.com/accessibleapps/libloader git+https://github.com/accessibleapps/libloader
git+https://github.com/accessibleapps/platform_utils git+https://github.com/accessibleapps/platform_utils
git+https://github.com/accessibleapps/accessible_output2 git+https://github.com/accessibleapps/accessible_output2

4
src/controller/mainController.py
View File

@ -254,8 +254,8 @@ class Controller(object):
# Connection checker executed each minute. # Connection checker executed each minute.
self.checker_function = RepeatingTimer(60, self.check_connection) self.checker_function = RepeatingTimer(60, self.check_connection)
# self.checker_function.start() # self.checker_function.start()
self.save_db = RepeatingTimer(300, self.save_data_in_db) # self.save_db = RepeatingTimer(300, self.save_data_in_db)
self.save_db.start() # self.save_db.start()
log.debug("Setting updates to buffers every %d seconds..." % (60*config.app["app-settings"]["update_period"],)) log.debug("Setting updates to buffers every %d seconds..." % (60*config.app["app-settings"]["update_period"],))
self.update_buffers_function = RepeatingTimer(60*config.app["app-settings"]["update_period"], self.update_buffers) self.update_buffers_function = RepeatingTimer(60*config.app["app-settings"]["update_period"], self.update_buffers)
self.update_buffers_function.start() self.update_buffers_function.start()

39
src/sessions/base.py
View File

@ -11,12 +11,13 @@ import time
import sound import sound
import logging import logging
import config_utils import config_utils
import shelve import sqlitedict
import application import application
import os import os
from . import session_exceptions as Exceptions from . import session_exceptions as Exceptions
log = logging.getLogger("sessionmanager.session") log = logging.getLogger("sessionmanager.session")
class baseSession(object): class baseSession(object):
""" toDo: Decorators does not seem to be working when using them in an inherited class.""" """ toDo: Decorators does not seem to be working when using them in an inherited class."""
@ -81,18 +82,19 @@ class baseSession(object):
os.remove(shelfname+".dat") os.remove(shelfname+".dat")
return return
try: try:
if not os.path.exists(shelfname+".dat"): self.db.commit()
output.speak("Generating database, this might take a while.",True) # if not os.path.exists(shelfname+".dat"):
shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c') # output.speak("Generating database, this might take a while.",True)
for key, value in list(self.db.items()): # shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c')
if type(key) != str and type(key) != str: # for key, value in list(self.db.items()):
output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True) # if type(key) != str and type(key) != str:
log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!") # output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True)
if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]: # log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!")
shelf[key]=value[self.settings["general"]["persist_size"]:] # if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]:
else: # shelf[key]=value[self.settings["general"]["persist_size"]:]
shelf[key]=value # else:
shelf.close() # shelf[key]=value
# shelf.close()
except: except:
output.speak("An exception occurred while shelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True) output.speak("An exception occurred while shelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True)
log.exception("Exception while shelving" + shelfname) log.exception("Exception while shelving" + shelfname)
@ -106,10 +108,13 @@ class baseSession(object):
os.remove(shelfname+".dat") os.remove(shelfname+".dat")
return return
try: try:
shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c') self.db=sqlitedict.SqliteDict(os.path.join(paths.config_path(), shelfname), 'c')
for key,value in list(shelf.items()): if self.db.get("cursors") == None:
self.db[key]=value cursors = dict(direct_messages=-1)
shelf.close() self.db["cursors"] = cursors
# for key,value in list(shelf.items()):
# self.db[key]=value
# shelf.close()
except: except:
output.speak("An exception occurred while deshelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True) output.speak("An exception occurred while deshelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True)
log.exception("Exception while deshelving" + shelfname) log.exception("Exception while deshelving" + shelfname)

26
src/sessions/twitter/session.py
View File

@ -38,6 +38,7 @@ class Session(base.baseSession):
self.db[name] = [] self.db[name] = []
if ("users" in self.db) == False: if ("users" in self.db) == False:
self.db["users"] = {} self.db["users"] = {}
objects = self.db[name]
if ignore_older and len(self.db[name]) > 0: if ignore_older and len(self.db[name]) > 0:
if self.settings["general"]["reverse_timelines"] == False: if self.settings["general"]["reverse_timelines"] == False:
last_id = self.db[name][0].id last_id = self.db[name][0].id
@ -52,12 +53,13 @@ class Session(base.baseSession):
i = self.check_quoted_status(i) i = self.check_quoted_status(i)
i = self.check_long_tweet(i) i = self.check_long_tweet(i)
if i == False: continue if i == False: continue
if self.settings["general"]["reverse_timelines"] == False: self.db[name].append(i) if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: self.db[name].insert(0, i) else: objects.insert(0, i)
num = num+1 num = num+1
if hasattr(i, "user"): if hasattr(i, "user"):
if (i.user.id in self.db["users"]) == False: if (i.user.id in self.db["users"]) == False:
self.db["users"][i.user.id] = i.user self.db["users"][i.user.id] = i.user
self.db[name] = objects
return num return num
def order_people(self, name, data): def order_people(self, name, data):
@ -68,11 +70,13 @@ class Session(base.baseSession):
num = 0 num = 0
if (name in self.db) == False: if (name in self.db) == False:
self.db[name] = [] self.db[name] = []
objects = self.db[name]
for i in data: for i in data:
if utils.find_item(i.id, self.db[name]) == None: if utils.find_item(i.id, self.db[name]) == None:
if self.settings["general"]["reverse_timelines"] == False: self.db[name].append(i) if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: self.db[name].insert(0, i) else: objects.insert(0, i)
num = num+1 num = num+1
self.db[name] = objects
return num return num
def order_direct_messages(self, data): def order_direct_messages(self, data):
@ -83,19 +87,25 @@ class Session(base.baseSession):
sent = 0 sent = 0
if ("direct_messages" in self.db) == False: if ("direct_messages" in self.db) == False:
self.db["direct_messages"] = [] self.db["direct_messages"] = []
if ("sent_direct_messages" in self.db) == False:
self.db["sent_direct_messages"] = []
objects = self.db["direct_messages"]
sent_objects = self.db["sent_direct_messages"]
for i in data: for i in data:
# Twitter returns sender_id as str, which must be converted to int in order to match to our user_id object. # Twitter returns sender_id as str, which must be converted to int in order to match to our user_id object.
if int(i.message_create["sender_id"]) == self.db["user_id"]: if int(i.message_create["sender_id"]) == self.db["user_id"]:
if "sent_direct_messages" in self.db and utils.find_item(i.id, self.db["sent_direct_messages"]) == None: if "sent_direct_messages" in self.db and utils.find_item(i.id, self.db["sent_direct_messages"]) == None:
if self.settings["general"]["reverse_timelines"] == False: self.db["sent_direct_messages"].append(i) if self.settings["general"]["reverse_timelines"] == False: sent_objects.append(i)
else: self.db["sent_direct_messages"].insert(0, i) else: sent_objects.insert(0, i)
sent = sent+1 sent = sent+1
else: else:
if utils.find_item(i.id, self.db["direct_messages"]) == None: if utils.find_item(i.id, self.db["direct_messages"]) == None:
if self.settings["general"]["reverse_timelines"] == False: self.db["direct_messages"].append(i) if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: self.db["direct_messages"].insert(0, i) else: objects.insert(0, i)
incoming = incoming+1 incoming = incoming+1
pub.sendMessage("sent-dms-updated", total=sent, account=self.db["user_name"]) pub.sendMessage("sent-dms-updated", total=sent, account=self.db["user_name"])
self.db["direct_messages"] = objects
self.db["sent_direct_messages"] = sent_objects
return incoming return incoming
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):