mcvsoftware/twblue
1
0
Fork 0
mirror of https://github.com/MCV-Software/TWBlue.git synced 2025-08-27 10:19:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use slitedict to attempt to reduce memory usage when caching tweets

Browse Source
This commit is contained in:
Manuel Cortez
2021-06-23 13:40:21 -05:00
parent 39e1fb017c
commit 382acf7c8c
4 changed files with 43 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/sessions/base.py
View File

@@ -11,12 +11,13 @@ import time
import sound
import logging
import config_utils
import shelve
import sqlitedict
import application
import os
from . import session_exceptions as Exceptions
log = logging.getLogger("sessionmanager.session")
class baseSession(object):
""" toDo: Decorators does not seem to be working when using them in an inherited class."""
@@ -81,18 +82,19 @@ class baseSession(object):
os.remove(shelfname+".dat")
return
try:
if not os.path.exists(shelfname+".dat"):
output.speak("Generating database, this might take a while.",True)
shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c')
for key, value in list(self.db.items()):
if type(key) != str and type(key) != str:
output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True)
log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!")
if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]:
shelf[key]=value[self.settings["general"]["persist_size"]:]
else:
shelf[key]=value
shelf.close()
self.db.commit()
# if not os.path.exists(shelfname+".dat"):
# output.speak("Generating database, this might take a while.",True)
# shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c')
# for key, value in list(self.db.items()):
# if type(key) != str and type(key) != str:
# output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True)
# log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!")
# if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]:
# shelf[key]=value[self.settings["general"]["persist_size"]:]
# else:
# shelf[key]=value
# shelf.close()
except:
output.speak("An exception occurred while shelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True)
log.exception("Exception while shelving" + shelfname)
@@ -106,10 +108,13 @@ class baseSession(object):
os.remove(shelfname+".dat")
return
try:
shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c')
for key,value in list(shelf.items()):
self.db[key]=value
shelf.close()
self.db=sqlitedict.SqliteDict(os.path.join(paths.config_path(), shelfname), 'c')
if self.db.get("cursors") == None:
cursors = dict(direct_messages=-1)
self.db["cursors"] = cursors
# for key,value in list(shelf.items()):
# self.db[key]=value
# shelf.close()
except:
output.speak("An exception occurred while deshelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True)
log.exception("Exception while deshelving" + shelfname)