mcvsoftware/twblue
1
0
Fork 0
mirror of https://github.com/MCV-Software/TWBlue.git synced 2024-11-22 19:28:09 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use slitedict to attempt to reduce memory usage when caching tweets

Browse Source
This commit is contained in:
Manuel Cortez 2021-06-23 13:40:21 -05:00
parent 39e1fb017c
commit 382acf7c8c
4 changed files with 43 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -31,6 +31,7 @@ cx_freeze
tweepy
twitter-text-parser
pyenchant
sqlitedict
git+https://github.com/accessibleapps/libloader
git+https://github.com/accessibleapps/platform_utils
git+https://github.com/accessibleapps/accessible_output2

4
src/controller/mainController.py
View File

@ -254,8 +254,8 @@ class Controller(object):
# Connection checker executed each minute.
self.checker_function = RepeatingTimer(60, self.check_connection)
# self.checker_function.start()
self.save_db = RepeatingTimer(300, self.save_data_in_db)
self.save_db.start()
# self.save_db = RepeatingTimer(300, self.save_data_in_db)
# self.save_db.start()
log.debug("Setting updates to buffers every %d seconds..." % (60*config.app["app-settings"]["update_period"],))
self.update_buffers_function = RepeatingTimer(60*config.app["app-settings"]["update_period"], self.update_buffers)
self.update_buffers_function.start()

39
src/sessions/base.py
View File

@ -11,12 +11,13 @@ import time
import sound
import logging
import config_utils
import shelve
import sqlitedict
import application
import os
from . import session_exceptions as Exceptions
log = logging.getLogger("sessionmanager.session")
class baseSession(object):
""" toDo: Decorators does not seem to be working when using them in an inherited class."""
@ -81,18 +82,19 @@ class baseSession(object):
os.remove(shelfname+".dat")
return
try:
if not os.path.exists(shelfname+".dat"):
output.speak("Generating database, this might take a while.",True)
shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c')
for key, value in list(self.db.items()):
if type(key) != str and type(key) != str:
output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True)
log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!")
if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]:
shelf[key]=value[self.settings["general"]["persist_size"]:]
else:
shelf[key]=value
shelf.close()
self.db.commit()
# if not os.path.exists(shelfname+".dat"):
# output.speak("Generating database, this might take a while.",True)
# shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c')
# for key, value in list(self.db.items()):
# if type(key) != str and type(key) != str:
# output.speak("Uh oh, while shelving the database, a key of type " + str(type(key)) + " has been found. It will be converted to type str, but this will cause all sorts of problems on deshelve. Please bring this to the attention of the " + application.name + " developers immediately. More information about the error will be written to the error log.",True)
# log.error("Uh oh, " + str(key) + " is of type " + str(type(key)) + "!")
# if type(value) == list and self.settings["general"]["persist_size"] != -1 and len(value) > self.settings["general"]["persist_size"]:
# shelf[key]=value[self.settings["general"]["persist_size"]:]
# else:
# shelf[key]=value
# shelf.close()
except:
output.speak("An exception occurred while shelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True)
log.exception("Exception while shelving" + shelfname)
@ -106,10 +108,13 @@ class baseSession(object):
os.remove(shelfname+".dat")
return
try:
shelf=shelve.open(os.path.join(paths.config_path(), shelfname),'c')
for key,value in list(shelf.items()):
self.db[key]=value
shelf.close()
self.db=sqlitedict.SqliteDict(os.path.join(paths.config_path(), shelfname), 'c')
if self.db.get("cursors") == None:
cursors = dict(direct_messages=-1)
self.db["cursors"] = cursors
# for key,value in list(shelf.items()):
# self.db[key]=value
# shelf.close()
except:
output.speak("An exception occurred while deshelving the " + application.name + " database. It will be deleted and rebuilt automatically. If this error persists, send the error log to the " + application.name + " developers.",True)
log.exception("Exception while deshelving" + shelfname)

26
src/sessions/twitter/session.py
View File

@ -38,6 +38,7 @@ class Session(base.baseSession):
self.db[name] = []
if ("users" in self.db) == False:
self.db["users"] = {}
objects = self.db[name]
if ignore_older and len(self.db[name]) > 0:
if self.settings["general"]["reverse_timelines"] == False:
last_id = self.db[name][0].id
@ -52,12 +53,13 @@ class Session(base.baseSession):
i = self.check_quoted_status(i)
i = self.check_long_tweet(i)
if i == False: continue
if self.settings["general"]["reverse_timelines"] == False: self.db[name].append(i)
else: self.db[name].insert(0, i)
if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: objects.insert(0, i)
num = num+1
if hasattr(i, "user"):
if (i.user.id in self.db["users"]) == False:
self.db["users"][i.user.id] = i.user
self.db[name] = objects
return num
def order_people(self, name, data):
@ -68,11 +70,13 @@ class Session(base.baseSession):
num = 0
if (name in self.db) == False:
self.db[name] = []
objects = self.db[name]
for i in data:
if utils.find_item(i.id, self.db[name]) == None:
if self.settings["general"]["reverse_timelines"] == False: self.db[name].append(i)
else: self.db[name].insert(0, i)
if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: objects.insert(0, i)
num = num+1
self.db[name] = objects
return num
def order_direct_messages(self, data):
@ -83,19 +87,25 @@ class Session(base.baseSession):
sent = 0
if ("direct_messages" in self.db) == False:
self.db["direct_messages"] = []
if ("sent_direct_messages" in self.db) == False:
self.db["sent_direct_messages"] = []
objects = self.db["direct_messages"]
sent_objects = self.db["sent_direct_messages"]
for i in data:
# Twitter returns sender_id as str, which must be converted to int in order to match to our user_id object.
if int(i.message_create["sender_id"]) == self.db["user_id"]:
if "sent_direct_messages" in self.db and utils.find_item(i.id, self.db["sent_direct_messages"]) == None:
if self.settings["general"]["reverse_timelines"] == False: self.db["sent_direct_messages"].append(i)
else: self.db["sent_direct_messages"].insert(0, i)
if self.settings["general"]["reverse_timelines"] == False: sent_objects.append(i)
else: sent_objects.insert(0, i)
sent = sent+1
else:
if utils.find_item(i.id, self.db["direct_messages"]) == None:
if self.settings["general"]["reverse_timelines"] == False: self.db["direct_messages"].append(i)
else: self.db["direct_messages"].insert(0, i)
if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: objects.insert(0, i)
incoming = incoming+1
pub.sendMessage("sent-dms-updated", total=sent, account=self.db["user_name"])
self.db["direct_messages"] = objects
self.db["sent_direct_messages"] = sent_objects
return incoming
def __init__(self, *args, **kwargs):