From 8bdc933bceacb576cc596b2802386013752740c9 Mon Sep 17 00:00:00 2001 From: Manuel Cortez Date: Wed, 20 Jan 2021 15:56:20 -0600 Subject: [PATCH] Migrated direct messages. Cursor support still require testing --- src/controller/buffers/twitterBuffers.py | 113 ++++++++++++++--------- src/controller/mainController.py | 26 +++--- src/sessions/twitter/compose.py | 12 +-- src/sessions/twitter/session.py | 42 +++++---- 4 files changed, 110 insertions(+), 83 deletions(-) diff --git a/src/controller/buffers/twitterBuffers.py b/src/controller/buffers/twitterBuffers.py index 2b07c02c..ed2393cd 100644 --- a/src/controller/buffers/twitterBuffers.py +++ b/src/controller/buffers/twitterBuffers.py @@ -162,12 +162,29 @@ class baseBufferController(baseBuffers.buffer): self.execution_time = current_time log.debug("Starting stream for buffer %s, account %s and type %s" % (self.name, self.account, self.type)) log.debug("args: %s, kwargs: %s" % (self.args, self.kwargs)) - if self.name == "direct_messages": - number_of_items = self.session.get_cursored_stream(self.name, self.function, *self.args, **self.kwargs) - else: + if self.name != "direct_messages": val = self.session.call_paged(self.function, *self.args, **self.kwargs) - number_of_items = self.session.order_buffer(self.name, val) - log.debug("Number of items retrieved: %d" % (number_of_items,)) + else: + # 50 results are allowed per API call, so let's assume max value can be 50. + # reference: https://developer.twitter.com/en/docs/twitter-api/v1/direct-messages/sending-and-receiving/api-reference/list-events + if self.session.settings["general"]["max_tweets_per_call"] > 50: + count = 50 + else: + count = self.session.settings["general"]["max_tweets_per_call"] + # try to retrieve the cursor for the current buffer. + cursor = self.session.db["cursors"].get(self.name) + try: + # We need to assign all results somewhere else so the cursor variable would b generated. + val = Cursor(getattr(self.session.twitter, self.function), *self.args, **self.kwargs).items(count) + results = [i for i in val] + self.session.db["cursors"][self.name] = val.page_iterator.next_cursor + val = results + val.reverse() + except TweepError as e: + log.error("Error %s: %s" % (e.api_code, e.reason)) + return + number_of_items = self.session.order_buffer(self.name, val) + log.debug("Number of items retrieved: %d" % (number_of_items,)) self.put_items_on_list(number_of_items) if hasattr(self, "finished_timeline") and self.finished_timeline == False: if "-timeline" in self.name: @@ -268,11 +285,7 @@ class baseBufferController(baseBuffers.buffer): self.remove_item(i) def put_items_on_list(self, number_of_items): - # Define the list we're going to use as cursored stuff are a bit different. - if self.name != "direct_messages" and self.name != "sent_direct_messages": - list_to_use = self.session.db[self.name] - else: - list_to_use = self.session.db[self.name]["items"] + list_to_use = self.session.db[self.name] if number_of_items == 0 and self.session.settings["general"]["persist_size"] == 0: return log.debug("The list contains %d items " % (self.buffer.list.get_count(),)) log.debug("Putting %d items on the list" % (number_of_items,)) @@ -441,7 +454,7 @@ class baseBufferController(baseBuffers.buffer): screen_name = tweet.screen_name users = [screen_name] else: - screen_name = tweet.user["screen_name"] + screen_name = tweet.user.screen_name users = utils.get_all_users(tweet, self.session.db) dm = messages.dm(self.session, _(u"Direct message to %s") % (screen_name,), _(u"New direct message"), users) if dm.message.get_response() == widgetUtils.OK: @@ -452,9 +465,9 @@ class baseBufferController(baseBuffers.buffer): val = self.session.api_call(call_name="send_direct_message", recipient_id=recipient_id, text=text) if val != None: if self.session.settings["general"]["reverse_timelines"] == False: - self.session.db["sent_direct_messages"]["items"].append(val) + self.session.db["sent_direct_messages"].append(val) else: - self.session.db["sent_direct_messages"]["items"].insert(0, val) + self.session.db["sent_direct_messages"].insert(0, val) pub.sendMessage("sent-dm", data=val, user=self.session.db["user_name"]) if hasattr(dm.message, "destroy"): dm.message.destroy() @@ -611,53 +624,65 @@ class baseBufferController(baseBuffers.buffer): class directMessagesController(baseBufferController): def get_more_items(self): + # 50 results are allowed per API call, so let's assume max value can be 50. + # reference: https://developer.twitter.com/en/docs/twitter-api/v1/direct-messages/sending-and-receiving/api-reference/list-events + if self.session.settings["general"]["max_tweets_per_call"] > 50: + count = 50 + else: + count = self.session.settings["general"]["max_tweets_per_call"] + total = 0 + # try to retrieve the cursor for the current buffer. + cursor = self.session.db["cursors"].get(self.name) try: - items = self.session.get_more_items(self.function, dm=True, name=self.name, count=self.session.settings["general"]["max_tweets_per_call"], cursor=self.session.db[self.name]["cursor"], *self.args, **self.kwargs) + items = Cursor(getattr(self.session.twitter, self.function), cursor=cursor, *self.args, **self.kwargs).items(count) + results = [i for i in items] + self.session.db["cursors"][self.name] = items.page_iterator.next_cursor + items = results except TweepError as e: - output.speak(e.reason, True) + log.error("Error %s: %s" % (e.api_code, e.reason)) return if items == None: return sent = [] + received = [] for i in items: - if i["message_create"]["sender_id"] == self.session.db["user_id"]: + if int(i.message_create["sender_id"]) == self.session.db["user_id"]: if self.session.settings["general"]["reverse_timelines"] == False: - self.session.db["sent_direct_messages"]["items"].insert(0, i) + self.session.db["sent_direct_messages"].insert(0, i) + sent.append(i) else: - self.session.db["sent_direct_messages"]["items"].append(i) - sent.append(i) + self.session.db["sent_direct_messages"].append(i) + sent.insert(0, i) else: if self.session.settings["general"]["reverse_timelines"] == False: - self.session.db[self.name]["items"].insert(0, i) + self.session.db[self.name].insert(0, i) + received.append(i) else: - self.session.db[self.name]["items"].append(i) + self.session.db[self.name].append(i) + received.insert(0, i) + total = total+1 pub.sendMessage("more-sent-dms", data=sent, account=self.session.db["user_name"]) selected = self.buffer.list.get_selected() + if self.session.settings["general"]["reverse_timelines"] == True: - for i in items: - if i["message_create"]["sender_id"] == self.session.db["user_id"]: + for i in received: + if int(i.message_create["sender_id"]) == self.session.db["user_id"]: continue tweet = self.compose_function(i, self.session.db, self.session.settings["general"]["relative_times"], self.session.settings["general"]["show_screen_names"], self.session) self.buffer.list.insert_item(True, *tweet) self.buffer.list.select_item(selected) else: - for i in items: - if i["message_create"]["sender_id"] == self.session.db["user_id"]: + for i in received: + if int(i.message_create["sender_id"]) == self.session.db["user_id"]: continue tweet = self.compose_function(i, self.session.db, self.session.settings["general"]["relative_times"], self.session.settings["general"]["show_screen_names"], self.session) self.buffer.list.insert_item(True, *tweet) - output.speak(_(u"%s items retrieved") % (len(items)), True) - - def get_tweet(self): - tweet = self.session.db[self.name]["items"][self.buffer.list.get_selected()] - return tweet - - get_right_tweet = get_tweet + output.speak(_(u"%s items retrieved") % (total), True) @_tweets_exist def reply(self, *args, **kwargs): tweet = self.get_right_tweet() - screen_name = self.session.get_user(tweet.message_create["sender_id"])["screen_name"] + screen_name = self.session.get_user(tweet.message_create["sender_id"]).screen_name message = messages.reply(self.session, _(u"Mention"), _(u"Mention to %s") % (screen_name,), "@%s " % (screen_name,), [screen_name,]) if message.message.get_response() == widgetUtils.OK: if config.app["app-settings"]["remember_mention_and_longtweet"]: @@ -675,7 +700,7 @@ class directMessagesController(baseBufferController): tweet = self.get_tweet() if platform.system() == "Windows" and self.session.settings["general"]["relative_times"] == True: # fix this: - original_date = arrow.get(int(tweet["created_timestamp"][:-3])) + original_date = arrow.get(int(tweet.created_timestamp)) ts = original_date.humanize(locale=languageHandler.getLanguage()) self.buffer.list.list.SetItem(self.buffer.list.get_selected(), 2, ts) if self.session.settings['sound']['indicate_audio'] and utils.is_audio(tweet): @@ -686,15 +711,15 @@ class directMessagesController(baseBufferController): def clear_list(self): dlg = commonMessageDialogs.clear_list() if dlg == widgetUtils.YES: - self.session.db[self.name]["items"] = [] + self.session.db[self.name] = [] self.buffer.list.clear() def auto_read(self, number_of_items): if number_of_items == 1 and self.name in self.session.settings["other_buffers"]["autoread_buffers"] and self.name not in self.session.settings["other_buffers"]["muted_buffers"] and self.session.settings["sound"]["session_mute"] == False: if self.session.settings["general"]["reverse_timelines"] == False: - tweet = self.session.db[self.name]["items"][-1] + tweet = self.session.db[self.name][-1] else: - tweet = self.session.db[self.name]["items"][0] + tweet = self.session.db[self.name][0] output.speak(_(u"New direct message")) output.speak(" ".join(self.compose_function(tweet, self.session.db, self.session.settings["general"]["relative_times"], self.session.settings["general"]["show_screen_names"], self.session))) elif number_of_items > 1 and self.name in self.session.settings["other_buffers"]["autoread_buffers"] and self.name not in self.session.settings["other_buffers"]["muted_buffers"] and self.session.settings["sound"]["session_mute"] == False: @@ -708,7 +733,7 @@ class sentDirectMessagesController(directMessagesController): def __init__(self, *args, **kwargs): super(sentDirectMessagesController, self).__init__(*args, **kwargs) if ("sent_direct_messages" in self.session.db) == False: - self.session.db["sent_direct_messages"] = {"items": []} + self.session.db["sent_direct_messages"] = [] def get_more_items(self): output.speak(_(u"Getting more items cannot be done in this buffer. Use the direct messages buffer instead.")) @@ -720,11 +745,11 @@ class sentDirectMessagesController(directMessagesController): if self.session.settings["general"]["reverse_timelines"] == True: for i in items: tweet = self.compose_function(i, self.session.db, self.session.settings["general"]["relative_times"], self.session.settings["general"]["show_screen_names"], self.session) - self.buffer.list.insert_item(True, *tweet) + self.buffer.list.insert_item(False, *tweet) else: for i in items: tweet = self.compose_function(i, self.session.db, self.session.settings["general"]["relative_times"], self.session.settings["general"]["show_screen_names"], self.session) - self.buffer.list.insert_item(True, *tweet) + self.buffer.list.insert_item(False, *tweet) class listBufferController(baseBufferController): def __init__(self, parent, function, name, sessionObject, account, sound=None, bufferType=None, list_id=None, *args, **kwargs): @@ -820,7 +845,7 @@ class peopleBufferController(baseBufferController): @_tweets_exist def reply(self, *args, **kwargs): tweet = self.get_right_tweet() - screen_name = tweet["screen_name"] + screen_name = tweet.screen_name message = messages.reply(self.session, _(u"Mention"), _(u"Mention to %s") % (screen_name,), "@%s " % (screen_name,), [screen_name,]) if message.message.get_response() == widgetUtils.OK: if config.app["app-settings"]["remember_mention_and_longtweet"]: @@ -844,7 +869,7 @@ class peopleBufferController(baseBufferController): val = self.session.get_cursored_stream(self.name, self.function, *self.args, **self.kwargs) self.put_items_on_list(val) if hasattr(self, "finished_timeline") and self.finished_timeline == False: - self.username = self.session.api_call("show_user", **self.kwargs)["screen_name"] + self.username = self.session.api_call("get_user", **self.kwargs).screen_name self.finished_timeline = True if val > 0 and self.sound != None and self.session.settings["sound"]["session_mute"] == False and self.name not in self.session.settings["other_buffers"]["muted_buffers"] and play_sound == True: self.session.sound.play(self.sound) @@ -921,7 +946,7 @@ class peopleBufferController(baseBufferController): self.buffer.list.clear() def interact(self): - user.profileController(self.session, user=self.get_right_tweet()["screen_name"]) + user.profileController(self.session, user=self.get_right_tweet().screen_name) def show_menu(self, ev, pos=0, *args, **kwargs): menu = menus.peoplePanelMenu() @@ -954,7 +979,7 @@ class peopleBufferController(baseBufferController): def open_in_browser(self, *args, **kwargs): tweet = self.get_tweet() output.speak(_(u"Opening item in web browser...")) - url = "https://twitter.com/{screen_name}".format(screen_name=tweet["screen_name"]) + url = "https://twitter.com/{screen_name}".format(screen_name=tweet.screen_name) webbrowser.open(url) class searchBufferController(baseBufferController): diff --git a/src/controller/mainController.py b/src/controller/mainController.py index 5f8c672a..608dc900 100644 --- a/src/controller/mainController.py +++ b/src/controller/mainController.py @@ -528,7 +528,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) dlg = dialogs.utils.selectUserDialog(_(u"Select the user"), users) @@ -545,7 +545,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) dlg = dialogs.utils.selectUserDialog(_(u"Select the user"), users) @@ -573,7 +573,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) dlg = dialogs.utils.selectUserDialog(_(u"Select the user"), users) @@ -667,7 +667,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) u = userActionsController.userActionsController(buff, users) @@ -679,7 +679,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) u = userActionsController.userActionsController(buff, users, "unfollow") @@ -691,7 +691,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) u = userActionsController.userActionsController(buff, users, "mute") @@ -703,7 +703,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) u = userActionsController.userActionsController(buff, users, "unmute") @@ -715,7 +715,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) u = userActionsController.userActionsController(buff, users, "block") @@ -727,7 +727,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) u = userActionsController.userActionsController(buff, users, "unblock") @@ -739,7 +739,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) u = userActionsController.userActionsController(buff, users, "report") @@ -804,7 +804,7 @@ class Controller(object): elif buffer.type == "dm": non_tweet = buffer.get_formatted_message() item = buffer.get_right_tweet() - original_date = arrow.get(int(item["created_timestamp"][:-3])) + original_date = arrow.get(int(item.created_timestamp)) date = original_date.shift(seconds=buffer.session.db["utc_offset"]).format(_(u"MMM D, YYYY. H:m"), locale=languageHandler.getLanguage()) msg = messages.viewTweet(non_tweet, [], False, date=date) else: @@ -826,7 +826,7 @@ class Controller(object): if buff.type == "people": users = [tweet.screen_name] elif buff.type == "dm": - users = [buff.session.get_user(tweet.message_create["sender_id"])["screen_name"]] + users = [buff.session.get_user(tweet.message_create["sender_id"]).screen_name] else: users = utils.get_all_users(tweet, buff.session.db) dlg = dialogs.userSelection.selectUserDialog(users=users, default=default) @@ -1259,7 +1259,7 @@ class Controller(object): def manage_sent_dm(self, data, user): buffer = self.search_buffer("sent_direct_messages", user) - if buffer == Nogne: return + if buffer == None: return play_sound = "dm_sent.ogg" if "sent_direct_messages" not in buffer.session.settings["other_buffers"]["muted_buffers"]: self.notify(buffer.session, play_sound=play_sound) diff --git a/src/sessions/twitter/compose.py b/src/sessions/twitter/compose.py index d2a73e9f..7f5c3620 100644 --- a/src/sessions/twitter/compose.py +++ b/src/sessions/twitter/compose.py @@ -82,26 +82,26 @@ def compose_direct_message(item, db, relative_times, show_screen_names=False, se if system == "Windows": # Let's remove the last 3 digits in the timestamp string. # Twitter sends their "epoch" timestamp with 3 digits for milliseconds and arrow doesn't like it. - original_date = arrow.get(int(item.created_timestamp[:-3])) + original_date = arrow.get(int(item.created_timestamp)) if relative_times == True: ts = original_date.humanize(locale=languageHandler.curLang[:2]) else: ts = original_date.shift(seconds=db["utc_offset"]).format(_(u"dddd, MMMM D, YYYY H:m:s"), locale=languageHandler.curLang[:2]) else: ts = item.created_timestamp - text = StripChars(item.message_create.message_data.text) + text = StripChars(item.message_create["message_data"]["text"]) source = "DM" sender = session.get_user(item.message_create["sender_id"]) - if db["user_name"] == sender["screen_name"]: + if db["user_name"] == sender.screen_name: if show_screen_names: user = _(u"Dm to %s ") % (session.get_user(item.message_create["target"]["recipient_id"]).screen_name) else: - user = _(u"Dm to %s ") % (session.get_user(item["message_create"]["target"]["recipient_id"]).name) + user = _(u"Dm to %s ") % (session.get_user(item.message_create["target"]["recipient_id"]).name) else: if show_screen_names: - user = sender["screen_name"] + user = sender.screen_name else: - user = sender["name"] + user = sender.name if text[-1] in chars: text=text+"." text = utils.expand_urls(text, item.message_create["message_data"]["entities"]) return [user+", ", text, ts+", ", source] diff --git a/src/sessions/twitter/session.py b/src/sessions/twitter/session.py index 00dcbeae..0105678a 100644 --- a/src/sessions/twitter/session.py +++ b/src/sessions/twitter/session.py @@ -1,8 +1,5 @@ # -*- coding: utf-8 -*- """ This is the main session needed to access all Twitter Features.""" -from __future__ import absolute_import -from __future__ import unicode_literals -from builtins import range import os import time import logging @@ -12,8 +9,8 @@ import config import output import application from pubsub import pub -from twython import Twython import tweepy +from tweepy.error import TweepError from mysc.thread_utils import call_threaded from keys import keyring from sessions import base @@ -32,6 +29,8 @@ class Session(base.baseSession): data list: A list with tweets. ignore_older bool: if set to True, items older than the first element on the list will be ignored. returns the number of items that have been added in this execution""" + if name == "direct_messages": + return self.order_direct_messages(data) num = 0 last_id = None if (name in self.db) == False: @@ -87,24 +86,27 @@ class Session(base.baseSession): incoming = 0 sent = 0 if ("direct_messages" in self.db) == False: - self.db["direct_messages"] = {} - self.db["direct_messages"]["items"] = [] + self.db["direct_messages"] = [] for i in data: - if i.message_create.sender_id == self.db["user_id"]: - if "sent_direct_messages" in self.db and utils.find_item(i.id, self.db["sent_direct_messages"]["items"]) == None: - if self.settings["general"]["reverse_timelines"] == False: self.db["sent_direct_messages"]["items"].append(i) - else: self.db["sent_direct_messages"]["items"].insert(0, i) + # Twitter returns sender_id as str, which must be converted to int in order to match to our user_id object. + if int(i.message_create["sender_id"]) == self.db["user_id"]: + if "sent_direct_messages" in self.db and utils.find_item(i.id, self.db["sent_direct_messages"]) == None: + if self.settings["general"]["reverse_timelines"] == False: self.db["sent_direct_messages"].append(i) + else: self.db["sent_direct_messages"].insert(0, i) sent = sent+1 else: - if utils.find_item(i.id, self.db["direct_messages"]["items"]) == None: - if self.settings["general"]["reverse_timelines"] == False: self.db["direct_messages"]["items"].append(i) - else: self.db["direct_messages"]["items"].insert(0, i) + if utils.find_item(i.id, self.db["direct_messages"]) == None: + if self.settings["general"]["reverse_timelines"] == False: self.db["direct_messages"].append(i) + else: self.db["direct_messages"].insert(0, i) incoming = incoming+1 pub.sendMessage("sent-dms-updated", total=sent, account=self.db["user_name"]) return incoming def __init__(self, *args, **kwargs): super(Session, self).__init__(*args, **kwargs) + # Adds here the optional cursors objects. + cursors = dict(direct_messages=-1) + self.db["cursors"] = cursors self.reconnection_function_active = False self.counter = 0 self.lists = [] @@ -410,11 +412,11 @@ class Session(base.baseSession): returns an user dict.""" if ("users" in self.db) == False or (id in self.db["users"]) == False: try: - user = self.twitter.show_user(id=id) - except TwythonError: + user = self.twitter.get_user(id=id) + except TweepError as err: user = dict(screen_name="deleted_account", name="Deleted account") return user - self.db["users"][user["id_str"]] = user + self.db["users"][user.id_str] = user return user else: return self.db["users"][id] @@ -429,8 +431,8 @@ class Session(base.baseSession): return user["id_str"] else: for i in list(self.db["users"].keys()): - if self.db["users"][i]["screen_name"] == screen_name: - return self.db["users"][i]["id_str"] + if self.db["users"][i].screen_name == screen_name: + return self.db["users"][i].id_str user = utils.if_user_exists(self.twitter, screen_name) - self.db["users"][user["id_str"]] = user - return user["id_str"] \ No newline at end of file + self.db["users"][user.id_str] = user + return user.id_str \ No newline at end of file