Merge pull request #414 from manuelcortez/improved_conversations

Improve conversation Support

.gitlab-ci.yml

@@ -38,6 +38,8 @@ twblue32:
     - cd ..
     - mv src/dist artifacts/TWBlue
     - move src/twblue.zip artifacts/twblue_x86.zip
+    # Move the generated script nsis file to artifacts, so we won't need python when generating the installer.
+    - move scripts/twblue.nsi artifacts/twblue.nsi
   only:
     - tags
   artifacts:
@@ -96,8 +98,9 @@ generate_versions:
   script:
     - move artifacts/TWBlue scripts/
     - move artifacts/TWBlue64 scripts/
+    - move artifacts/twblue.nsi scripts/installer.nsi
     - cd scripts
-    - '&$env:NSIS twblue.nsi'
+    - '&$env:NSIS installer.nsi'
     - move twblue_setup.exe ../artifacts
   only:
     - tags

scripts/twblue.nsi

@@ -15,10 +15,10 @@ SetCompressor /solid lzma
 SetDatablockOptimize on
 VIAddVersionKey ProductName "TWBlue"
 VIAddVersionKey LegalCopyright "Copyright 2014-2021 Manuel Cortéz."
-VIAddVersionKey ProductVersion "0.95"
+VIAddVersionKey ProductVersion "0.95.0"
-VIAddVersionKey FileVersion "0.95"
+VIAddVersionKey FileVersion "0.95.0"
-VIProductVersion "0.95"
+VIProductVersion "0.95.0"
-VIFileVersion "0.95"
+VIFileVersion "0.95.0"
 !insertmacro MUI_PAGE_WELCOME
 !define MUI_LICENSEPAGE_RADIOBUTTONS
 !insertmacro MUI_PAGE_LICENSE "license.txt"
@@ -32,7 +32,7 @@ var StartMenuFolder
 !insertmacro MUI_PAGE_FINISH
 !insertmacro MUI_UNPAGE_CONFIRM
 !insertmacro MUI_UNPAGE_INSTFILES
-  !insertmacro MUI_LANGUAGE "English"
+!insertmacro MUI_LANGUAGE "English"
 !insertmacro MUI_LANGUAGE "French"
 !insertmacro MUI_LANGUAGE "Spanish"
 !insertmacro MUI_LANGUAGE "Italian"

src/application.py

@@ -3,7 +3,6 @@ import datetime
 
 name = 'TWBlue'
 short_name='twblue'
-version = "11"
 update_url = 'https://twblue.es/updates/updates.php'
 mirror_update_url = 'https://raw.githubusercontent.com/manuelcortez/TWBlue/next-gen/updates/updates.json'
 authors = ["Manuel Cortéz", "José Manuel Delicado"]
@@ -14,3 +13,4 @@ translators = ["Manuel Cortéz (English)", "Mohammed Al Shara, Hatoun Felemban (
 url = u"https://twblue.es"
 report_bugs_url = "https://github.com/manuelcortez/twblue/issues"
 supported_languages = []
+version = "11"

src/controller/buffers/twitter/search.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import time
 import platform
+import locale
 if platform.system() == "Windows":
     from wxUI import  commonMessageDialogs
 elif platform.system() == "Linux":
@@ -8,6 +9,7 @@ elif platform.system() == "Linux":
     from gtkUI import commonMessageDialogs
 import widgetUtils
 import logging
+from tweepy.errors import TweepyException
 from . import base, people
 
 log = logging.getLogger("controller.buffers.twitter.searchBuffer")
@@ -64,35 +66,11 @@ class SearchPeopleBuffer(people.PeopleBuffer):
 class ConversationBuffer(SearchBuffer):
 
     def start_stream(self, start=False, mandatory=False, play_sound=True, avoid_autoreading=False):
-        # starts stream every 3 minutes.
         current_time = time.time()
         if self.execution_time == 0 or current_time-self.execution_time >= 180 or mandatory == True:
             self.execution_time = current_time
-            if start == True:
+            results = self.get_replies(self.tweet)
-                self.statuses = []
+            number_of_items = self.session.order_buffer(self.name, results)
-                self.ids = []
-                self.statuses.append(self.tweet)
-                self.ids.append(self.tweet.id)
-                tweet = self.tweet
-                if not hasattr(tweet, "in_reply_to_status_id"):
-                    tweet.in_reply_to_status_id = None
-                while tweet.in_reply_to_status_id != None:
-                    try:
-                        tweet = self.session.twitter.get_status(id=tweet.in_reply_to_status_id, tweet_mode="extended")
-                    except TweepError as err:
-                        break
-                    self.statuses.insert(0, tweet)
-                    self.ids.append(tweet.id)
-                if tweet.in_reply_to_status_id == None:
-                    self.kwargs["since_id"] = tweet.id
-                    self.ids.append(tweet.id)
-            val2 = self.session.search(self.name, tweet_mode="extended", *self.args, **self.kwargs)
-            for i in val2:
-                if i.in_reply_to_status_id in self.ids:
-                    self.statuses.append(i)
-                    self.ids.append(i.id)
-                    tweet = i
-            number_of_items = self.session.order_buffer(self.name, self.statuses)
             log.debug("Number of items retrieved: %d" % (number_of_items,))
             self.put_items_on_list(number_of_items)
             if number_of_items > 0 and self.sound != None and self.session.settings["sound"]["session_mute"] == False and self.name not in self.session.settings["other_buffers"]["muted_buffers"] and play_sound == True:
@@ -113,3 +91,33 @@ class ConversationBuffer(SearchBuffer):
             return True
         elif dlg == widgetUtils.NO:
             return False
+
+    def get_replies(self, tweet):
+        """ Try to retrieve the whole conversation for the passed object by using a mix between calls to API V1.1 and V2 """
+        results = []
+        # If the tweet that starts the conversation is a reply to something else, let's try to get the parent tweet first.
+        if hasattr(self, "in_reply_to_status_id") and self.tweet.in_reply_to_status_id != None:
+            try:
+                tweet2 = self.session.twitter_v2.get_tweet(id=self.tweet.in_reply_to_status_id, user_auth=True, tweet_fields=["conversation_id"])
+                results.append(tweet2)
+            except TweepyException as e:
+                log.exception("There was an error attempting to retrieve a parent tweet for the conversation for {}".format(self.name))
+        # Now, try to fetch the tweet initiating the conversation in V2 so we can get conversation_id
+        try:
+            tweet = self.session.twitter_v2.get_tweet(id=self.tweet.id, user_auth=True, tweet_fields=["conversation_id"])
+            results.append(tweet.data)
+            term = "conversation_id:{}".format(tweet.data.conversation_id)
+            tweets = self.session.twitter_v2.search_recent_tweets(term, user_auth=True, max_results=98)
+            if tweets.data != None:
+                results.extend(tweets.data)
+        except TweepyException as e:
+            log.exception("There was an error when attempting to retrieve the whole conversation for buffer {}".format(self.buffer.name))
+        new_results = []
+        ids = [tweet.id for tweet in results]
+        try:
+            results = self.session.twitter.lookup_statuses(ids, include_ext_alt_text=True, tweet_mode="extended")
+            results.sort(key=lambda x: x.id)
+        except TweepyException as e:
+            log.exception("There was an error attempting to retrieve tweets for Twitter API V1.1, in conversation buffer {}".format(self.name))
+            return []
+        return results

src/sessions/twitter/session.py

@@ -8,6 +8,7 @@ import wx
 import config
 import output
 import application
+import appkeys
 from pubsub import pub
 import tweepy
 from tweepy.errors import TweepyException, Forbidden, NotFound
@@ -135,9 +136,10 @@ class Session(base.baseSession):
         if self.settings["twitter"]["user_key"] != None and self.settings["twitter"]["user_secret"] != None:
             try:
                 log.debug("Logging in to twitter...")
-                self.auth = tweepy.OAuthHandler(keyring.get("api_key"), keyring.get("api_secret"))
+                self.auth = tweepy.OAuthHandler(appkeys.twitter_api_key, appkeys.twitter_api_secret)
                 self.auth.set_access_token(self.settings["twitter"]["user_key"], self.settings["twitter"]["user_secret"])
                 self.twitter = tweepy.API(self.auth)
+                self.twitter_v2 = tweepy.Client(consumer_key=appkeys.twitter_api_key, consumer_secret=appkeys.twitter_api_secret, access_token=self.settings["twitter"]["user_key"], access_token_secret=self.settings["twitter"]["user_secret"])
                 if verify_credentials == True:
                     self.credentials = self.twitter.verify_credentials()
                 self.logged = True
@@ -156,7 +158,7 @@ class Session(base.baseSession):
         if self.logged == True:
             raise Exceptions.AlreadyAuthorisedError("The authorisation process is not needed at this time.")
         else:
-            self.auth = tweepy.OAuthHandler(keyring.get("api_key"), keyring.get("api_secret"))
+            self.auth = tweepy.OAuthHandler(appkeys.twitter_api_key, appkeys.twitter_api_secret)
             redirect_url = self.auth.get_authorization_url()
             webbrowser.open_new_tab(redirect_url)
             self.authorisation_dialog = authorisationDialog()
@@ -523,7 +525,7 @@ class Session(base.baseSession):
     def start_streaming(self):
         if config.app["app-settings"]["no_streaming"]:
             return
-        self.stream = streaming.Stream(twitter_api=self.twitter, user=self.db["user_name"], user_id=self.db["user_id"], muted_users=self.db["muted_users"], consumer_key=keyring.get("api_key"), consumer_secret=keyring.get("api_secret"), access_token=self.settings["twitter"]["user_key"], access_token_secret=self.settings["twitter"]["user_secret"], chunk_size=1025)
+        self.stream = streaming.Stream(twitter_api=self.twitter, user=self.db["user_name"], user_id=self.db["user_id"], muted_users=self.db["muted_users"], consumer_key=appkeys.twitter_api_key, consumer_secret=appkeys.twitter_api_secret, access_token=self.settings["twitter"]["user_key"], access_token_secret=self.settings["twitter"]["user_secret"], chunk_size=1025)
         self.stream_thread = call_threaded(self.stream.filter, follow=self.stream.users, stall_warnings=True)
 
     def stop_streaming(self):

src/write_version_data.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python# -*- coding: iso-8859-1 -*-
 """ Write version info (taken from the last commit) to application.py. This method has been implemented this way for running updates.
 This file is not intended to be called by the user. It will be used only by the Gitlab CI runner."""
+import os
 import requests
 from codecs import open
 
@@ -27,3 +28,9 @@ file2 = open("..\\scripts\\twblue.nsi", "w", encoding="utf-8")
 file2.write(contents)
 file2.close()
 print("done")
+file3 = open("appkeys.py", "w")
+keys = """twitter_api_key = {}
+twitter_api_secret = {}
+""".format(os.environ.get("TWITTER_API_KEY"), os.environ.get("TWITTER_API_SECRET"))
+file3.write(keys)
+file3.close()