mcvsoftware/twblue
1
0
Fork 0
mirror of https://github.com/MCV-Software/TWBlue.git synced 2024-10-17 01:49:14 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Streaming API: Ignore retweets if original tweet is present in a buffer

Browse Source
This commit is contained in:
Manuel Cortez 2021-07-16 10:22:51 -05:00
parent b2b9cd810f
commit 24d1ad093d
3 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/controller/buffers/twitter/base.py
View File

@ -237,7 +237,7 @@ class BaseBuffer(base.Buffer):
items_db = self.session.db[self.name] items_db = self.session.db[self.name]
self.session.add_users_from_results(items) self.session.add_users_from_results(items)
for i in items: for i in items:
if utils.is_allowed(i, self.session.settings, self.name) == True and utils.find_item(i.id, self.session.db[self.name]) == None: if utils.is_allowed(i, self.session.settings, self.name) == True and utils.find_item(i, self.session.db[self.name]) == None:
i = reduce.reduce_tweet(i) i = reduce.reduce_tweet(i)
i = self.session.check_quoted_status(i) i = self.session.check_quoted_status(i)
i = self.session.check_long_tweet(i) i = self.session.check_long_tweet(i)

8
src/sessions/twitter/session.py
View File

@ -51,7 +51,7 @@ class Session(base.baseSession):
if i.id < last_id: if i.id < last_id:
log.error("Ignoring an older tweet... Last id: {0}, tweet id: {1}".format(last_id, i.id)) log.error("Ignoring an older tweet... Last id: {0}, tweet id: {1}".format(last_id, i.id))
continue continue
if utils.find_item(i.id, self.db[name]) == None and utils.is_allowed(i, self.settings, name) == True: if utils.find_item(i, self.db[name]) == None and utils.is_allowed(i, self.settings, name) == True:
if i == False: continue if i == False: continue
reduced_object = reduce.reduce_tweet(i) reduced_object = reduce.reduce_tweet(i)
reduced_object = self.check_quoted_status(reduced_object) reduced_object = self.check_quoted_status(reduced_object)
@ -72,7 +72,7 @@ class Session(base.baseSession):
self.db[name] = [] self.db[name] = []
objects = self.db[name] objects = self.db[name]
for i in data: for i in data:
if utils.find_item(i.id, self.db[name]) == None: if utils.find_item(i, self.db[name]) == None:
if self.settings["general"]["reverse_timelines"] == False: objects.append(i) if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: objects.insert(0, i) else: objects.insert(0, i)
num = num+1 num = num+1
@ -94,12 +94,12 @@ class Session(base.baseSession):
for i in data: for i in data:
# Twitter returns sender_id as str, which must be converted to int in order to match to our user_id object. # Twitter returns sender_id as str, which must be converted to int in order to match to our user_id object.
if int(i.message_create["sender_id"]) == self.db["user_id"]: if int(i.message_create["sender_id"]) == self.db["user_id"]:
if "sent_direct_messages" in self.db and utils.find_item(i.id, self.db["sent_direct_messages"]) == None: if "sent_direct_messages" in self.db and utils.find_item(i, self.db["sent_direct_messages"]) == None:
if self.settings["general"]["reverse_timelines"] == False: sent_objects.append(i) if self.settings["general"]["reverse_timelines"] == False: sent_objects.append(i)
else: sent_objects.insert(0, i) else: sent_objects.insert(0, i)
sent = sent+1 sent = sent+1
else: else:
if utils.find_item(i.id, self.db["direct_messages"]) == None: if utils.find_item(i, self.db["direct_messages"]) == None:
if self.settings["general"]["reverse_timelines"] == False: objects.append(i) if self.settings["general"]["reverse_timelines"] == False: objects.append(i)
else: objects.insert(0, i) else: objects.insert(0, i)
incoming = incoming+1 incoming = incoming+1

10
src/sessions/twitter/utils.py
View File

@ -60,9 +60,13 @@ def find_urls (tweet, twitter_media=False):
urls.append(i) urls.append(i)
return urls return urls
def find_item(id, listItem): def find_item(item, listItems):
for i in range(0, len(listItem)): for i in range(0, len(listItems)):
if listItem[i].id == id: return i if listItems[i].id == item.id:
return i
# Check also retweets.
if hasattr(item, "retweeted_status") and item.retweeted_status.id == listItems[i].id:
return i
return None return None
def find_list(name, lists): def find_list(name, lists):