Goodbye, Pycurl. Now TWBlue uses requests and requests-based packages for all http connections. Remember to run git submodule update before submiting any commits

This commit is contained in:
Jose Manuel Delicado 2016-05-07 19:26:40 +02:00
parent f6fec67d52
commit 7ad5e6fa37
3 changed files with 71 additions and 96 deletions

View File

@ -35,7 +35,6 @@ Although most dependencies can be found in the windows-dependencies directory, w
If you want to build both x86 and x64 binaries, you can install python x86 to C:\python27 and python x64 to C:\python27x64, for example.
* [wxPython](http://www.wxpython.org) for Python 2.7, version 3.0.2.0
* [Python windows extensions (pywin32)](http://www.sourceforge.net/projects/pywin32/) for python 2.7, build 220
* [Pycurl](http://pycurl.sourceforge.net) 7.43.0 for Python 2.7: [downloads](https://pypi.python.org/pypi/pycurl/7.43.0)
* [PyEnchant,](http://pythonhosted.org/pyenchant/) version 1.6.6.
x64 version has been built by TWBlue developers, so you only will find it in windows-dependencies folder
@ -63,6 +62,7 @@ setuptools installs a script, called easy_install. You can find it in the python
* pypubsub
* configobj
* requests-oauthlib
* requests-toolbelt
* future
* pygeocoder
* suds
@ -73,7 +73,7 @@ setuptools installs a script, called easy_install. You can find it in the python
easy_install will automatically get the additional libraries that these packages need to work properly.
Run the following command to quickly install and upgrade all packages and their dependencies:
easy_install -Z --upgrade six configobj goslate markdown future suds requests oauthlib requests-oauthlib pypubsub pygeocoder arrow python-dateutil futures markdown microsofttranslator
easy_install -Z --upgrade six configobj goslate markdown future suds requests oauthlib requests-oauthlib requests-toolbelt pypubsub pygeocoder arrow==0.6 python-dateutil futures markdown microsofttranslator
#### Other dependencies

View File

@ -1,43 +1,40 @@
# -*- coding: utf-8 -*-
import pycurl
import sys
import threading
import time
import json
import logging
from utils import convert_bytes
from pubsub import pub
log = logging.getLogger("extra.AudioUploader.transfer")
class Transfer(object):
def __init__(self, obj=None, url=None, filename=None, follow_location=True, completed_callback=None, verbose=False, *args, **kwargs):
self.url = url
self.filename = filename
from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor
import requests
import os
class Upload(object):
def __init__(self, field=None, obj=None, url=None, filename=None, follow_location=True, completed_callback=None, verbose=False, *args, **kwargs):
super(Upload, self).__init__(*args, **kwargs)
self.url=url
self.filename=filename
log.debug("Uploading audio to %s, filename %s" % (url, filename))
self.curl = pycurl.Curl()
self.start_time = None
self.completed_callback = completed_callback
self.background_thread = None
self.transfer_rate = 0
self.curl.setopt(self.curl.XFERINFOFUNCTION, self.progress_callback)
self.curl.setopt(self.curl.URL, url)
self.curl.setopt(self.curl.NOPROGRESS, 0)
self.curl.setopt(self.curl.HTTP_VERSION, self.curl.CURL_HTTP_VERSION_1_0)
self.curl.setopt(self.curl.FOLLOWLOCATION, int(follow_location))
self.curl.setopt(self.curl.VERBOSE, int(verbose))
super(Transfer, self).__init__(*args, **kwargs)
self.obj = obj
self.m = MultipartEncoder(fields={field:(os.path.basename(self.filename), open(self.filename, 'rb'), "application/octet-stream")})
self.monitor = MultipartEncoderMonitor(self.m, self.progress_callback)
self.response=None
self.obj=obj
self.follow_location=follow_location
#the verbose parameter is deprecated and will be removed soon
def elapsed_time(self):
if not self.start_time:
return 0
return time.time() - self.start_time
def progress_callback(self, down_total, down_current, up_total, up_current):
def progress_callback(self, monitor):
progress = {}
progress["total"] = up_total
progress["current"] = up_current
progress["total"] = monitor.len
progress["current"] = monitor.bytes_read
if progress["current"] == 0:
progress["percent"] = 0
self.transfer_rate = 0
@ -54,8 +51,7 @@ class Transfer(object):
def perform_transfer(self):
log.debug("starting upload...")
self.start_time = time.time()
self.curl.perform()
self.curl.close()
self.response=requests.post(url=self.url, data=self.monitor, headers={"Content-Type":self.m.content_type}, allow_redirects=self.follow_location, stream=True)
log.debug("Upload finished.")
self.complete_transfer()
@ -66,28 +62,7 @@ class Transfer(object):
def complete_transfer(self):
if callable(self.completed_callback):
self.curl.close()
self.completed_callback(self.obj)
class Upload(Transfer):
def __init__(self, field=None, filename=None, *args, **kwargs):
super(Upload, self).__init__(filename=filename, *args, **kwargs)
self.response = dict()
self.curl.setopt(self.curl.POST, 1)
if isinstance(filename, unicode):
local_filename = filename.encode(sys.getfilesystemencoding())
else:
local_filename = filename
self.curl.setopt(self.curl.HTTPPOST, [(field, (self.curl.FORM_FILE, local_filename, self.curl.FORM_FILENAME, filename.encode("utf-8")))])
self.curl.setopt(self.curl.HEADERFUNCTION, self.header_callback)
self.curl.setopt(self.curl.WRITEFUNCTION, self.body_callback)
def header_callback(self, content):
self.response['header'] = content
def body_callback(self, content):
self.response['body'] = content
def get_url(self):
return json.loads(self.response['body'])['url']
return self.response.json()['url']

@ -1 +1 @@
Subproject commit 3ca08aac722b5554b4070cde99cbaa6986e8998c
Subproject commit 1b69006f66c2e421b3f6f6b91fd898755f407488