requests_toolbelt.MultipartEncoder

Here are the examples of the python api requests_toolbelt.MultipartEncoder taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

24 Examples 7

Example 1

Project: acd_cli
Source File: content.py
View license
    def clear_file(self, node_id: str) -> dict:
        """Clears a file's content by overwriting it with an empty BytesIO.

        :param node_id: valid file node ID"""

        m = MultipartEncoder(fields={('content', (' ', io.BytesIO(), _get_mimetype()))})

        r = self.BOReq.put(self.content_url + 'nodes/' + node_id + '/content', params={},
                           data=m, stream=True, headers={'Content-Type': m.content_type})

        if r.status_code not in OK_CODES:
            raise RequestError(r.status_code, r.text)

        return r.json()

Example 2

Project: acd_cli
Source File: content.py
View license
    def overwrite_file(self, node_id: str, file_name: str,
                       read_callbacks: list = None, deduplication=False) -> dict:
        params = {} if deduplication else {'suppress': 'deduplication'}

        basename = os.path.basename(file_name)
        mime_type = _get_mimetype(basename)
        f = _tee_open(file_name, callbacks=read_callbacks)

        # basename is ignored
        m = MultipartEncoder(fields={('content', (quote_plus(basename), f, mime_type))})

        r = self.BOReq.put(self.content_url + 'nodes/' + node_id + '/content', params=params,
                           data=m, stream=True, headers={'Content-Type': m.content_type})

        if r.status_code not in OK_CODES:
            raise RequestError(r.status_code, r.text)

        return r.json()

Example 3

Project: clam
Source File: client.py
View license
    def addinputfile(self, project, inputtemplate, sourcefile, **kwargs):
        """Add/upload an input file to the CLAM service. Supports proper file upload streaming.

        project - the ID of the project you want to add the file to.
        inputtemplate - The input template you want to use to add this file (InputTemplate instance)
        sourcefile - The file you want to add: string containing a filename (or instance of ``file``)

        Keyword arguments (optional but recommended!):
            * ``filename`` - the filename on the server (will be same as sourcefile if not specified)
            * ``metadata`` - A metadata object.
            * ``metafile`` - A metadata file (filename)

        Any other keyword arguments will be passed as metadata and matched with the input template's parameters.

        Example::

            client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file")

        With metadata, assuming such metadata parameters are defined::

            client.addinputfile("myproject", "someinputtemplate", "/path/to/local/file", parameter1="blah", parameterX=3.5)

        """
        if isinstance( inputtemplate, str) or (sys.version < '3' and isinstance( inputtemplate, unicode)): #pylint: disable=undefined-variable
            data = self.get(project) #causes an extra query to server
            inputtemplate = data.inputtemplate(inputtemplate)
        elif not isinstance(inputtemplate, clam.common.data.InputTemplate):
            raise Exception("inputtemplate must be instance of InputTemplate. Get from CLAMData.inputtemplate(id)")

        if not isinstance(sourcefile, IOBase):
            sourcefile = open(sourcefile,'rb')
            if 'filename' in kwargs:
                filename = self.getinputfilename(inputtemplate, kwargs['filename'])
            else:
                filename = self.getinputfilename(inputtemplate, os.path.basename(sourcefile.name) )

        data = {"file": (filename,sourcefile,inputtemplate.formatclass.mimetype), 'inputtemplate': inputtemplate.id}
        for key, value in kwargs.items():
            if key == 'filename':
                pass #nothing to do
            elif key == 'metadata':
                assert isinstance(value, clam.common.data.CLAMMetaData)
                data['metadata'] =  value.xml()
            elif key == 'metafile':
                data['metafile'] = open(value,'rb')
            else:
                data[key] = value


        requestparams = self.initrequest(data)
        if 'auth'in requestparams:
            #TODO: streaming support doesn't work with authentication unfortunately, disabling streaming for now:
            del data['file']
            requestparams['data'] = data
            requestparams['files'] = [('file', (filename,sourcefile, inputtemplate.formatclass.mimetype))]
            if 'metafile' in kwargs:
                del data['metafile']
                requestparams['files'].append(('metafile',('.'+ filename + '.METADATA', open(kwargs['metafile'],'rb'), 'text/xml')))
        else:
            #streaming support
            encodeddata = MultipartEncoder(fields=requestparams['data']) #from requests-toolbelt, necessary for streaming support
            requestparams['data'] = encodeddata
            requestparams['headers']['Content-Type'] = encodeddata.content_type
        r = requests.post(self.url + project + '/input/' + filename,**requestparams)
        sourcefile.close()

        if r.status_code == 400:
            raise clam.common.data.BadRequest()
        elif r.status_code == 401:
            raise clam.common.data.AuthRequired()
        elif r.status_code == 403:
            if r.text[0] == '<':
                #XML response
                return self._parseupload(r.text)
            else:
                raise clam.common.data.PermissionDenied(r.text)
        elif r.status_code == 404:
            raise clam.common.data.NotFound(r.text)
        elif r.status_code == 500:
            raise clam.common.data.ServerError(r.text)
        elif r.status_code == 405:
            raise clam.common.data.ServerError("Server returned 405: Method not allowed for POST on " + self.url + project + '/input/' + filename)
        elif r.status_code == 408:
            raise clam.common.data.TimeOut()
        elif not (r.status_code >= 200 and r.status_code <= 299):
            raise Exception("An error occured, return code " + str(r.status_code))

        return self._parseupload(r.text)

Example 4

Project: repositorytools
Source File: repository.py
View license
    def _upload_artifact(self, local_artifact, path_prefix, repo_id, hostname_for_download=None, use_direct_put=False):

        filename = os.path.basename(local_artifact.local_path)
        logger.info('-> Uploading %s', filename)
        logger.debug('local artifact: %s', local_artifact)

        # rgavf stands for repo-group-local_artifact-version-filename
        gavf = '{group}/{name}/{ver}/{filename}'.format(group=local_artifact.group.replace('.', '/'),
                                                        name=local_artifact.artifact, ver=local_artifact.version,
                                                        filename=filename)
        rgavf = '{repo_id}/{gavf}'.format(repo_id=repo_id, gavf=gavf)

        with open(local_artifact.local_path, 'rb') as f:
            if not use_direct_put:
                data = {
                    'g':local_artifact.group,
                    'a':local_artifact.artifact,
                    'v':local_artifact.version,
                    'r':repo_id,
                    'e': local_artifact.extension,
                    'p': local_artifact.extension,
                    'hasPom': 'false'
                }


                data_list = data.items()
                data_list.append( ('file', (filename, f, 'text/plain') ))
                m_for_logging = MultipartEncoder(fields=data_list)
                logger.debug('payload: %s', m_for_logging.to_string())

                f.seek(0)
                m = MultipartEncoder(fields=data_list)
                headers = {'Content-Type': m.content_type}

                self._send('service/local/artifact/maven/content', method='POST', data=m, headers=headers)

                result = RemoteArtifact(group=local_artifact.group, artifact=local_artifact.artifact,
                                      version=local_artifact.version, classifier=local_artifact.classifier,
                                      extension=local_artifact.extension, repo_id=repo_id)
                self.resolve_artifact(result)
                return result

            else:
                headers = {'Content-Type': 'application/x-rpm'}
                remote_path = '{path_prefix}/{rgavf}'.format(path_prefix=path_prefix, rgavf=rgavf)
                self._send(remote_path, method='PUT', headers=headers, data=f)

                # if not specified, use repository url
                hostname_for_download = hostname_for_download or self._repository_url
                url = '{hostname}/content/repositories/{rgavf}'.format(hostname=hostname_for_download, rgavf=rgavf)

                # get classifier and extension from nexus
                path = 'service/local/repositories/{repo_id}/content/{gavf}?describe=maven2'.format(repo_id=repo_id, gavf=gavf)
                maven_metadata = self._send_json(path)['data']

                return RemoteArtifact(group=maven_metadata['groupId'], artifact=maven_metadata['artifactId'],
                                      version=maven_metadata['version'], classifier=maven_metadata.get('classifier', ''),
                                      extension=maven_metadata.get('extension', ''), url=url, repo_id=repo_id)

Example 5

Project: raco
Source File: connection.py
View license
    def upload_file(self, relation_key, schema, data, overwrite=None,
                    delimiter=None, binary=None, is_little_endian=None):
        """Upload a file in a streaming manner to Myria.

        Args:
            relation_key: relation to be created.
            schema: schema of the relation.
            data: the bytes to be uploaded.
            overwrite: optional boolean indicating that an existing relation
                should be overwritten. Myria default is False.
            delimiter: optional character which delimits a CSV file. Only valid
                if binary is False. Myria default is ','.
            binary: optional boolean indicating that the data is encoded as
                a packed binary. Myria default is False.
            is_little_endian: optional boolean indicating that the binary data
                is in little-Endian. Myria default is False.
        """

        from requests_toolbelt import MultipartEncoder

        fields = [('relationKey', relation_key), ('schema', schema),
                  ('overwrite', overwrite), ('delimiter', delimiter),
                  ('binary', binary), ('isLittleEndian', is_little_endian)]
        fields = [(name, (name, json.dumps(value), 'application/json'))
                  for (name, value) in fields]
        # data must be last
        if binary:
            data_type = 'application/octet-stream'
        else:
            data_type = 'text/plain'
        fields.append(('data', ('data', data, data_type)))

        m = MultipartEncoder(fields=fields)
        r = self._session.post(self._url_start + '/dataset', data=m,
                               headers={'Content-Type': m.content_type})
        if r.status_code not in (200, 201):
            raise MyriaError('Error %d: %s'
                             % (r.status_code, r.text))
        return r.json()

Example 6

Project: acd_cli
Source File: content.py
View license
    def create_file(self, file_name: str, parent: str = None) -> dict:
        params = {'suppress': 'deduplication'}

        basename = os.path.basename(file_name)
        metadata = {'kind': 'FILE', 'name': basename}
        if parent:
            metadata['parents'] = [parent]
        mime_type = _get_mimetype(basename)
        f = io.BytesIO()

        # basename is ignored
        m = MultipartEncoder(fields=OrderedDict([('metadata', json.dumps(metadata)),
                                                 ('content', (quote_plus(basename), f, mime_type))])
                             )

        ok_codes = [http.CREATED]
        r = self.BOReq.post(self.content_url + 'nodes', params=params, data=m,
                            acc_codes=ok_codes, headers={'Content-Type': m.content_type})

        if r.status_code not in ok_codes:
            raise RequestError(r.status_code, r.text)
        return r.json()

Example 7

Project: acd_cli
Source File: content.py
View license
    def upload_file(self, file_name: str, parent: str = None,
                    read_callbacks=None, deduplication=False) -> dict:
        params = {'suppress': 'deduplication'}
        if deduplication and os.path.getsize(file_name) > 0:
            params = {}

        basename = os.path.basename(file_name)
        metadata = {'kind': 'FILE', 'name': basename}
        if parent:
            metadata['parents'] = [parent]
        mime_type = _get_mimetype(basename)
        f = _tee_open(file_name, callbacks=read_callbacks)

        # basename is ignored
        m = MultipartEncoder(fields=OrderedDict([('metadata', json.dumps(metadata)),
                                                 (
                                                     'content',
                                                     (quote_plus(basename), f, mime_type))]))

        ok_codes = [http.CREATED]
        r = self.BOReq.post(self.content_url + 'nodes', params=params, data=m,
                            acc_codes=ok_codes, stream=True,
                            headers={'Content-Type': m.content_type})

        if r.status_code not in ok_codes:
            raise RequestError(r.status_code, r.text)
        return r.json()

Example 8

Project: drivesink
Source File: drivesink.py
View license
    def upload_child_file(self, name, local_path, existing_node=None):
        logging.info("Uploading %s to %s", local_path, self.node["name"])
        mime_type = _get_mimetype(name)
        m = requests_toolbelt.MultipartEncoder([
            ("metadata", json.dumps({
                "name": name,
                "kind": "FILE",
                "parents": [self.node["id"]],
            })),
            ("content", (name, open(local_path, "rb"), mime_type))])
        if existing_node:
            """
            # TODO: this is under-documented and currently 500s on Amazon's side
            node = CloudNode(DriveSink.instance().request_content(
                "%%snodes/%s/content" % existing_node.node["id"],
                method="put", data=m, headers={"Content-Type": m.content_type}))
            """
            old_info = DriveSink.instance().request_metadata(
                "%%s/trash/%s" % existing_node.node["id"], method="put")
        node = CloudNode(DriveSink.instance().request_content(
            "%snodes", method="post", data=m,
            headers={"Content-Type": m.content_type}))
        self._children[name] = node

Example 9

View license
    def imageUploadOnly(self, imageFile):
        if re.match(r'[a-zA-z]+://[^\s]*', imageFile):
            # Upload Image from URL
            response = requests.Session().get(
                url=imageFile,
                headers={'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0'},
                timeout=120,
                stream=True,
                verify=False
            )
            fileContent = response.content
            mimeType = response.headers['content-type']
            fileName = '%s.%s' % (uuid4().hex, re.sub(r'image/', '', mimeType))
            formData = MultipartEncoder(fields={'image': (fileName, fileContent, mimeType)})
        else:
            # Upload Image from Local File
            fileContent = codecs.open(imageFile, 'rb')
            mimeType = 'image/%s' % (imghdr.what(fileContent))
            fileName = '%s.%s' % (uuid4().hex, re.sub(r'image/', '', mimeType))
            formData = MultipartEncoder(fields={'image': (fileName, fileContent, mimeType)})
        url = 'https://cloud.smartisan.com/apps/note/index.php?r=image/upload'
        response = self.session.post(
            url=url,
            headers={'Content-Type': formData.content_type},
            data=self.genChunks(formData),
            timeout=120,
            stream=True
        )
        self.checkResponse(response)
        content = json.loads(response.content)
        respData = content['data']
        return respData

Example 10

Project: ciscosparkapi
Source File: messages.py
View license
    def create(self, roomId=None, toPersonId=None, toPersonEmail=None,
               text=None, markdown=None, files=None):
        """Posts a message to a room.

        Posts a message, and optionally, a media content attachment, to a room.

        You must specify either a roomId, toPersonId or toPersonEmail when
        posting a message, and you must supply some message content (text,
        markdown, files).

        Args:
            roomId(string_types): The room ID.
            toPersonId(string_types): The ID of the recipient when sending a
                private 1:1 message.
            toPersonEmail(string_types): The email address of the recipient
                when sending a private 1:1 message.
            text(string_types): The message, in plain text. If markdown is
                specified this parameter may be optionally used to provide
                alternate text forUI clients that do not support rich text.
            markdown(string_types): The message, in markdown format.
            files(list): A list containing local paths or URL references for
                the message attachment(s).  The files attribute currently only
                takes a list containing one (1) filename or URL as an input.
                This is a Spark API limitation that may be lifted at a later
                date.

        Returns:
            Message: With the details of the created message.

        Raises:
            AssertionError: If the parameter types are incorrect.
            ciscosparkapiException: If the required arguments are not
                specified.
            SparkApiError: If the Cisco Spark cloud returns an error.

        """
        # Process args
        assert roomId is None or isinstance(roomId, string_types)
        assert toPersonId is None or isinstance(toPersonId, string_types)
        assert toPersonEmail is None or isinstance(toPersonEmail, string_types)
        assert text is None or isinstance(text, string_types)
        assert markdown is None or isinstance(markdown, string_types)
        assert files is None or isinstance(files, list)
        post_data = {}
        # Where is message to be posted?
        if roomId:
            post_data['roomId'] = roomId
        elif toPersonId:
            post_data['toPersonId'] = toPersonId
        elif toPersonEmail:
            post_data['toPersonEmail'] = toPersonEmail
        else:
            error_message = "You must specify a roomId, toPersonId, or " \
                            "toPersonEmail to which you want to post a new " \
                            "message."
            raise ciscosparkapiException(error_message)
        # Ensure some message 'content' is provided.
        if not text and not markdown and not files:
            error_message = "You must supply some message content (text, " \
                            "markdown, files) when posting a message."
            raise ciscosparkapiException(error_message)
        # Process the content.
        if text:
            post_data['text'] = text
        if markdown:
            post_data['markdown'] = markdown
        upload_local_file = False
        if files:
            if len(files) > 1:
                error_message = "The files attribute currently only takes a " \
                                "list containing one (1) filename or URL as " \
                                "an input.  This is a Spark API limitation " \
                                "that may be lifted at a later date."
                raise ciscosparkapiException(error_message)
            if is_web_url(files[0]):
                post_data['files'] = files
            elif is_local_file(files[0]):
                upload_local_file = True
                post_data['files'] = open_local_file(files[0])
            else:
                error_message = "The provided files argument does not " \
                                "contain a valid URL or local file path."
                raise ciscosparkapiException(error_message)
        # API request
        if upload_local_file:
            try:
                multipart_data = MultipartEncoder(post_data)
                headers = {'Content-type': multipart_data.content_type}
                json_obj = self.session.post('messages',
                                             data=multipart_data,
                                             headers=headers)
            finally:
                post_data['files'].file_object.close()
        else:
            json_obj = self.session.post('messages', json=post_data)
        # Return a Message object created from the response JSON data
        return Message(json_obj)

Example 11

View license
    def upload(self, file_path, progress=None):
        if not os.path.exists(file_path):
            raise IOError("[Errno 2] No such file or directory: '%s'" % file_path)

        if sys.version[0] == 2 and isinstance(file_path, unicode):
            file_path = file_path.encode('utf8')

        file_path = os.path.abspath(os.path.expanduser(file_path))

        result = self.get('/file/upload')

        m = MultipartEncoder(fields={'file': (os.path.basename(file_path), open(file_path, 'rb'))})

        headers = {
            'User-Agent': 'Dailymotion-Python/%s (Python %s)' % (__version__, __python_version__),
            'Content-Type': m.content_type
        }

        r = requests.post(result['upload_url'], data=m, headers=headers, timeout=self.timeout)

        try:
            response = json.loads(r.text)
        except ValueError as e:
            raise DailymotionUploadInvalidResponse('Invalid API server response.\n%s' % response)
        if 'error' in response:
            raise DailymotionUploadError(response['error'])

        return response['url']

Example 12

Project: eulfedora
Source File: api.py
View license
    def upload(self, data, callback=None, content_type=None,
               size=None):
        '''
        Upload a multi-part file for content to ingest.  Returns a
        temporary upload id that can be used as a datstream location.

        :param data: content string, file-like object, or iterable with
            content to be uploaded
        :param callback: optional callback method to monitor the upload;
            see :mod:`requests-toolbelt` documentation for more
            details: https://toolbelt.readthedocs.org/en/latest/user.html#uploading-data
        :param content_type: optional content type of the data
        :param size: optional size of the data; required when using an
            iterable for the data

        :returns: upload id on success
        '''
        url = 'upload'
        # fedora only expects content uploaded as multipart file;
        # make string content into a file-like object so requests.post
        # sends it the way Fedora expects.
        # NOTE: checking for both python 2.x next method and
        # python 3.x __next__ to test if data is iteraable
        if not hasattr(data, 'read') and \
          not (hasattr(data, '__next__') or hasattr(data, 'next')):
            data = six.BytesIO(force_bytes(data))

        # if data is an iterable, wrap in a readable iterator that
        # requests-toolbelt can read data from
        elif not hasattr(data, 'read') and \
          (hasattr(data, '__next__') or hasattr(data, 'next')):
            if size is None:
                raise Exception('Cannot upload iterable with unknown size')
            data = ReadableIterator(data, size)

        # use requests-toolbelt multipart encoder to avoid reading
        # the full content of large files into memory
        menc = MultipartEncoder(fields={'file': ('file', data, content_type)})

        if callback is not None:
            menc = MultipartEncoderMonitor(menc, callback)

        headers = {'Content-Type': menc.content_type}

        if size:
            # latest version of requests requires str or bytes, not int
            if not isinstance(size, six.string_types):
                size = str(size)
            headers['Content-Length'] = size

        try:
            response = self.post(url, data=menc, headers=headers)
        except OverflowError:
            # Python __len__ uses integer so it is limited to system maxint,
            # and requests and requests-toolbelt use len() throughout.
            # This results in an overflow error when trying to upload a file
            # larger than system maxint (2GB on 32-bit OSes).
            # See http://bugs.python.org/issue12159
            msg = 'upload content larger than system maxint (32-bit OS limitation)'
            logger.error('OverflowError: %s', msg)
            raise OverflowError(msg)

        if response.status_code == requests.codes.accepted:
            return response.text.strip()

Example 13

Project: jottalib
Source File: JFS.py
View license
    def post(self, url, content='', files=None, params=None, extra_headers={}, upload_callback=None):
        'HTTP Post files[] or content (unicode string) to url'
        if not url.startswith('http'):
            # relative url
            url = self.rootpath + url

        log.debug('posting content (len %s) to url %s', len(content) if content is not None else '?', url)
        headers = self.session.headers.copy()
        headers.update(**extra_headers)

        if not files is None:
            m = requests_toolbelt.MultipartEncoder(fields=files)
            if upload_callback is not None:
                m_len = m.len # compute value for callback closure
                def callback(monitor):
                    upload_callback(monitor, m_len)

                m = requests_toolbelt.MultipartEncoderMonitor(m, callback)
            headers['content-type'] = m.content_type
        else:
            m = content
        url = self.escapeUrl(url)
        r = self.session.post(url, data=m, params=params, headers=headers)
        if not r.ok:
            log.warning('HTTP POST failed: %s', r.text)
            raise JFSError(r.reason)
        return self.getObject(r) # return a JFS* class

Example 14

Project: jottalib
Source File: JFS.py
View license
    def up(self, path, fileobject, upload_callback=None, resume_offset=None):
        "Upload a fileobject to path, HTTP POST-ing to up.jottacloud.com, using the JottaCloud API"
        """

        *** WHAT DID I DO?: created file
        ***

        POST https://up.jottacloud.com/jfs/**USERNAME**/Jotta/Sync/testFolder/testFile.txt?cphash=d41d8cd98f00b204e9800998ecf8427e HTTP/1.1
        User-Agent: Desktop_Jottacloud 3.0.22.203 Windows_8 6.2.9200 x86_64
        Authorization: Basic ******************
        X-JottaAPIVersion: 2.2
        X-Jfs-DeviceName: **CENSORED**
        JCreated: 2014-10-26T12:33:09Z+00:00
        JModified: 2014-10-26T12:33:09Z+00:00
        JMd5: d41d8cd98f00b204e9800998ecf8427e
        JSize: 0
        jx_csid: dOq1NCRer6uxuR/bFxihasj4QzBU3Tn7S2jVF1CE71YW1fGhxPFYYsw2T0XYjnJBtxKQzhWixmg+u5kp8bJtvMpIFHbhSDmPPSk+PVBf2UdFhXxli4YEII9a97eO4XBfn5QWAV1LJ2Z9l59jmnLkJQgfOyexkuQbxHdSLgQPXu8=
        jx_lisence: M1v3p31oQf2OXvyAn2GvfS2I2oiMXrw+cofuMVHHI/2K+wlxhj22VkON6fN6fJMsGNcMzvcFYfmKPgL0Yf8TCO5A/6ULk6N8LctY3+fPegx+Jgbyc4hh0IXwnOdqa+UZ6Lg1ub4VXr5XnX3P3IxeVDg0VbcJnzv4TbFA+oMXmfM=
        Content-Type: application/octet-stream
        Content-Length: 0
        Connection: Keep-Alive
        Accept-Encoding: gzip
        Accept-Language: nb-NO,en,*
        Host: up.jottacloud.com
        """
        url = path.replace('www.jottacloud.com', 'up.jottacloud.com')
        # Calculate file length
        fileobject.seek(0,2)
        contentlen = fileobject.tell()

        # Rewind read head to correct offset
        # If we're resuming an incomplete upload, continue from that offset
        try:
            fileobject.seek(resume_offset)
        except TypeError as e:
            if resume_offset is None:
                fileobject.seek(0)
        except IOError as e:
            log.exception(e)
            log.warning('Could not seek to file offset %r, re-starting upload of %r from 0',
                        resume_offset,
                        url)
            fileobject.seek(0)


        # Calculate file md5 hash
        md5hash = calculate_md5(fileobject)

        log.debug('posting content (len %s, hash %s) to url %r', contentlen, md5hash, url)
        try:
            mtime = os.path.getmtime(fileobject.name)
            timestamp = datetime.datetime.fromtimestamp(mtime).isoformat()
        except Exception as e:
            if hasattr(fileobject, 'name'):
                log.exception('Problems getting mtime from fileobjet: %r', e)
            timestamp = datetime.datetime.now().isoformat()
        params = {'cphash': md5hash}
        m = requests_toolbelt.MultipartEncoder({
             'md5': ('', md5hash),
             'modified': ('', timestamp),
             'created': ('', timestamp),
             'file': (os.path.basename(url), fileobject, 'application/octet-stream'),
        })
        headers = {'JMd5':md5hash,
                   'JCreated': timestamp,
                   'JModified': timestamp,
                   'X-Jfs-DeviceName': 'Jotta',
                   'JSize': str(contentlen), # headers have to be strings or bytes , cf #122
                   'jx_csid': '',
                   'jx_lisence': '',
                   'content-type': m.content_type,
                   }
        fileobject.seek(0) # rewind read index for requests.post
        files = {'md5': ('', md5hash),
                 'modified': ('', timestamp),
                 'created': ('', timestamp),
                 'file': (os.path.basename(url), fileobject, 'application/octet-stream')}
        return self.post(url, None, files=files, params=params, extra_headers=headers, upload_callback=upload_callback)

Example 15

Project: Photini
Source File: facebook.py
View license
    def do_upload(self, fileobj, image_type, image, params):
        fields = {
            'photo'   : ('source', fileobj),
            'no_story': str(params['no_story']),
            }
        title = image.metadata.title
        description = image.metadata.description
        if title and description:
            caption = title.value + '\n\n' + description.value
        elif title:
            caption = title.value
        elif description:
            caption = description.value
        else:
            caption = ''
        if caption:
            fields['caption'] = caption
        date_taken = image.metadata.date_taken
        if date_taken:
            fields['backdated_time'] = date_taken.datetime.isoformat()
            if date_taken.precision <= 5:
                fields['backdated_time_granularity'] = (
                    'year', 'month', 'day', 'hour', 'min')[date_taken.precision - 1]
        latlong = image.metadata.latlong
        if latlong and params['geo_tag']:
            nearest = None, 1.0e12
            for place in self.get_cities(latlong):
                dist = self.distance(latlong, place['location'])
                if dist < nearest[1]:
                    nearest = place, dist
            nearest = nearest[0]
            if nearest:
                fields['place'] = nearest['id']
        data = MultipartEncoder(fields=fields)
        headers = {'Content-Type' : data.content_type}
        url = 'https://graph.facebook.com/v2.6/' + params['album_id'] + '/photos'
        try:
            self.post(url, data=data, headers=headers)
        except Exception as ex:
            return str(ex)
        return ''

Example 16

Project: Photini
Source File: facebook.py
View license
    def do_upload(self, fileobj, image_type, image, params):
        fields = {
            'photo'   : ('source', fileobj),
            'no_story': str(params['no_story']),
            }
        title = image.metadata.title
        description = image.metadata.description
        if title and description:
            caption = title.value + '\n\n' + description.value
        elif title:
            caption = title.value
        elif description:
            caption = description.value
        else:
            caption = ''
        if caption:
            fields['caption'] = caption
        date_taken = image.metadata.date_taken
        if date_taken:
            fields['backdated_time'] = date_taken.datetime.isoformat()
            if date_taken.precision <= 5:
                fields['backdated_time_granularity'] = (
                    'year', 'month', 'day', 'hour', 'min')[date_taken.precision - 1]
        latlong = image.metadata.latlong
        if latlong and params['geo_tag']:
            nearest = None, 1.0e12
            for place in self.get_cities(latlong):
                dist = self.distance(latlong, place['location'])
                if dist < nearest[1]:
                    nearest = place, dist
            nearest = nearest[0]
            if nearest:
                fields['place'] = nearest['id']
        data = MultipartEncoder(fields=fields)
        headers = {'Content-Type' : data.content_type}
        url = 'https://graph.facebook.com/v2.6/' + params['album_id'] + '/photos'
        try:
            self.post(url, data=data, headers=headers)
        except Exception as ex:
            return str(ex)
        return ''

Example 17

Project: karl
Source File: client.py
View license
    def upload(self, name, f):
        # some attachment file names may have hidden new lines and tabs, ugh
        name = name.replace('\n',' ')
        name = name.replace('\t',' ')
        data = MultipartEncoder([
            ('attributes', json.dumps(
                {'name': name, 'parent': {'id': self.id}})),
            ('file', (name, f)),
        ])

        client = self.client
        response = client.api_call(
            'post',
            client.upload_url,
            data=data,
            headers={'Content-Type': data.content_type})

        self.contents()[name] = uploaded = BoxFile(
            client, response.json()['entries'][0]['id'])
        return uploaded

Example 18

Project: karl
Source File: client.py
View license
    def upload(self, name, f):
        # some attachment file names may have hidden new lines and tabs, ugh
        name = name.replace('\n',' ')
        name = name.replace('\t',' ')
        data = MultipartEncoder([
            ('attributes', json.dumps(
                {'name': name, 'parent': {'id': self.id}})),
            ('file', (name, f)),
        ])

        client = self.client
        response = client.api_call(
            'post',
            client.upload_url,
            data=data,
            headers={'Content-Type': data.content_type})

        self.contents()[name] = uploaded = BoxFile(
            client, response.json()['entries'][0]['id'])
        return uploaded

Example 19

View license
    def http_request(self, url, request_type, status_codes=(), headers={},
                     stream=False, data="", params=None,
                     severe_status_codes=(),
                     use_access_token=False, action_string="OneDrive HTTP",
                     max_tries=6,
                     use_multipart_encoder=False,
                     multipart_encoder_fields=None,
                     multipart_encoder_content=None,
                     multipart_hash_file=None):
        logger = logging.getLogger("multidrive")

        if use_access_token is True:
            headers['Authorization'] = "Bearer " + self.get_access_token()

        if use_multipart_encoder is True:
            cur_multipart_file = open(multipart_encoder_content[1], 'rb')
            multipart_hash_file.set_file(cur_multipart_file)
            cur_multipart_content = ('content', (multipart_encoder_content[0],
                                                 multipart_hash_file,
                                                 multipart_encoder_content[2]))
            multipart_encoder_fields.append(cur_multipart_content)
            data = MultipartEncoder(fields=multipart_encoder_fields)
            headers["Content-Type"] = data.content_type

        try:
            if request_type == RequestType.GET:
                response = requests.get(url, headers=headers, params=params,
                                        stream=stream)
            elif request_type == RequestType.PUT:
                response = requests.put(url, headers=headers, data=data,
                                        params=params)
            elif request_type == RequestType.POST:
                response = requests.post(url, headers=headers, data=data)
            if use_multipart_encoder is True:
                logger.info("Current hash: "+multipart_hash_file.get_md5())
        except requests.exceptions.ConnectionError as err:
            logger.warning("ConnectionError: {}".format(err))
            response = None
        finally:
            if use_multipart_encoder is True:
                cur_multipart_file.close()

        tries = 0
        while response is None or (response.status_code not in status_codes
                                   and tries < max_tries):
            tries += 1
            logger.warning("Retry " + str(tries))
            sleep_length = float(1 << tries) / 2

            if response is not None:
                logger.warning("{}: Connection failed Code: {}"
                               .format(action_string,
                                       str(response.status_code)))
                logger.warning("Error: {}".format(response.text))
                logger.warning("Headers: {}".format(str(response.headers)))

                if 'Retry-After' in response.headers:
                    logger.warning("Server requested wait of {} seconds".
                                   format(response.headers['Retry-After']))
                    sleep_length = int(response.headers['Retry-After'])
                elif response.status_code in severe_status_codes:
                    logger.warning("Server Error, increasing wait time")
                    sleep_length *= 20

            logger.warning("Waiting {} second(s) for next attempt"
                           .format(str(sleep_length)))

            time.sleep(sleep_length)
            # Force refresh of token once in case it's expired
            if (tries == 1):
                self.refresh_access_token()

            if use_access_token is True:
                headers['Authorization'] = "Bearer " + self.get_access_token()

            if use_multipart_encoder is True:
                cur_multipart_file = open(multipart_encoder_content[1], 'rb')
                multipart_hash_file.set_file(cur_multipart_file)
                cur_multipart_content = ('content',
                                         (multipart_encoder_content[0],
                                          multipart_hash_file,
                                          multipart_encoder_content[2]))
                multipart_encoder_fields.append(cur_multipart_content)
                data = MultipartEncoder(fields=multipart_encoder_fields)

            try:
                if request_type == RequestType.GET:
                    response = requests.get(url, headers=headers,
                                            params=params, stream=stream)
                elif request_type == RequestType.PUT:
                    response = requests.put(url, headers=headers, data=data,
                                            params=params)
                elif request_type == RequestType.POST:
                    response = requests.post(url, headers=headers, data=data)
            except requests.exceptions.ConnectionError as err:
                logger.warning("ConnectionError: {}".format(err))
                response = None
            finally:
                if use_multipart_encoder is True:
                    cur_multipart_file.close()

        if response is None:
            raise RemoteConnectionError("{}: Unable to complete request."
                                        .format(action_string))
        if response.status_code not in status_codes:
            logger.warning("{}: Connection failed Code: {}"
                           .format(action_string, str(response.status_code)))
            logger.warning("Error: {}".format(response.text))
            logger.warning("Headers: {}".format(str(response.headers)))
            logger.warning("Retry " + str(tries))
            raise RemoteConnectionError("{}: Unable to complete request."
                                        .format(action_string))

        return response

Example 20

View license
    def http_request(self, url, request_type, status_codes=(), headers={},
                     stream=False, data="", params=None,
                     severe_status_codes=(),
                     use_access_token=False, action_string="OneDrive HTTP",
                     max_tries=6,
                     use_multipart_encoder=False,
                     multipart_encoder_fields=None,
                     multipart_encoder_content=None,
                     multipart_hash_file=None):
        logger = logging.getLogger("multidrive")

        if use_access_token is True:
            headers['Authorization'] = "Bearer " + self.get_access_token()

        if use_multipart_encoder is True:
            cur_multipart_file = open(multipart_encoder_content[1], 'rb')
            multipart_hash_file.set_file(cur_multipart_file)
            cur_multipart_content = ('content', (multipart_encoder_content[0],
                                                 multipart_hash_file,
                                                 multipart_encoder_content[2]))
            multipart_encoder_fields.append(cur_multipart_content)
            data = MultipartEncoder(fields=multipart_encoder_fields)
            headers["Content-Type"] = data.content_type

        try:
            if request_type == RequestType.GET:
                response = requests.get(url, headers=headers, params=params,
                                        stream=stream)
            elif request_type == RequestType.PUT:
                response = requests.put(url, headers=headers, data=data,
                                        params=params)
            elif request_type == RequestType.POST:
                response = requests.post(url, headers=headers, data=data)
            if use_multipart_encoder is True:
                logger.info("Current hash: "+multipart_hash_file.get_md5())
        except requests.exceptions.ConnectionError as err:
            logger.warning("ConnectionError: {}".format(err))
            response = None
        finally:
            if use_multipart_encoder is True:
                cur_multipart_file.close()

        tries = 0
        while response is None or (response.status_code not in status_codes
                                   and tries < max_tries):
            tries += 1
            logger.warning("Retry " + str(tries))
            sleep_length = float(1 << tries) / 2

            if response is not None:
                logger.warning("{}: Connection failed Code: {}"
                               .format(action_string,
                                       str(response.status_code)))
                logger.warning("Error: {}".format(response.text))
                logger.warning("Headers: {}".format(str(response.headers)))

                if 'Retry-After' in response.headers:
                    logger.warning("Server requested wait of {} seconds".
                                   format(response.headers['Retry-After']))
                    sleep_length = int(response.headers['Retry-After'])
                elif response.status_code in severe_status_codes:
                    logger.warning("Server Error, increasing wait time")
                    sleep_length *= 20

            logger.warning("Waiting {} second(s) for next attempt"
                           .format(str(sleep_length)))

            time.sleep(sleep_length)
            # Force refresh of token once in case it's expired
            if (tries == 1):
                self.refresh_access_token()

            if use_access_token is True:
                headers['Authorization'] = "Bearer " + self.get_access_token()

            if use_multipart_encoder is True:
                cur_multipart_file = open(multipart_encoder_content[1], 'rb')
                multipart_hash_file.set_file(cur_multipart_file)
                cur_multipart_content = ('content',
                                         (multipart_encoder_content[0],
                                          multipart_hash_file,
                                          multipart_encoder_content[2]))
                multipart_encoder_fields.append(cur_multipart_content)
                data = MultipartEncoder(fields=multipart_encoder_fields)

            try:
                if request_type == RequestType.GET:
                    response = requests.get(url, headers=headers,
                                            params=params, stream=stream)
                elif request_type == RequestType.PUT:
                    response = requests.put(url, headers=headers, data=data,
                                            params=params)
                elif request_type == RequestType.POST:
                    response = requests.post(url, headers=headers, data=data)
            except requests.exceptions.ConnectionError as err:
                logger.warning("ConnectionError: {}".format(err))
                response = None
            finally:
                if use_multipart_encoder is True:
                    cur_multipart_file.close()

        if response is None:
            raise RemoteConnectionError("{}: Unable to complete request."
                                        .format(action_string))
        if response.status_code not in status_codes:
            logger.warning("{}: Connection failed Code: {}"
                           .format(action_string, str(response.status_code)))
            logger.warning("Error: {}".format(response.text))
            logger.warning("Headers: {}".format(str(response.headers)))
            logger.warning("Retry " + str(tries))
            raise RemoteConnectionError("{}: Unable to complete request."
                                        .format(action_string))

        return response

Example 21

Project: baidu-pcs-python-sdk
Source File: api.py
View license
    @check_token
    def _request(self, uri, method, url=None, extra_params=None,
                 data=None, files=None, **kwargs):
        params = {
            'method': method,
            'access_token': self.access_token
        }
        if extra_params:
            params.update(extra_params)
            self._remove_empty_items(params)

        if not url:
            url = self.api_template.format(uri)
        api = url

        if data or files:
            api = '%s?%s' % (url, urlencode(params))
            if data:
                self._remove_empty_items(data)
            else:
                self._remove_empty_items(files)
                data = MultipartEncoder(files)
                if kwargs.get('headers'):
                    kwargs['headers']['Content-Type'] = data.content_type
                else:
                    kwargs['headers'] = {'Content-Type': data.content_type}
            response = requests.post(api, data=data, **kwargs)
        else:
            response = requests.get(api, params=params, **kwargs)
        return response

Example 22

Project: onitu
Source File: auth.py
View license
    def do_upload(self, filename, url, params=None, fileobj=None):
        '''Performs a file upload to the given URL with the given parameters,
        signed with OAuth.

        @return: the response content
        '''

        # work-around to allow non-ascii characters in file name
        # Flickr doesn't store the name but does use it as a default title
        if 'title' not in params:
            params['title'] = os.path.basename(filename)

        # work-around for Flickr expecting 'photo' to be excluded
        # from the oauth signature:
        #   1. create a dummy request without 'photo'
        #   2. create real request and use auth headers from the dummy one
        dummy_req = requests.Request('POST', url, data=params,
                                     auth=self.oauth,
                                     headers={'Connection': 'close'})

        prepared = dummy_req.prepare()
        headers = prepared.headers
        self.log.debug('do_upload: prepared headers = %s', headers)

        if not fileobj:
            fileobj = open(filename, 'rb')
        params['photo'] = ('dummy name', fileobj)
        m = MultipartEncoder(fields=params)
        auth = {'Authorization': headers.get('Authorization'),
                'Content-Type': m.content_type,
                'Connection': 'close'}
        self.log.debug('POST %s', auth)
        req = requests.post(url, data=m, headers=auth)

        # check the response headers / status code.
        if req.status_code != 200:
            self.log.error('do_upload: Status code %i received, content:',
                           req.status_code)

            for part in req.content.split('&'):
                self.log.error('    %s', urllib_parse.unquote(part))

            raise exceptions.FlickrError('do_upload: Status code %s received'
                                         % req.status_code)

        return req.content

Example 23

Project: ansible-pan
Source File: panos_import.py
View license
def import_file(xapi, module, ip_address, file_, category):
    xapi.keygen()

    params = {
        'type': 'import',
        'category': category,
        'key': xapi.api_key
    }

    filename = os.path.basename(file_)

    mef = requests_toolbelt.MultipartEncoder(
        fields={
            'file': (filename, open(file_, 'rb'), 'application/octet-stream')
        }
    )

    r = requests.post(
        'https://'+ip_address+'/api/',
        verify=False,
        params=params,
        headers={'Content-Type': mef.content_type},
        data=mef
    )

    # if something goes wrong just raise an exception
    r.raise_for_status()

    resp = xml.etree.ElementTree.fromstring(r.content)
    if resp.attrib['status'] == 'error':
        module.fail_json(msg=r.content)

    return True, filename

Example 24

Project: AstroBox
Source File: cloud.py
View license
	def uploadImageFile(self, print_id, imageBuf):
		try:
			m = MultipartEncoder(fields=[('file',('snapshot.jpg', imageBuf))])
			r = requests.post(
				"%s/prints/%s/image" % (self.apiHost, print_id),
				data= m,
				headers= {'Content-Type': m.content_type},
				auth= self.hmacAuth
			)
			m = None #Free the memory?
			status_code = r.status_code
		except:
			status_code = 500

		if status_code == 201:
			data = r.json()
			return data

		else:
			return None