io.BytesIO

Here are the examples of the python api io.BytesIO taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

View license
def create_fake_jpeg_tar(seed, min_num_images=5, max_num_images=50,
                         min_size=20, size_range=30, filenames=None,
                         random=True, gzip_probability=0.5, offset=0):
    """Create a TAR file of randomly generated JPEG files.

    Parameters
    ----------
    seed : int or sequence
        Seed for a `numpy.random.RandomState`.
    min_num_images : int, optional
        The minimum number of images to put in the TAR file.
    max_num_images : int, optional
        The maximum number of images to put in the TAR file.
    min_size : int, optional
        The minimum width and minimum height of each image.
    size_range : int, optional
        Maximum number of pixels added to `min_size` for image
        dimensions.
    filenames : list, optional
        If provided, use these filenames. Otherwise generate them
        randomly. Must be at least `max_num_images` long.
    random : bool, optional
        If `False`, substitute an image full of a single number,
        the order of that image in processing.
    gzip_probability : float, optional
        With this probability, randomly gzip the JPEG file without
        appending a gzip suffix.
    offset : int, optional
        Where to start the hashes for filenames. Default: 0.

    Returns
    -------
    tar_data : bytes
        A TAR file represented as raw bytes, containing between
        `min_num_images` and `max_num_images` JPEG files (inclusive).

    Notes
    -----
    Randomly choose between RGB, L and CMYK mode images. Also randomly
    gzips JPEGs to simulate the idiotic distribution format of
    ILSVRC2010.

    """
    rng = numpy.random.RandomState(seed)
    images = []
    if filenames is None:
        files = []
    else:
        if len(filenames) < max_num_images:
            raise ValueError('need at least max_num_images = %d filenames' %
                             max_num_images)
        files = filenames
    for i in xrange(rng.random_integers(min_num_images, max_num_images)):
        if filenames is None:
            max_len = 27  # so that with suffix, 32 characters
            files.append('%s.JPEG' %
                         hashlib.sha1(bytes(i + offset)).hexdigest()[:max_len])
        im = rng.random_integers(0, 255,
                                 size=(rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(1, 4)))
        if not random:
            im *= 0
            assert (im == 0).all()
            im += i
            assert numpy.isscalar(i)
            assert (im == i).all()
        if im.shape[-1] == 2:
            im = im[:, :, :1]
        images.append(im)
    files = sorted(files)
    temp_tar = io.BytesIO()
    with tarfile.open(fileobj=temp_tar, mode='w') as tar:
        for fn, image in zip(files, images):
            try:
                with NamedTemporaryFile(mode='wb', suffix='.JPEG',
                                        delete=False) as f:
                    if rng.uniform() < gzip_probability:
                        gzip_data = io.BytesIO()
                        with gzip.GzipFile(mode='wb', fileobj=gzip_data) as gz:
                            gz.write(create_jpeg_data(image))
                        f.write(gzip_data.getvalue())
                    else:
                        f.write(create_jpeg_data(image))
                tar.add(f.name, arcname=fn)
            finally:
                os.remove(f.name)
    ordered_files = []
    with tarfile.open(fileobj=io.BytesIO(temp_tar.getvalue()),
                      mode='r') as tar:
        for info in tar.getmembers():
            ordered_files.append(info.name)
    return temp_tar.getvalue(), ordered_files

Example 2

Project: qgisSpaceSyntaxToolkit
Source File: test_gexf.py
View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('0',label='Hello')
        self.simple_directed_graph.add_node('1',label='World')
        self.simple_directed_graph.add_edge('0','1',id='0')

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd" version="1.1">
  <meta lastmodifieddate="2009-03-20">
    <creator>Gephi.org</creator>
    <description>A Web network</description>
  </meta>
  <graph defaultedgetype="directed">
    <attributes class="node">
      <attribute id="0" title="url" type="string"/>
      <attribute id="1" title="indegree" type="integer"/>
      <attribute id="2" title="frog" type="boolean">
        <default>true</default>
      </attribute>
    </attributes>
    <nodes>
      <node id="0" label="Gephi">
        <attvalues>
          <attvalue for="0" value="http://gephi.org"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
      <node id="1" label="Webatlas">
        <attvalues>
          <attvalue for="0" value="http://webatlas.fr"/>
          <attvalue for="1" value="2"/>
          <attvalue for="2" value="False"/>
        </attvalues>
      </node>
      <node id="2" label="RTGI">
        <attvalues>
          <attvalue for="0" value="http://rtgi.fr"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="true"/>
        </attvalues>
      </node>
      <node id="3" label="BarabasiLab">
        <attvalues>
          <attvalue for="0" value="http://barabasilab.com"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="True"/>
        </attvalues>
      </node>
    </nodes>
    <edges>
      <edge id="0" source="0" target="1"/>
      <edge id="1" source="0" target="2"/>
      <edge id="2" source="1" target="0"/>
      <edge id="3" source="2" target="1"/>
      <edge id="4" source="0" target="3"/>
    </edges>
  </graph>
</gexf>
"""
        self.attribute_graph=nx.DiGraph()
        self.attribute_graph.graph['node_default']={'frog':True}
        self.attribute_graph.add_node('0',
                                      label='Gephi',
                                      url='http://gephi.org',
                                      indegree=1,
                                      frog=False)
        self.attribute_graph.add_node('1',
                                      label='Webatlas',
                                      url='http://webatlas.fr',
                                      indegree=2,
                                      frog=False)

        self.attribute_graph.add_node('2',
                                      label='RTGI',
                                      url='http://rtgi.fr',
                                      indegree=1,
                                      frog=True)

        self.attribute_graph.add_node('3',
                                      label='BarabasiLab',
                                      url='http://barabasilab.com',
                                      indegree=1,
                                      frog=True)
        self.attribute_graph.add_edge('0','1',id='0')
        self.attribute_graph.add_edge('0','2',id='1')
        self.attribute_graph.add_edge('1','0',id='2')
        self.attribute_graph.add_edge('2','1',id='3')
        self.attribute_graph.add_edge('0','3',id='4')
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('0',label='Hello')
        self.simple_undirected_graph.add_node('1',label='World')
        self.simple_undirected_graph.add_edge('0','1',id='0')

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 3

View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<!-- This file was written by the JAVA GraphML Library.-->
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G" edgedefault="directed">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n3"/>
    <node id="n4"/>
    <node id="n5"/>
    <node id="n6"/>
    <node id="n7"/>
    <node id="n8"/>
    <node id="n9"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
    <edge source="n3" target="n5"/>
    <edge source="n3" target="n4"/>
    <edge source="n4" target="n6"/>
    <edge source="n6" target="n5"/>
    <edge source="n5" target="n7"/>
    <edge source="n6" target="n8"/>
    <edge source="n8" target="n7"/>
    <edge source="n8" target="n9"/>
  </graph>
</graphml>"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('n10')
        self.simple_directed_graph.add_edge('n0','n2',id='foo')
        self.simple_directed_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ('n3','n5'),
                                                   ('n3','n4'),
                                                   ('n4','n6'),
                                                   ('n6','n5'),
                                                   ('n5','n7'),
                                                   ('n6','n8'),
                                                   ('n8','n7'),
                                                   ('n8','n9'),
                                                   ])

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
        http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <key id="d0" for="node" attr.name="color" attr.type="string">
    <default>yellow</default>
  </key>
  <key id="d1" for="edge" attr.name="weight" attr.type="double"/>
  <graph id="G" edgedefault="directed">
    <node id="n0">
      <data key="d0">green</data>
    </node>
    <node id="n1"/>
    <node id="n2">
      <data key="d0">blue</data>
    </node>
    <node id="n3">
      <data key="d0">red</data>
    </node>
    <node id="n4"/>
    <node id="n5">
      <data key="d0">turquoise</data>
    </node>
    <edge id="e0" source="n0" target="n2">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e1" source="n0" target="n1">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e2" source="n1" target="n3">
      <data key="d1">2.0</data>
    </edge>
    <edge id="e3" source="n3" target="n2"/>
    <edge id="e4" source="n2" target="n4"/>
    <edge id="e5" source="n3" target="n5"/>
    <edge id="e6" source="n5" target="n4">
      <data key="d1">1.1</data>
    </edge>
  </graph>
</graphml>
"""
        self.attribute_graph=nx.DiGraph(id='G')
        self.attribute_graph.graph['node_default']={'color':'yellow'}
        self.attribute_graph.add_node('n0',color='green')
        self.attribute_graph.add_node('n2',color='blue')
        self.attribute_graph.add_node('n3',color='red')
        self.attribute_graph.add_node('n4')
        self.attribute_graph.add_node('n5',color='turquoise')
        self.attribute_graph.add_edge('n0','n2',id='e0',weight=1.0)
        self.attribute_graph.add_edge('n0','n1',id='e1',weight=1.0)
        self.attribute_graph.add_edge('n1','n3',id='e2',weight=2.0)
        self.attribute_graph.add_edge('n3','n2',id='e3')
        self.attribute_graph.add_edge('n2','n4',id='e4')
        self.attribute_graph.add_edge('n3','n5',id='e5')
        self.attribute_graph.add_edge('n5','n4',id='e6',weight=1.1)
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
  </graph>
</graphml>"""
#    <edge source="n8" target="n10" directed="false"/>
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('n10')
        self.simple_undirected_graph.add_edge('n0','n2',id='foo')
        self.simple_undirected_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ])

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 4

Project: fwlite
Source File: basesocket.py
View license
    def readline(self, size=-1):
        buf = self._rbuffer
        buf.seek(0, 2)  # seek end
        if buf.tell() > 0:
            # check if we already have it in our buffer
            buf.seek(0)
            bline = buf.readline(size)
            if bline.endswith('\n') or len(bline) == size:
                self._rbuffer = io.BytesIO()
                self._rbuffer.write(buf.read())
                return bline
            del bline
        if size < 0:
            # Read until \n or EOF, whichever comes first
            buf.seek(0, 2)  # seek end
            self._rbuffer = io.BytesIO()  # reset _rbuf.  we consume it via buf.
            while True:
                try:
                    data = self.recv(self.bufsize)
                except socket.error as e:
                    if e.args[0] == errno.EINTR:
                        continue
                    raise
                if not data:
                    break
                nl = data.find(b'\n')
                if nl >= 0:
                    nl += 1
                    buf.write(data[:nl])
                    self._rbuffer.write(data[nl:])
                    break
                buf.write(data)
            del data
            return buf.getvalue()
        else:
            # Read until size bytes or \n or EOF seen, whichever comes first
            buf.seek(0, 2)  # seek end
            buf_len = buf.tell()
            if buf_len >= size:
                buf.seek(0)
                rv = buf.read(size)
                self._rbuffer = io.BytesIO()
                self._rbuffer.write(buf.read())
                return rv
            self._rbuffer = io.BytesIO()  # reset _rbuf.  we consume it via buf.
            while True:
                try:
                    data = self.recv(self.bufsize)
                except socket.error as e:
                    if e.args[0] == errno.EINTR:
                        continue
                    raise
                if not data:
                    break
                left = size - buf_len
                # did we just receive a newline?
                nl = data.find(b'\n', 0, left)
                if nl >= 0:
                    nl += 1
                    # save the excess data to _rbuf
                    self._rbuffer.write(data[nl:])
                    if buf_len:
                        buf.write(data[:nl])
                        break
                    else:
                        # Shortcut.  Avoid data copy through buf when returning
                        # a substring of our first recv().
                        return data[:nl]
                n = len(data)
                if n == size and not buf_len:
                    # Shortcut.  Avoid data copy through buf when
                    # returning exactly all of our first recv().
                    return data
                if n >= left:
                    buf.write(data[:left])
                    self._rbuffer.write(data[left:])
                    break
                buf.write(data)
                buf_len += n
                # assert buf_len == buf.tell()
            return buf.getvalue()

Example 5

Project: py-starbound
Source File: repair.py
View license
def main():
    p = optparse.OptionParser('Usage: %prog [options] <input file>')
    p.add_option('-f', '--force', dest='force',
                 action='store_true', default=False,
                 help='ignore some errors')
    p.add_option('-o', '--output', dest='output',
                 help='where to output repaired world (defaults to input file '
                      'path with .repaired added to the end)')
    p.add_option('-w', '--blank-world', dest='world',
                 help='the blank .world file that was created in place of the '
                      '.fail one (for metadata recovery)')
    options, arguments = p.parse_args()
    # Get the path from arguments.
    if len(arguments) != 1:
        p.error('incorrect number of arguments')
    try:
        fh = open(arguments[0], 'rb')
        file_size = os.fstat(fh.fileno()).st_size
        world = starbound.World(fh)
    except Exception as e:
        p.error('could not open fail file ({})'.format(e))
    # Output path (defaults to fail file + .repaired).
    if options.output:
        out_name = options.output
    else:
        out_name = arguments[0] + '.repaired'
    # Ensure the user doesn't accidentally overwrite existing files.
    if os.path.isfile(out_name):
        if options.force:
            print('warning: overwriting existing file')
        else:
            p.error('"{}" already exists'.format(out_name))
    # Allow user to use the fresh world for metadata (which should be the same).
    if options.world:
        fail_name = os.path.basename(arguments[0])
        world_name = os.path.basename(options.world)
        if fail_name[:len(world_name)] != world_name:
            if options.force:
                print('warning: .fail and .world filenames do not match')
            else:
                p.error('.fail and .world filenames do not match')
        try:
            blank_world = starbound.World(open(options.world, 'rb'))
        except Exception as e:
            p.error('could not open blank world ({})'.format(e))
    # This dict will contain all the keys and their data.
    data = dict()
    try:
        world.read_metadata()
        metadata, version = world.metadata, world.metadata_version
    except Exception as e:
        if options.world:
            try:
                print('warning: restoring metadata using blank world')
                blank_world.read_metadata()
                metadata, version = blank_world.metadata, blank_world.metadata_version
            except Exception as e:
                p.error('failed to restore metadata ({})'.format(e))
        else:
            p.error('metadata section is corrupt ({})'.format(e))
    try:
        size = metadata['worldTemplate']['size']
    except Exception as e:
        size = [-1, -1]
        print('warning: failed to read world size ({})'.format(e))
    regions_x = int(math.ceil(size[0] / 32))
    regions_y = int(math.ceil(size[1] / 32))
    print('attempting to recover {}×{} regions...'.format(regions_x, regions_y))
    block_count = int((file_size - starbound.btreedb5.HEADER_SIZE) / world.block_size)
    blocks_per_percent = block_count // 100 + 1
    nodes_recovered = 0
    percent = 0
    # Find all leaves and try to read them individually.
    for index in range(block_count):
        if index % blocks_per_percent == 0:
            print('{}% ({} nodes recovered)'.format(percent, nodes_recovered))
            percent += 1
        # Seek to the block and only process it if it's a leaf.
        world.stream.seek(starbound.btreedb5.HEADER_SIZE + world.block_size * index)
        if world.stream.read(2) != starbound.btreedb5.LEAF:
            continue
        stream = starbound.btreedb5.LeafReader(world)
        try:
            num_keys, = struct.unpack('>i', stream.read(4))
        except Exception as e:
            print('failed to read keys of leaf block #{}: {}'.format(index, e))
            continue
        # Ensure that the number of keys makes sense, otherwise skip the leaf.
        if num_keys > 100:
            continue
        for i in range(num_keys):
            try:
                cur_key = stream.read(world.key_size)
                cur_data = starbound.sbon.read_bytes(stream)
            except Exception as e:
                print('could not read key/data: {}'.format(e))
                break
            layer, x, y = struct.unpack('>BHH', cur_key)
            # Skip this leaf if we encounter impossible indexes.
            if layer == 0 and (x != 0 or y != 0):
                break
            if layer not in (0, 1, 2) or x >= regions_x or y >= regions_y:
                break
            result = None
            if cur_key in data:
                # Duplicates should be checked up against the index, which always wins.
                # TODO: Make this code run again.
                try:
                    #result = world.get(layer, x, y)
                    result = None
                except Exception:
                    world.swap_root()
                    try:
                        #result = world.get(layer, x, y)
                        result = None
                    except Exception:
                        pass
                    world.swap_root()
            # Use the data from this leaf if not using the index.
            if not result:
                try:
                    result = zlib.decompress(cur_data)
                except Exception as e:
                    print('broken leaf node: {}'.format(e))
                    continue
            # Validate the data before storing it.
            try:
                if layer == 0:
                    temp_stream = io.BytesIO(result)
                    temp_stream.seek(8)
                    name, _, _ = starbound.read_versioned_json(temp_stream)
                    assert name == 'WorldMetadata', 'broken world metadata'
                elif layer == 1:
                    assert len(result) == 3 + 32 * 32 * 30, 'broken region data'
                elif layer == 2:
                    temp_stream = io.BytesIO(result)
                    for _ in range(starbound.sbon.read_varint(temp_stream)):
                        starbound.read_versioned_json(temp_stream)
            except Exception as e:
                print('invalid key data: {}'.format(e))
                continue
            # Count the node the first time it's stored.
            if cur_key not in data:
                nodes_recovered += 1
            data[cur_key] = zlib.compress(result)
    METADATA_KEY = b'\x00\x00\x00\x00\x00'
    # Ensure that the metadata key is in the data.
    if METADATA_KEY not in data:
        if options.world:
            try:
                data[METADATA_KEY] = blank_world.get(0, 0, 0)
            except Exception:
                p.error('failed to recover metadata from alternate world')
        else:
            if options.force:
                try:
                    data[METADATA_KEY] = world.get(0, 0, 0)
                    print('warning: using partially recovered metadata')
                except Exception:
                    p.error('failed to recover partial metadata')
            else:
                p.error('failed to recover metadata; use -w to load metadata '
                        'from another world, or -f to attempt partial recovery')
    print('done! {} nodes recovered'.format(nodes_recovered))
    print('creating BTree database...')
    # Try not to exceed this number of keys per leaf.
    LEAF_KEYS_TRESHOLD = 10
    # Try not to exceed this size for a leaf.
    LEAF_SIZE_TRESHOLD = world.block_size * .8
    # Fill indexes up to this ratio.
    INDEX_FILL = .9
    # 6 is the number of bytes used for signature + next block pointer.
    LEAF_BYTES = world.block_size - 6
    # 11 is the number of bytes in the index header.
    INDEX_BYTES = world.block_size - 11
    # Maximum number of keys that can go into an index.
    INDEX_MAX_KEYS = int(INDEX_BYTES // (world.key_size + 4) * INDEX_FILL)
    # The data of individual blocks will be stored in this list.
    blocks = []
    buffer = io.BytesIO()

    # This will create an initial leaf and connect it to following leaves which
    # will all contain the data currently in the buffer.
    def dump_buffer():
        buffer_size = buffer.tell()
        buffer.seek(0)
        block_data = b'LL' + struct.pack('>i', num_keys) + buffer.read(LEAF_BYTES - 4)
        while buffer.tell() < buffer_size:
            blocks.append(block_data + struct.pack('>i', len(blocks) + 1))
            block_data = b'LL' + buffer.read(LEAF_BYTES)
        blocks.append(block_data.ljust(world.block_size - 4, b'\x00') + struct.pack('>i', -1))
        # Empty the buffer.
        buffer.seek(0)
        buffer.truncate()

    # The number of keys that will be stored in the next created leaf.
    num_keys = 0
    # Map of key range to leaf block pointer.
    range_to_leaf = dict()
    # All the keys, sorted (important).
    keys = sorted(data)
    # Build all the leaf blocks.
    min_key = None
    for key in keys:
        if not num_keys:
            # Remember the first key of the leaf.
            min_key = key
        buffer.write(key)
        starbound.sbon.write_bytes(buffer, data[key])
        num_keys += 1
        # Empty buffer once one of the tresholds is reached.
        if num_keys >= LEAF_KEYS_TRESHOLD or buffer.tell() >= LEAF_SIZE_TRESHOLD:
            range_to_leaf[(min_key, key)] = len(blocks)
            dump_buffer()
            num_keys = 0
    # Empty any remaining data in the buffer.
    if buffer.tell():
        range_to_leaf[(min_key, key)] = len(blocks)
        dump_buffer()
    print('created {} blocks containing world data'.format(len(blocks)))

    def build_index_level(range_to_block, level=0):
        # Get a list of ranges that this index level needs to point to.
        index_ranges = sorted(range_to_block)
        # The new list of ranges that the next level of indexes can use.
        new_ranges = dict()
        for i in range(0, len(index_ranges), INDEX_MAX_KEYS):
            ranges = index_ranges[i:i + INDEX_MAX_KEYS]
            min_key, _ = ranges[0]
            _, max_key = ranges[-1]
            left_block = range_to_block[ranges.pop(0)]
            index_data = io.BytesIO()
            index_data.write(b'II' + struct.pack('>Bii', level, len(ranges), left_block))
            for key_range in ranges:
                index_data.write(key_range[0] + struct.pack('>i', range_to_block[key_range]))
            new_ranges[(min_key, max_key)] = len(blocks)
            blocks.append(index_data.getvalue().ljust(world.block_size, b'\x00'))
        print('- created {} index(es) for level {}'.format(len(new_ranges), level))
        return new_ranges

    # Build the indexes in multiple levels up to a single root node.
    print('creating root node...')
    root_is_leaf = True
    level = 0
    current_index = range_to_leaf
    while len(current_index) > 1:
        current_index = build_index_level(current_index, level)
        root_is_leaf = False
        level += 1
    root_node = list(current_index.values())[0]
    # Also build an alternative root node.
    print('creating alternate root node...')
    alternate_root_is_leaf = True
    level = 0
    current_index = range_to_leaf
    while len(current_index) > 1:
        current_index = build_index_level(current_index, level)
        alternate_root_is_leaf = False
        level += 1
    alternate_root_node = list(current_index.values())[0]
    # The last two blocks will be free blocks.
    blocks.append(b'FF\xFF\xFF\xFF\xFF' + b'\x00' * (world.block_size - 6))
    blocks.append(b'FF\xFF\xFF\xFF\xFF' + b'\x00' * (world.block_size - 6))
    print('writing all the data to disk...')
    with open(out_name, 'wb') as f:
        header = struct.pack(starbound.btreedb5.HEADER,
            b'BTreeDB5',
            world.block_size,
            world.name.encode('utf-8') + b'\x00' * (16 - len(world.name)),
            world.key_size,
            False,
            len(blocks) - 1,
            14282,  # XXX: Unknown value!
            root_node,
            root_is_leaf,
            len(blocks) - 2,
            14274,  # XXX: Unknown value!
            alternate_root_node,
            alternate_root_is_leaf)
        f.write(header)
        for block in blocks:
            f.write(block)
    print('done!')

Example 6

Project: Ragout
Source File: test_gexf.py
View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('0',label='Hello')
        self.simple_directed_graph.add_node('1',label='World')
        self.simple_directed_graph.add_edge('0','1',id='0')

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd" version="1.1">
  <meta lastmodifieddate="2009-03-20">
    <creator>Gephi.org</creator>
    <description>A Web network</description>
  </meta>
  <graph defaultedgetype="directed">
    <attributes class="node">
      <attribute id="0" title="url" type="string"/>
      <attribute id="1" title="indegree" type="integer"/>
      <attribute id="2" title="frog" type="boolean">
        <default>true</default>
      </attribute>
    </attributes>
    <nodes>
      <node id="0" label="Gephi">
        <attvalues>
          <attvalue for="0" value="http://gephi.org"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="1" label="Webatlas">
        <attvalues>
          <attvalue for="0" value="http://webatlas.fr"/>
          <attvalue for="1" value="2"/>
        </attvalues>
      </node>
      <node id="2" label="RTGI">
        <attvalues>
          <attvalue for="0" value="http://rtgi.fr"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="3" label="BarabasiLab">
        <attvalues>
          <attvalue for="0" value="http://barabasilab.com"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
    </nodes>
    <edges>
      <edge id="0" source="0" target="1"/>
      <edge id="1" source="0" target="2"/>
      <edge id="2" source="1" target="0"/>
      <edge id="3" source="2" target="1"/>
      <edge id="4" source="0" target="3"/>
    </edges>
  </graph>
</gexf>
"""
        self.attribute_graph=nx.DiGraph()
        self.attribute_graph.graph['node_default']={'frog':True}
        self.attribute_graph.add_node('0',
                                      label='Gephi',
                                      url='http://gephi.org',
                                      indegree=1)
        self.attribute_graph.add_node('1',
                                      label='Webatlas',
                                      url='http://webatlas.fr',
                                      indegree=2)

        self.attribute_graph.add_node('2',
                                      label='RTGI',
                                      url='http://rtgi.fr',
                                      indegree=1)

        self.attribute_graph.add_node('3',
                                      label='BarabasiLab',
                                      url='http://barabasilab.com',
                                      indegree=1,
                                      frog=False)
        self.attribute_graph.add_edge('0','1',id='0')
        self.attribute_graph.add_edge('0','2',id='1')
        self.attribute_graph.add_edge('1','0',id='2')
        self.attribute_graph.add_edge('2','1',id='3')
        self.attribute_graph.add_edge('0','3',id='4')
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('0',label='Hello')
        self.simple_undirected_graph.add_node('1',label='World')
        self.simple_undirected_graph.add_edge('0','1',id='0')

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 7

Project: Ragout
Source File: test_graphml.py
View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<!-- This file was written by the JAVA GraphML Library.-->
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G" edgedefault="directed">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n3"/>
    <node id="n4"/>
    <node id="n5"/>
    <node id="n6"/>
    <node id="n7"/>
    <node id="n8"/>
    <node id="n9"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
    <edge source="n3" target="n5"/>
    <edge source="n3" target="n4"/>
    <edge source="n4" target="n6"/>
    <edge source="n6" target="n5"/>
    <edge source="n5" target="n7"/>
    <edge source="n6" target="n8"/>
    <edge source="n8" target="n7"/>
    <edge source="n8" target="n9"/>
  </graph>
</graphml>"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('n10')
        self.simple_directed_graph.add_edge('n0','n2',id='foo')
        self.simple_directed_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ('n3','n5'),
                                                   ('n3','n4'),
                                                   ('n4','n6'),
                                                   ('n6','n5'),
                                                   ('n5','n7'),
                                                   ('n6','n8'),
                                                   ('n8','n7'),
                                                   ('n8','n9'),
                                                   ])

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
        http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <key id="d0" for="node" attr.name="color" attr.type="string">
    <default>yellow</default>
  </key>
  <key id="d1" for="edge" attr.name="weight" attr.type="double"/>
  <graph id="G" edgedefault="directed">
    <node id="n0">
      <data key="d0">green</data>
    </node>
    <node id="n1"/>
    <node id="n2">
      <data key="d0">blue</data>
    </node>
    <node id="n3">
      <data key="d0">red</data>
    </node>
    <node id="n4"/>
    <node id="n5">
      <data key="d0">turquoise</data>
    </node>
    <edge id="e0" source="n0" target="n2">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e1" source="n0" target="n1">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e2" source="n1" target="n3">
      <data key="d1">2.0</data>
    </edge>
    <edge id="e3" source="n3" target="n2"/>
    <edge id="e4" source="n2" target="n4"/>
    <edge id="e5" source="n3" target="n5"/>
    <edge id="e6" source="n5" target="n4">
      <data key="d1">1.1</data>
    </edge>
  </graph>
</graphml>
"""
        self.attribute_graph=nx.DiGraph(id='G')
        self.attribute_graph.graph['node_default']={'color':'yellow'}
        self.attribute_graph.add_node('n0',color='green')
        self.attribute_graph.add_node('n2',color='blue')
        self.attribute_graph.add_node('n3',color='red')
        self.attribute_graph.add_node('n4')
        self.attribute_graph.add_node('n5',color='turquoise')
        self.attribute_graph.add_edge('n0','n2',id='e0',weight=1.0)
        self.attribute_graph.add_edge('n0','n1',id='e1',weight=1.0)
        self.attribute_graph.add_edge('n1','n3',id='e2',weight=2.0)
        self.attribute_graph.add_edge('n3','n2',id='e3')
        self.attribute_graph.add_edge('n2','n4',id='e4')
        self.attribute_graph.add_edge('n3','n5',id='e5')
        self.attribute_graph.add_edge('n5','n4',id='e6',weight=1.1)
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
  </graph>
</graphml>"""
#    <edge source="n8" target="n10" directed="false"/>
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('n10')
        self.simple_undirected_graph.add_edge('n0','n2',id='foo')
        self.simple_undirected_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ])

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 8

Project: girder
Source File: file_test.py
View license
    def _testDownloadCollection(self):
        """
        Test downloading an entire collection as a zip file.
        """
        # Create a collection
        resp = self.request(
            path='/collection', method='POST', user=self.user, params={
                'name': 'Test Collection'
            })
        self.assertStatusOk(resp)
        collection = resp.json

        # Create a folder in the collection
        resp = self.request(
            path='/folder', method='POST', user=self.user, params={
                'name': 'Test Folder',
                'parentId': collection['_id'],
                'parentType': 'collection'
            })
        self.assertStatusOk(resp)

        test = resp.json
        contents = os.urandom(64)  # Generate random file contents

        # Upload the file into that subfolder
        resp = self.request(
            path='/file', method='POST', user=self.user, params={
                'parentType': 'folder',
                'parentId': test['_id'],
                'name': 'random.bin',
                'size': len(contents)
            })
        self.assertStatusOk(resp)

        uploadId = resp.json['_id']

        # Send the file contents
        fields = [('offset', 0), ('uploadId', uploadId)]
        files = [('chunk', 'random.bin', contents)]
        resp = self.multipartRequest(
            path='/file/chunk', user=self.user, fields=fields, files=files)
        self.assertStatusOk(resp)

        # Download the collection
        path = '/collection/%s/download' % str(collection['_id'])
        resp = self.request(
            path=path,
            method='GET', user=self.user, isJson=False)
        self.assertStatusOk(resp)
        self.assertEqual(resp.headers['Content-Disposition'],
                         'attachment; filename="Test Collection.zip"')
        self.assertEqual(resp.headers['Content-Type'], 'application/zip')
        zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
        self.assertTrue(zip.testzip() is None)

        extracted = zip.read('Test Collection/Test Folder/random.bin')
        self.assertEqual(extracted, contents)

        # Make collection public
        collection = self.model('collection').load(collection['_id'],
                                                   force=True)
        collection['public'] = True
        collection = self.model('collection').save(collection)

        # Download the collection as anonymous
        path = '/collection/%s/download' % str(collection['_id'])
        resp = self.request(
            path=path,
            method='GET', user=None, isJson=False)
        self.assertStatusOk(resp)
        zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
        # Zip file should have no entries
        self.assertFalse(zip.namelist())

        # Upload a known MIME-type file into the collection
        contents = b'not a jpeg'
        resp = self.request(
            path='/file', method='POST', user=self.user, params={
                'parentType': 'folder',
                'parentId': test['_id'],
                'name': 'fake.jpeg',
                'size': len(contents),
                'mimeType': 'image/jpeg'
            })
        self.assertStatusOk(resp)

        uploadId = resp.json['_id']

        # Send the file contents
        fields = [('offset', 0), ('uploadId', uploadId)]
        files = [('chunk', 'fake.jpeg', contents)]
        resp = self.multipartRequest(
            path='/file/chunk', user=self.user, fields=fields, files=files)
        self.assertStatusOk(resp)

        # Download the collection using a MIME type filter
        path = '/collection/%s/download' % str(collection['_id'])
        resp = self.request(
            path=path, method='GET', user=self.user, isJson=False, params={
                'mimeFilter': json.dumps(['image/png', 'image/jpeg'])
            })
        self.assertStatusOk(resp)
        self.assertEqual(resp.headers['Content-Disposition'],
                         'attachment; filename="Test Collection.zip"')
        self.assertEqual(resp.headers['Content-Type'], 'application/zip')
        zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
        self.assertTrue(zip.testzip() is None)

        # Only the jpeg should exist in the zip
        path = 'Test Collection/Test Folder/fake.jpeg'
        self.assertEqual(zip.namelist(), [path])
        extracted = zip.read(path)
        self.assertEqual(extracted, contents)

Example 9

Project: girder
Source File: resource_test.py
View license
    def testDownloadResources(self):
        self._createFiles()
        resourceList = {
            'collection': [str(self.collection['_id'])],
            'user': [str(self.admin['_id'])]
            }
        # We should fail with bad json, an empty list, an invalid item in the
        # list, or a list that is an odd format.
        resp = self.request(
            path='/resource/download', method='GET', user=self.admin, params={
                'resources': 'this_is_not_json',
            }, isJson=False)
        self.assertStatus(resp, 400)
        resp = self.request(
            path='/resource/download', method='GET', user=self.admin, params={
                'resources': json.dumps('this_is_not_a_dict_of_resources')
            }, isJson=False)
        self.assertStatus(resp, 400)
        resp = self.request(
            path='/resource/download', method='GET', user=self.admin, params={
                'resources': json.dumps({'not_a_resource': ['not_an_id']})
            }, isJson=False)
        self.assertStatus(resp, 400)
        resp = self.request(
            path='/resource/download', method='GET', user=self.admin, params={
                'resources': json.dumps({'item': []})
            }, isJson=False)
        self.assertStatus(resp, 400)
        resp = self.request(
            path='/resource/download', method='GET', user=self.admin, params={
                'resources': json.dumps({'item': [str(self.admin['_id'])]})
            }, isJson=False)
        self.assertStatus(resp, 400)
        # Download the resources
        resp = self.request(
            path='/resource/download', method='GET', user=self.admin, params={
                'resources': json.dumps(resourceList),
                'includeMetadata': True
            }, isJson=False)
        self.assertStatusOk(resp)
        self.assertEqual(resp.headers['Content-Type'], 'application/zip')
        zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
        self.assertTrue(zip.testzip() is None)
        self.assertHasKeys(self.expectedZip, zip.namelist())
        self.assertHasKeys(zip.namelist(), self.expectedZip)
        for name in zip.namelist():
            expected = self.expectedZip[name]
            if isinstance(expected, dict):
                self.assertEqual(json.loads(zip.read(name).decode('utf8')),
                                 json.loads(json.dumps(expected, default=str)))
            else:
                if not isinstance(expected, six.binary_type):
                    expected = expected.encode('utf8')
                self.assertEqual(expected, zip.read(name))
        # Download the same resources again, this time triggering the large zip
        # file creation (artifically forced).  We could do this naturally by
        # downloading >65536 files, but that would make the test take several
        # minutes.
        girder.utility.ziputil.Z_FILECOUNT_LIMIT = 5
        resourceList = {
            'item': [str(item['_id']) for item in self.items]
            }
        resp = self.request(
            path='/resource/download', method='POST', user=self.admin, params={
                'resources': json.dumps(resourceList),
                'includeMetadata': True
            }, isJson=False,
            additionalHeaders=[('X-HTTP-Method-Override', 'GET')])
        self.assertStatusOk(resp)
        self.assertEqual(resp.headers['Content-Type'], 'application/zip')
        zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
        self.assertTrue(zip.testzip() is None)

        # Test deleting resources
        resourceList = {
            'collection': [str(self.collection['_id'])],
            'folder': [str(self.adminSubFolder['_id'])],
            }
        resp = self.request(
            path='/resource', method='DELETE', user=self.admin, params={
                'resources': json.dumps(resourceList),
                'progress': True
            }, isJson=False)
        self.assertStatusOk(resp)
        # Make sure progress record exists and that it is set to expire soon
        notifs = list(self.model('notification').get(self.admin))
        self.assertEqual(len(notifs), 1)
        self.assertEqual(notifs[0]['type'], 'progress')
        self.assertEqual(notifs[0]['data']['state'], ProgressState.SUCCESS)
        self.assertEqual(notifs[0]['data']['title'], 'Deleting resources')
        self.assertEqual(notifs[0]['data']['message'], 'Done')
        self.assertEqual(notifs[0]['data']['total'], 6)
        self.assertEqual(notifs[0]['data']['current'], 6)
        self.assertTrue(notifs[0]['expires'] < datetime.datetime.utcnow() +
                        datetime.timedelta(minutes=1))
        # Test deletes using a body on the request
        resourceList = {
            'item': [str(self.items[1]['_id'])]
            }
        resp = self.request(
            path='/resource', method='DELETE', user=self.admin,
            body=urllib.parse.urlencode({
                'resources': json.dumps(resourceList)
            }),
            type='application/x-www-form-urlencoded', isJson=False)
        self.assertStatusOk(resp)
        # Test deletes using POST and override method
        resourceList = {
            'item': [str(self.items[0]['_id'])]
            }
        resp = self.request(
            path='/resource', method='POST', user=self.admin, params={
                'resources': json.dumps(resourceList)
            }, isJson=False,
            additionalHeaders=[('X-HTTP-Method-Override', 'DELETE')])
        self.assertStatusOk(resp)
        # All of the items should be gone now
        resp = self.request(path='/item', method='GET', user=self.admin,
                            params={'text': 'Item'})
        self.assertStatusOk(resp)
        self.assertEqual(len(resp.json), 0)

        # Add a file under the admin private folder
        item = self.model('item').createItem(
            'Private Item', self.admin, self.adminPrivateFolder)
        _, path, contents = self._uploadFile('private_file', item)
        self.assertEqual(path, 'goodlogin/Private/Private Item/private_file')

        # Download as admin, should get private file
        resp = self.request(
            path='/resource/download', method='GET', user=self.admin, params={
                'resources': json.dumps({'user': [str(self.admin['_id'])]})
            }, isJson=False)
        self.assertStatusOk(resp)
        self.assertEqual(resp.headers['Content-Type'], 'application/zip')
        zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
        self.assertTrue(zip.testzip() is None)
        self.assertEqual(zip.namelist(), [path])
        self.assertEqual(zip.read(path), contents)

        # Download as normal user, should get empty zip
        resp = self.request(
            path='/resource/download', method='GET', user=self.user, params={
                'resources': json.dumps({'user': [str(self.admin['_id'])]})
            }, isJson=False)
        self.assertStatusOk(resp)
        self.assertEqual(resp.headers['Content-Type'], 'application/zip')
        zip = zipfile.ZipFile(io.BytesIO(self.getBody(resp, text=False)), 'r')
        self.assertTrue(zip.testzip() is None)
        self.assertEqual(zip.namelist(), [])

Example 10

Project: gkno_launcher
Source File: test_gexf.py
View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('0',label='Hello')
        self.simple_directed_graph.add_node('1',label='World')
        self.simple_directed_graph.add_edge('0','1',id='0')

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd" version="1.1">
  <meta lastmodifieddate="2009-03-20">
    <creator>Gephi.org</creator>
    <description>A Web network</description>
  </meta>
  <graph defaultedgetype="directed">
    <attributes class="node">
      <attribute id="0" title="url" type="string"/>
      <attribute id="1" title="indegree" type="integer"/>
      <attribute id="2" title="frog" type="boolean">
        <default>true</default>
      </attribute>
    </attributes>
    <nodes>
      <node id="0" label="Gephi">
        <attvalues>
          <attvalue for="0" value="http://gephi.org"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="1" label="Webatlas">
        <attvalues>
          <attvalue for="0" value="http://webatlas.fr"/>
          <attvalue for="1" value="2"/>
        </attvalues>
      </node>
      <node id="2" label="RTGI">
        <attvalues>
          <attvalue for="0" value="http://rtgi.fr"/>
          <attvalue for="1" value="1"/>
        </attvalues>
      </node>
      <node id="3" label="BarabasiLab">
        <attvalues>
          <attvalue for="0" value="http://barabasilab.com"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
    </nodes>
    <edges>
      <edge id="0" source="0" target="1"/>
      <edge id="1" source="0" target="2"/>
      <edge id="2" source="1" target="0"/>
      <edge id="3" source="2" target="1"/>
      <edge id="4" source="0" target="3"/>
    </edges>
  </graph>
</gexf>
"""
        self.attribute_graph=nx.DiGraph()
        self.attribute_graph.graph['node_default']={'frog':True}
        self.attribute_graph.add_node('0',
                                      label='Gephi',
                                      url='http://gephi.org',
                                      indegree=1)
        self.attribute_graph.add_node('1',
                                      label='Webatlas',
                                      url='http://webatlas.fr',
                                      indegree=2)

        self.attribute_graph.add_node('2',
                                      label='RTGI',
                                      url='http://rtgi.fr',
                                      indegree=1)

        self.attribute_graph.add_node('3',
                                      label='BarabasiLab',
                                      url='http://barabasilab.com',
                                      indegree=1,
                                      frog=False)
        self.attribute_graph.add_edge('0','1',id='0')
        self.attribute_graph.add_edge('0','2',id='1')
        self.attribute_graph.add_edge('1','0',id='2')
        self.attribute_graph.add_edge('2','1',id='3')
        self.attribute_graph.add_edge('0','3',id='4')
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('0',label='Hello')
        self.simple_undirected_graph.add_node('1',label='World')
        self.simple_undirected_graph.add_edge('0','1',id='0')

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 11

Project: gkno_launcher
Source File: test_graphml.py
View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<!-- This file was written by the JAVA GraphML Library.-->
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G" edgedefault="directed">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n3"/>
    <node id="n4"/>
    <node id="n5"/>
    <node id="n6"/>
    <node id="n7"/>
    <node id="n8"/>
    <node id="n9"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
    <edge source="n3" target="n5"/>
    <edge source="n3" target="n4"/>
    <edge source="n4" target="n6"/>
    <edge source="n6" target="n5"/>
    <edge source="n5" target="n7"/>
    <edge source="n6" target="n8"/>
    <edge source="n8" target="n7"/>
    <edge source="n8" target="n9"/>
  </graph>
</graphml>"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('n10')
        self.simple_directed_graph.add_edge('n0','n2',id='foo')
        self.simple_directed_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ('n3','n5'),
                                                   ('n3','n4'),
                                                   ('n4','n6'),
                                                   ('n6','n5'),
                                                   ('n5','n7'),
                                                   ('n6','n8'),
                                                   ('n8','n7'),
                                                   ('n8','n9'),
                                                   ])

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
        http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <key id="d0" for="node" attr.name="color" attr.type="string">
    <default>yellow</default>
  </key>
  <key id="d1" for="edge" attr.name="weight" attr.type="double"/>
  <graph id="G" edgedefault="directed">
    <node id="n0">
      <data key="d0">green</data>
    </node>
    <node id="n1"/>
    <node id="n2">
      <data key="d0">blue</data>
    </node>
    <node id="n3">
      <data key="d0">red</data>
    </node>
    <node id="n4"/>
    <node id="n5">
      <data key="d0">turquoise</data>
    </node>
    <edge id="e0" source="n0" target="n2">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e1" source="n0" target="n1">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e2" source="n1" target="n3">
      <data key="d1">2.0</data>
    </edge>
    <edge id="e3" source="n3" target="n2"/>
    <edge id="e4" source="n2" target="n4"/>
    <edge id="e5" source="n3" target="n5"/>
    <edge id="e6" source="n5" target="n4">
      <data key="d1">1.1</data>
    </edge>
  </graph>
</graphml>
"""
        self.attribute_graph=nx.DiGraph(id='G')
        self.attribute_graph.graph['node_default']={'color':'yellow'}
        self.attribute_graph.add_node('n0',color='green')
        self.attribute_graph.add_node('n2',color='blue')
        self.attribute_graph.add_node('n3',color='red')
        self.attribute_graph.add_node('n4')
        self.attribute_graph.add_node('n5',color='turquoise')
        self.attribute_graph.add_edge('n0','n2',id='e0',weight=1.0)
        self.attribute_graph.add_edge('n0','n1',id='e1',weight=1.0)
        self.attribute_graph.add_edge('n1','n3',id='e2',weight=2.0)
        self.attribute_graph.add_edge('n3','n2',id='e3')
        self.attribute_graph.add_edge('n2','n4',id='e4')
        self.attribute_graph.add_edge('n3','n5',id='e5')
        self.attribute_graph.add_edge('n5','n4',id='e6',weight=1.1)
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
  </graph>
</graphml>"""
#    <edge source="n8" target="n10" directed="false"/>
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('n10')
        self.simple_undirected_graph.add_edge('n0','n2',id='foo')
        self.simple_undirected_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ])

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 12

Project: Piexif
Source File: s_test.py
View license
    def test_merge_segments(self):
        # Remove APP0, when both APP0 and APP1 exists.
        with open(INPUT_FILE1, "rb") as f:
            original = f.read()
        segments = _common.split_into_segments(original)
        new_data = _common.merge_segments(segments)
        segments = _common.split_into_segments(new_data)
        self.assertFalse([1][0:2] == b"\xff\xe0"
                        and segments[2][0:2] == b"\xff\xe1")
        self.assertEqual(segments[1][0:2], b"\xff\xe1")
        o = io.BytesIO(new_data)
        without_app0 = o.getvalue()
        Image.open(o).close()

        exif = _common.get_exif_seg(segments)

        # Remove Exif, when second 'merged_segments' arguments is None
        # and no APP0.
        segments = _common.split_into_segments(without_app0)
        new_data = _common.merge_segments(segments, None)
        segments = _common.split_into_segments(new_data)
        self.assertNotEqual(segments[1][0:2], b"\xff\xe0")
        self.assertNotEqual(segments[1][0:2], b"\xff\xe1")
        self.assertNotEqual(segments[2][0:2], b"\xff\xe1")
        o = io.BytesIO(new_data)
        Image.open(o).close()

        # Insert exif to jpeg that has APP0 and Exif.
        o = io.BytesIO()
        i = Image.new("RGBA", (8, 8))
        i.save(o, format="jpeg", exif=exif)
        o.seek(0)
        segments = _common.split_into_segments(o.getvalue())
        new_data = _common.merge_segments(segments, exif)
        segments = _common.split_into_segments(new_data)
        self.assertFalse(segments[1][0:2] == b"\xff\xe0"
                         and segments[2][0:2] == b"\xff\xe1")
        self.assertEqual(segments[1], exif)
        o = io.BytesIO(new_data)
        Image.open(o).close()

        # Insert exif to jpeg that doesn't have APP0 and Exif.
        with open(NOAPP01_FILE, "rb") as f:
            original = f.read()
        segments = _common.split_into_segments(original)
        new_data = _common.merge_segments(segments, exif)
        segments = _common.split_into_segments(new_data)
        self.assertEqual(segments[1][0:2], b"\xff\xe1")
        o = io.BytesIO(new_data)
        Image.open(o).close()

        # Remove Exif, when second 'merged_segments' arguments is None
        # and Exif exists.
        with open(INPUT_FILE1, "rb") as f:
            original = f.read()
        segments = _common.split_into_segments(original)
        new_data = _common.merge_segments(segments, None)
        segments = _common.split_into_segments(new_data)
        self.assertNotEqual(segments[1][0:2], b"\xff\xe1")
        self.assertNotEqual(segments[2][0:2], b"\xff\xe1")
        o = io.BytesIO(new_data)
        Image.open(o).close()

Example 13

Project: fuel
Source File: test_convert_ilsvrc2010.py
View license
def create_fake_jpeg_tar(seed, min_num_images=5, max_num_images=50,
                         min_size=20, size_range=30, filenames=None,
                         random=True, gzip_probability=0.5, offset=0):
    """Create a TAR file of randomly generated JPEG files.

    Parameters
    ----------
    seed : int or sequence
        Seed for a `numpy.random.RandomState`.
    min_num_images : int, optional
        The minimum number of images to put in the TAR file.
    max_num_images : int, optional
        The maximum number of images to put in the TAR file.
    min_size : int, optional
        The minimum width and minimum height of each image.
    size_range : int, optional
        Maximum number of pixels added to `min_size` for image
        dimensions.
    filenames : list, optional
        If provided, use these filenames. Otherwise generate them
        randomly. Must be at least `max_num_images` long.
    random : bool, optional
        If `False`, substitute an image full of a single number,
        the order of that image in processing.
    gzip_probability : float, optional
        With this probability, randomly gzip the JPEG file without
        appending a gzip suffix.
    offset : int, optional
        Where to start the hashes for filenames. Default: 0.

    Returns
    -------
    tar_data : bytes
        A TAR file represented as raw bytes, containing between
        `min_num_images` and `max_num_images` JPEG files (inclusive).

    Notes
    -----
    Randomly choose between RGB, L and CMYK mode images. Also randomly
    gzips JPEGs to simulate the idiotic distribution format of
    ILSVRC2010.

    """
    rng = numpy.random.RandomState(seed)
    images = []
    if filenames is None:
        files = []
    else:
        if len(filenames) < max_num_images:
            raise ValueError('need at least max_num_images = %d filenames' %
                             max_num_images)
        files = filenames
    for i in xrange(rng.random_integers(min_num_images, max_num_images)):
        if filenames is None:
            max_len = 27  # so that with suffix, 32 characters
            files.append('%s.JPEG' %
                         hashlib.sha1(bytes(i + offset)).hexdigest()[:max_len])
        im = rng.random_integers(0, 255,
                                 size=(rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(1, 4)))
        if not random:
            im *= 0
            assert (im == 0).all()
            im += i
            assert numpy.isscalar(i)
            assert (im == i).all()
        if im.shape[-1] == 2:
            im = im[:, :, :1]
        images.append(im)
    files = sorted(files)
    temp_tar = io.BytesIO()
    with tarfile.open(fileobj=temp_tar, mode='w') as tar:
        for fn, image in zip(files, images):
            try:
                with NamedTemporaryFile(mode='wb', suffix='.JPEG',
                                        delete=False) as f:
                    if rng.uniform() < gzip_probability:
                        gzip_data = io.BytesIO()
                        with gzip.GzipFile(mode='wb', fileobj=gzip_data) as gz:
                            gz.write(create_jpeg_data(image))
                        f.write(gzip_data.getvalue())
                    else:
                        f.write(create_jpeg_data(image))
                tar.add(f.name, arcname=fn)
            finally:
                os.remove(f.name)
    ordered_files = []
    with tarfile.open(fileobj=io.BytesIO(temp_tar.getvalue()),
                      mode='r') as tar:
        for info in tar.getmembers():
            ordered_files.append(info.name)
    return temp_tar.getvalue(), ordered_files

Example 14

Project: fuel
Source File: test_convert_ilsvrc2010.py
View license
def create_fake_jpeg_tar(seed, min_num_images=5, max_num_images=50,
                         min_size=20, size_range=30, filenames=None,
                         random=True, gzip_probability=0.5, offset=0):
    """Create a TAR file of randomly generated JPEG files.

    Parameters
    ----------
    seed : int or sequence
        Seed for a `numpy.random.RandomState`.
    min_num_images : int, optional
        The minimum number of images to put in the TAR file.
    max_num_images : int, optional
        The maximum number of images to put in the TAR file.
    min_size : int, optional
        The minimum width and minimum height of each image.
    size_range : int, optional
        Maximum number of pixels added to `min_size` for image
        dimensions.
    filenames : list, optional
        If provided, use these filenames. Otherwise generate them
        randomly. Must be at least `max_num_images` long.
    random : bool, optional
        If `False`, substitute an image full of a single number,
        the order of that image in processing.
    gzip_probability : float, optional
        With this probability, randomly gzip the JPEG file without
        appending a gzip suffix.
    offset : int, optional
        Where to start the hashes for filenames. Default: 0.

    Returns
    -------
    tar_data : bytes
        A TAR file represented as raw bytes, containing between
        `min_num_images` and `max_num_images` JPEG files (inclusive).

    Notes
    -----
    Randomly choose between RGB, L and CMYK mode images. Also randomly
    gzips JPEGs to simulate the idiotic distribution format of
    ILSVRC2010.

    """
    rng = numpy.random.RandomState(seed)
    images = []
    if filenames is None:
        files = []
    else:
        if len(filenames) < max_num_images:
            raise ValueError('need at least max_num_images = %d filenames' %
                             max_num_images)
        files = filenames
    for i in xrange(rng.random_integers(min_num_images, max_num_images)):
        if filenames is None:
            max_len = 27  # so that with suffix, 32 characters
            files.append('%s.JPEG' %
                         hashlib.sha1(bytes(i + offset)).hexdigest()[:max_len])
        im = rng.random_integers(0, 255,
                                 size=(rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(1, 4)))
        if not random:
            im *= 0
            assert (im == 0).all()
            im += i
            assert numpy.isscalar(i)
            assert (im == i).all()
        if im.shape[-1] == 2:
            im = im[:, :, :1]
        images.append(im)
    files = sorted(files)
    temp_tar = io.BytesIO()
    with tarfile.open(fileobj=temp_tar, mode='w') as tar:
        for fn, image in zip(files, images):
            try:
                with NamedTemporaryFile(mode='wb', suffix='.JPEG',
                                        delete=False) as f:
                    if rng.uniform() < gzip_probability:
                        gzip_data = io.BytesIO()
                        with gzip.GzipFile(mode='wb', fileobj=gzip_data) as gz:
                            gz.write(create_jpeg_data(image))
                        f.write(gzip_data.getvalue())
                    else:
                        f.write(create_jpeg_data(image))
                tar.add(f.name, arcname=fn)
            finally:
                os.remove(f.name)
    ordered_files = []
    with tarfile.open(fileobj=io.BytesIO(temp_tar.getvalue()),
                      mode='r') as tar:
        for info in tar.getmembers():
            ordered_files.append(info.name)
    return temp_tar.getvalue(), ordered_files

Example 15

Project: Arelle
Source File: FileSource.py
View license
    def file(self, filepath, binary=False, stripDeclaration=False, encoding=None):
        ''' 
            for text, return a tuple of (open file handle, encoding)
            for binary, return a tuple of (open file handle, )
        '''
        archiveFileSource = self.fileSourceContainingFilepath(filepath)
        if archiveFileSource is not None:
            if filepath.startswith(archiveFileSource.basefile):
                archiveFileName = filepath[len(archiveFileSource.basefile) + 1:]
            else: # filepath.startswith(self.baseurl)
                archiveFileName = filepath[len(archiveFileSource.baseurl) + 1:]
            if archiveFileSource.isZip:
                try:
                    b = archiveFileSource.fs.read(archiveFileName.replace("\\","/"))
                    if binary:
                        return (io.BytesIO(b), )
                    if encoding is None:
                        encoding = XmlUtil.encoding(b)
                    if stripDeclaration:
                        b = stripDeclarationBytes(b)
                    return (FileNamedTextIOWrapper(filepath, io.BytesIO(b), encoding=encoding), 
                            encoding)
                except KeyError:
                    raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
            elif archiveFileSource.isTarGz:
                try:
                    fh = archiveFileSource.fs.extractfile(archiveFileName)
                    b = fh.read()
                    fh.close() # doesn't seem to close properly using a with construct
                    if binary:
                        return (io.BytesIO(b), )
                    if encoding is None:
                        encoding = XmlUtil.encoding(b)
                    if stripDeclaration:
                        b = stripDeclarationBytes(b)
                    return (FileNamedTextIOWrapper(filepath, io.BytesIO(b), encoding=encoding), 
                            encoding)
                except KeyError:
                    raise ArchiveFileIOError(self, archiveFileName)
            elif archiveFileSource.isEis:
                for docElt in self.eisDocument.iter(tag="{http://www.sec.gov/edgar/common}document"):
                    outfn = docElt.findtext("{http://www.sec.gov/edgar/common}conformedName")
                    if outfn == archiveFileName:
                        b64data = docElt.findtext("{http://www.sec.gov/edgar/common}contents")
                        if b64data:
                            b = base64.b64decode(b64data.encode("latin-1"))
                            # remove BOM codes if present
                            if len(b) > 3 and b[0] == 239 and b[1] == 187 and b[2] == 191:
                                start = 3;
                                length = len(b) - 3;
                                b = b[start:start + length]
                            else:
                                start = 0;
                                length = len(b);
                            if binary:
                                return (io.BytesIO(b), )
                            if encoding is None:
                                encoding = XmlUtil.encoding(b, default="latin-1")
                            return (io.TextIOWrapper(io.BytesIO(b), encoding=encoding), 
                                    encoding)
                raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
            elif archiveFileSource.isXfd:
                for data in archiveFileSource.xfdDocument.iter(tag="data"):
                    outfn = data.findtext("filename")
                    if outfn == archiveFileName:
                        b64data = data.findtext("mimedata")
                        if b64data:
                            b = base64.b64decode(b64data.encode("latin-1"))
                            # remove BOM codes if present
                            if len(b) > 3 and b[0] == 239 and b[1] == 187 and b[2] == 191:
                                start = 3;
                                length = len(b) - 3;
                                b = b[start:start + length]
                            else:
                                start = 0;
                                length = len(b);
                            if binary:
                                return (io.BytesIO(b), )
                            if encoding is None:
                                encoding = XmlUtil.encoding(b, default="latin-1")
                            return (io.TextIOWrapper(io.BytesIO(b), encoding=encoding), 
                                    encoding)
                raise ArchiveFileIOError(self, errno.ENOENT, archiveFileName)
            elif archiveFileSource.isInstalledTaxonomyPackage:
                # remove TAXONOMY_PACKAGE_FILE_NAME from file path
                if filepath.startswith(archiveFileSource.basefile):
                    l = len(archiveFileSource.basefile)
                    for f in TAXONOMY_PACKAGE_FILE_NAMES:
                        if filepath[l - len(f):l] == f:
                            filepath = filepath[0:l - len(f) - 1] + filepath[l:]
                            break
        if binary:
            return (openFileStream(self.cntlr, filepath, 'rb'), )
        else:
            return openXmlFileStream(self.cntlr, filepath, stripDeclaration)

Example 16

Project: networkx
Source File: test_gexf.py
View license
    def setUp(self):
        self.simple_directed_data = """<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_directed_graph = nx.DiGraph()
        self.simple_directed_graph.add_node('0', label='Hello')
        self.simple_directed_graph.add_node('1', label='World')
        self.simple_directed_graph.add_edge('0', '1', id='0')

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))

        self.attribute_data = """<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd" version="1.1">
  <meta lastmodifieddate="2009-03-20">
    <creator>Gephi.org</creator>
    <description>A Web network</description>
  </meta>
  <graph defaultedgetype="directed">
    <attributes class="node">
      <attribute id="0" title="url" type="string"/>
      <attribute id="1" title="indegree" type="integer"/>
      <attribute id="2" title="frog" type="boolean">
        <default>true</default>
      </attribute>
    </attributes>
    <nodes>
      <node id="0" label="Gephi">
        <attvalues>
          <attvalue for="0" value="http://gephi.org"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
      <node id="1" label="Webatlas">
        <attvalues>
          <attvalue for="0" value="http://webatlas.fr"/>
          <attvalue for="1" value="2"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
      <node id="2" label="RTGI">
        <attvalues>
          <attvalue for="0" value="http://rtgi.fr"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="true"/>
        </attvalues>
      </node>
      <node id="3" label="BarabasiLab">
        <attvalues>
          <attvalue for="0" value="http://barabasilab.com"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="true"/>
        </attvalues>
      </node>
    </nodes>
    <edges>
      <edge id="0" source="0" target="1"/>
      <edge id="1" source="0" target="2"/>
      <edge id="2" source="1" target="0"/>
      <edge id="3" source="2" target="1"/>
      <edge id="4" source="0" target="3"/>
    </edges>
  </graph>
</gexf>
"""
        self.attribute_graph = nx.DiGraph()
        self.attribute_graph.graph['node_default'] = {'frog': True}
        self.attribute_graph.add_node('0',
                                      label='Gephi',
                                      url='http://gephi.org',
                                      indegree=1, frog=False)
        self.attribute_graph.add_node('1',
                                      label='Webatlas',
                                      url='http://webatlas.fr',
                                      indegree=2, frog=False)
        self.attribute_graph.add_node('2',
                                      label='RTGI',
                                      url='http://rtgi.fr',
                                      indegree=1, frog=True)
        self.attribute_graph.add_node('3',
                                      label='BarabasiLab',
                                      url='http://barabasilab.com',
                                      indegree=1, frog=True)
        self.attribute_graph.add_edge('0', '1', id='0')
        self.attribute_graph.add_edge('0', '2', id='1')
        self.attribute_graph.add_edge('1', '0', id='2')
        self.attribute_graph.add_edge('2', '1', id='3')
        self.attribute_graph.add_edge('0', '3', id='4')
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data = """<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_undirected_graph = nx.Graph()
        self.simple_undirected_graph.add_node('0', label='Hello')
        self.simple_undirected_graph.add_node('1', label='World')
        self.simple_undirected_graph.add_edge('0', '1', id='0')

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 17

Project: networkx
Source File: test_graphml.py
View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<!-- This file was written by the JAVA GraphML Library.-->
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G" edgedefault="directed">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n3"/>
    <node id="n4"/>
    <node id="n5"/>
    <node id="n6"/>
    <node id="n7"/>
    <node id="n8"/>
    <node id="n9"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
    <edge source="n3" target="n5"/>
    <edge source="n3" target="n4"/>
    <edge source="n4" target="n6"/>
    <edge source="n6" target="n5"/>
    <edge source="n5" target="n7"/>
    <edge source="n6" target="n8"/>
    <edge source="n8" target="n7"/>
    <edge source="n8" target="n9"/>
  </graph>
</graphml>"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('n10')
        self.simple_directed_graph.add_edge('n0','n2',id='foo')
        self.simple_directed_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ('n3','n5'),
                                                   ('n3','n4'),
                                                   ('n4','n6'),
                                                   ('n6','n5'),
                                                   ('n5','n7'),
                                                   ('n6','n8'),
                                                   ('n8','n7'),
                                                   ('n8','n9'),
                                                   ])

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
        http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <key id="d0" for="node" attr.name="color" attr.type="string">
    <default>yellow</default>
  </key>
  <key id="d1" for="edge" attr.name="weight" attr.type="double"/>
  <graph id="G" edgedefault="directed">
    <node id="n0">
      <data key="d0">green</data>
    </node>
    <node id="n1"/>
    <node id="n2">
      <data key="d0">blue</data>
    </node>
    <node id="n3">
      <data key="d0">red</data>
    </node>
    <node id="n4"/>
    <node id="n5">
      <data key="d0">turquoise</data>
    </node>
    <edge id="e0" source="n0" target="n2">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e1" source="n0" target="n1">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e2" source="n1" target="n3">
      <data key="d1">2.0</data>
    </edge>
    <edge id="e3" source="n3" target="n2"/>
    <edge id="e4" source="n2" target="n4"/>
    <edge id="e5" source="n3" target="n5"/>
    <edge id="e6" source="n5" target="n4">
      <data key="d1">1.1</data>
    </edge>
  </graph>
</graphml>
"""
        self.attribute_graph=nx.DiGraph(id='G')
        self.attribute_graph.graph['node_default']={'color':'yellow'}
        self.attribute_graph.add_node('n0',color='green')
        self.attribute_graph.add_node('n2',color='blue')
        self.attribute_graph.add_node('n3',color='red')
        self.attribute_graph.add_node('n4')
        self.attribute_graph.add_node('n5',color='turquoise')
        self.attribute_graph.add_edge('n0','n2',id='e0',weight=1.0)
        self.attribute_graph.add_edge('n0','n1',id='e1',weight=1.0)
        self.attribute_graph.add_edge('n1','n3',id='e2',weight=2.0)
        self.attribute_graph.add_edge('n3','n2',id='e3')
        self.attribute_graph.add_edge('n2','n4',id='e4')
        self.attribute_graph.add_edge('n3','n5',id='e5')
        self.attribute_graph.add_edge('n5','n4',id='e6',weight=1.1)
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.attribute_numeric_type_data = """<?xml version='1.0' encoding='utf-8'?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <key attr.name="weight" attr.type="double" for="node" id="d1" />
  <key attr.name="weight" attr.type="double" for="edge" id="d0" />
  <graph edgedefault="directed">
    <node id="n0">
      <data key="d1">1</data>
    </node>
    <node id="n1">
      <data key="d1">2.0</data>
    </node>
    <edge source="n0" target="n1">
      <data key="d0">1</data>
    </edge>
    <edge source="n1" target="n1">
      <data key="d0">1.0</data>
    </edge>
  </graph>
</graphml>
"""

        self.attribute_numeric_type_graph = nx.DiGraph()
        self.attribute_numeric_type_graph.add_node('n0', weight=1)
        self.attribute_numeric_type_graph.add_node('n1', weight=2.0)
        self.attribute_numeric_type_graph.add_edge('n0', 'n1', weight=1)
        self.attribute_numeric_type_graph.add_edge('n1', 'n1', weight=1.0)
        self.attribute_numeric_type_fh = io.BytesIO(self.attribute_numeric_type_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
  </graph>
</graphml>"""
#    <edge source="n8" target="n10" directed="false"/>
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('n10')
        self.simple_undirected_graph.add_edge('n0','n2',id='foo')
        self.simple_undirected_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ])

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 18

Project: networkx
Source File: test_gexf.py
View license
    def setUp(self):
        self.simple_directed_data = """<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="directed">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_directed_graph = nx.DiGraph()
        self.simple_directed_graph.add_node('0', label='Hello')
        self.simple_directed_graph.add_node('1', label='World')
        self.simple_directed_graph.add_edge('0', '1', id='0')

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))

        self.attribute_data = """<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.gexf.net/1.1draft http://www.gexf.net/1.1draft/gexf.xsd" version="1.1">
  <meta lastmodifieddate="2009-03-20">
    <creator>Gephi.org</creator>
    <description>A Web network</description>
  </meta>
  <graph defaultedgetype="directed">
    <attributes class="node">
      <attribute id="0" title="url" type="string"/>
      <attribute id="1" title="indegree" type="integer"/>
      <attribute id="2" title="frog" type="boolean">
        <default>true</default>
      </attribute>
    </attributes>
    <nodes>
      <node id="0" label="Gephi">
        <attvalues>
          <attvalue for="0" value="http://gephi.org"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
      <node id="1" label="Webatlas">
        <attvalues>
          <attvalue for="0" value="http://webatlas.fr"/>
          <attvalue for="1" value="2"/>
          <attvalue for="2" value="false"/>
        </attvalues>
      </node>
      <node id="2" label="RTGI">
        <attvalues>
          <attvalue for="0" value="http://rtgi.fr"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="true"/>
        </attvalues>
      </node>
      <node id="3" label="BarabasiLab">
        <attvalues>
          <attvalue for="0" value="http://barabasilab.com"/>
          <attvalue for="1" value="1"/>
          <attvalue for="2" value="true"/>
        </attvalues>
      </node>
    </nodes>
    <edges>
      <edge id="0" source="0" target="1"/>
      <edge id="1" source="0" target="2"/>
      <edge id="2" source="1" target="0"/>
      <edge id="3" source="2" target="1"/>
      <edge id="4" source="0" target="3"/>
    </edges>
  </graph>
</gexf>
"""
        self.attribute_graph = nx.DiGraph()
        self.attribute_graph.graph['node_default'] = {'frog': True}
        self.attribute_graph.add_node('0',
                                      label='Gephi',
                                      url='http://gephi.org',
                                      indegree=1, frog=False)
        self.attribute_graph.add_node('1',
                                      label='Webatlas',
                                      url='http://webatlas.fr',
                                      indegree=2, frog=False)
        self.attribute_graph.add_node('2',
                                      label='RTGI',
                                      url='http://rtgi.fr',
                                      indegree=1, frog=True)
        self.attribute_graph.add_node('3',
                                      label='BarabasiLab',
                                      url='http://barabasilab.com',
                                      indegree=1, frog=True)
        self.attribute_graph.add_edge('0', '1', id='0')
        self.attribute_graph.add_edge('0', '2', id='1')
        self.attribute_graph.add_edge('1', '0', id='2')
        self.attribute_graph.add_edge('2', '1', id='3')
        self.attribute_graph.add_edge('0', '3', id='4')
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.simple_undirected_data = """<?xml version="1.0" encoding="UTF-8"?>
<gexf xmlns="http://www.gexf.net/1.1draft" version="1.1">
    <graph mode="static" defaultedgetype="undirected">
        <nodes>
            <node id="0" label="Hello" />
            <node id="1" label="Word" />
        </nodes>
        <edges>
            <edge id="0" source="0" target="1" />
        </edges>
    </graph>
</gexf>
"""
        self.simple_undirected_graph = nx.Graph()
        self.simple_undirected_graph.add_node('0', label='Hello')
        self.simple_undirected_graph.add_node('1', label='World')
        self.simple_undirected_graph.add_edge('0', '1', id='0')

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 19

Project: networkx
Source File: test_graphml.py
View license
    def setUp(self):
        self.simple_directed_data="""<?xml version="1.0" encoding="UTF-8"?>
<!-- This file was written by the JAVA GraphML Library.-->
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G" edgedefault="directed">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n3"/>
    <node id="n4"/>
    <node id="n5"/>
    <node id="n6"/>
    <node id="n7"/>
    <node id="n8"/>
    <node id="n9"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
    <edge source="n3" target="n5"/>
    <edge source="n3" target="n4"/>
    <edge source="n4" target="n6"/>
    <edge source="n6" target="n5"/>
    <edge source="n5" target="n7"/>
    <edge source="n6" target="n8"/>
    <edge source="n8" target="n7"/>
    <edge source="n8" target="n9"/>
  </graph>
</graphml>"""
        self.simple_directed_graph=nx.DiGraph()
        self.simple_directed_graph.add_node('n10')
        self.simple_directed_graph.add_edge('n0','n2',id='foo')
        self.simple_directed_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ('n3','n5'),
                                                   ('n3','n4'),
                                                   ('n4','n6'),
                                                   ('n6','n5'),
                                                   ('n5','n7'),
                                                   ('n6','n8'),
                                                   ('n8','n7'),
                                                   ('n8','n9'),
                                                   ])

        self.simple_directed_fh = \
            io.BytesIO(self.simple_directed_data.encode('UTF-8'))


        self.attribute_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns
        http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <key id="d0" for="node" attr.name="color" attr.type="string">
    <default>yellow</default>
  </key>
  <key id="d1" for="edge" attr.name="weight" attr.type="double"/>
  <graph id="G" edgedefault="directed">
    <node id="n0">
      <data key="d0">green</data>
    </node>
    <node id="n1"/>
    <node id="n2">
      <data key="d0">blue</data>
    </node>
    <node id="n3">
      <data key="d0">red</data>
    </node>
    <node id="n4"/>
    <node id="n5">
      <data key="d0">turquoise</data>
    </node>
    <edge id="e0" source="n0" target="n2">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e1" source="n0" target="n1">
      <data key="d1">1.0</data>
    </edge>
    <edge id="e2" source="n1" target="n3">
      <data key="d1">2.0</data>
    </edge>
    <edge id="e3" source="n3" target="n2"/>
    <edge id="e4" source="n2" target="n4"/>
    <edge id="e5" source="n3" target="n5"/>
    <edge id="e6" source="n5" target="n4">
      <data key="d1">1.1</data>
    </edge>
  </graph>
</graphml>
"""
        self.attribute_graph=nx.DiGraph(id='G')
        self.attribute_graph.graph['node_default']={'color':'yellow'}
        self.attribute_graph.add_node('n0',color='green')
        self.attribute_graph.add_node('n2',color='blue')
        self.attribute_graph.add_node('n3',color='red')
        self.attribute_graph.add_node('n4')
        self.attribute_graph.add_node('n5',color='turquoise')
        self.attribute_graph.add_edge('n0','n2',id='e0',weight=1.0)
        self.attribute_graph.add_edge('n0','n1',id='e1',weight=1.0)
        self.attribute_graph.add_edge('n1','n3',id='e2',weight=2.0)
        self.attribute_graph.add_edge('n3','n2',id='e3')
        self.attribute_graph.add_edge('n2','n4',id='e4')
        self.attribute_graph.add_edge('n3','n5',id='e5')
        self.attribute_graph.add_edge('n5','n4',id='e6',weight=1.1)
        self.attribute_fh = io.BytesIO(self.attribute_data.encode('UTF-8'))

        self.attribute_numeric_type_data = """<?xml version='1.0' encoding='utf-8'?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <key attr.name="weight" attr.type="double" for="node" id="d1" />
  <key attr.name="weight" attr.type="double" for="edge" id="d0" />
  <graph edgedefault="directed">
    <node id="n0">
      <data key="d1">1</data>
    </node>
    <node id="n1">
      <data key="d1">2.0</data>
    </node>
    <edge source="n0" target="n1">
      <data key="d0">1</data>
    </edge>
    <edge source="n1" target="n1">
      <data key="d0">1.0</data>
    </edge>
  </graph>
</graphml>
"""

        self.attribute_numeric_type_graph = nx.DiGraph()
        self.attribute_numeric_type_graph.add_node('n0', weight=1)
        self.attribute_numeric_type_graph.add_node('n1', weight=2.0)
        self.attribute_numeric_type_graph.add_edge('n0', 'n1', weight=1)
        self.attribute_numeric_type_graph.add_edge('n1', 'n1', weight=1.0)
        self.attribute_numeric_type_fh = io.BytesIO(self.attribute_numeric_type_data.encode('UTF-8'))

        self.simple_undirected_data="""<?xml version="1.0" encoding="UTF-8"?>
<graphml xmlns="http://graphml.graphdrawing.org/xmlns"  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
  <graph id="G">
    <node id="n0"/>
    <node id="n1"/>
    <node id="n2"/>
    <node id="n10"/>
    <edge id="foo" source="n0" target="n2"/>
    <edge source="n1" target="n2"/>
    <edge source="n2" target="n3"/>
  </graph>
</graphml>"""
#    <edge source="n8" target="n10" directed="false"/>
        self.simple_undirected_graph=nx.Graph()
        self.simple_undirected_graph.add_node('n10')
        self.simple_undirected_graph.add_edge('n0','n2',id='foo')
        self.simple_undirected_graph.add_edges_from([('n1','n2'),
                                                   ('n2','n3'),
                                                   ])

        self.simple_undirected_fh = io.BytesIO(self.simple_undirected_data.encode('UTF-8'))

Example 20

Project: odoorpc
Source File: db.py
View license
    def dump(self, password, db, format_='zip'):
        """Backup the `db` database. Returns the dump as a binary ZIP file
        containing the SQL dump file alongside the filestore directory (if any).

        >>> dump = odoo.db.dump('super_admin_passwd', 'prod') # doctest: +SKIP

        .. doctest::
            :hide:

            >>> dump = odoo.db.dump(SUPER_PWD, DB)

        If you get a timeout error, increase this one before performing the
        request:

        >>> timeout_backup = odoo.config['timeout']
        >>> odoo.config['timeout'] = 600    # Timeout set to 10 minutes
        >>> dump = odoo.db.dump('super_admin_passwd', 'prod')   # doctest: +SKIP
        >>> odoo.config['timeout'] = timeout_backup

        Write it on the file system:

        .. doctest::
            :options: +SKIP

            >>> with open('dump.zip', 'wb') as dump_zip:
            ...     dump_zip.write(dump.read())
            ...

        .. doctest::
            :hide:

            >>> with open('dump.zip', 'wb') as dump_zip:
            ...     fileno = dump_zip.write(dump.read())    # Python 3
            ...

        You can manipulate the file with the `zipfile` module for instance:

        .. doctest::
            :options: +SKIP

            >>> import zipfile
            >>> zipfile.ZipFile('dump.zip').namelist()
            ['dump.sql',
            'filestore/ef/ef2c882a36dbe90fc1e7e28d816ad1ac1464cfbb',
            'filestore/dc/dcf00aacce882bbfd117c0277e514f829b4c5bf0',
             ...]

        .. doctest::
            :hide:

            >>> import zipfile
            >>> zipfile.ZipFile('dump.zip').namelist() # doctest: +NORMALIZE_WHITESPACE
            ['dump.sql'...'filestore/...'...]

        The super administrator password is required to perform this method.

        *Python 2:*

        :return: `io.BytesIO`
        :raise: :class:`odoorpc.error.RPCError` (access denied / wrong database)
        :raise: `urllib2.URLError` (connection error)

        *Python 3:*

        :return: `io.BytesIO`
        :raise: :class:`odoorpc.error.RPCError` (access denied / wrong database)
        :raise: `urllib.error.URLError` (connection error)
        """
        args = [password, db]
        if v(self._odoo.version)[0] >= 9:
            args.append(format_)
        data = self._odoo.json(
            '/jsonrpc',
            {'service': 'db',
             'method': 'dump',
             'args': args})
        # Encode to bytes forced to be compatible with Python 3.2
        # (its 'base64.standard_b64decode()' function only accepts bytes)
        result = encode2bytes(data['result'])
        content = base64.standard_b64decode(result)
        return io.BytesIO(content)

Example 21

Project: pjuu
Source File: uploads.py
View license
def process_upload(upload, collection='uploads', image_size=(1280, 720),
                   thumbnail=True):
    """Processes the uploaded images in the posts and also the users avatars.
    This should be extensible in future to support the uploading of videos,
    audio, etc...

    :param _id: The ID for the post, user or anything you want as the filename
    :type _id: str
    :param upload: The uploaded Werkzeug FileStorage object
    :type upload: ``Werkzeug.datastructures.FileStorage``
    :param collection: The GridFS collection to upload the file too.
    :type collection: str
    :param image_size: The max height and width for the upload
    :type image_size: Tuple length 2 of int
    :param thumbnail: Is the image to have it's aspect ration kept?
    :type thumbnail: bool
    """
    try:
        # StringIO to take the uploaded image to transport to GridFS
        output = io.BytesIO()

        # All images are passed through Wand and turned in to PNG files.
        # Unless the image is a GIF and its format is kept
        # Will change if they need thumbnailing or resizing.
        img = Image(file=upload)

        # If the input file if a GIF then we need to know
        gif = True if img.format == 'GIF' else False
        animated_gif = True if gif and len(img.sequence) > 1 else False

        # Check the exif data.
        # If there is an orientation then transform the image so that
        # it is always looking up.
        try:
            exif_data = {
                k[5:]: v
                for k, v in img.metadata.items()
                if k.startswith('exif:')
            }

            orientation = exif_data.get('Orientation')

            orientation = int(orientation)

            if orientation:  # pragma: no branch
                if orientation == 2:
                    img.flop()
                elif orientation == 3:
                    img.rotate(180)
                elif orientation == 4:
                    img.flip()
                elif orientation == 5:
                    img.flip()
                    img.rotate(90)
                elif orientation == 6:
                    img.rotate(90)
                elif orientation == 7:
                    img.flip()
                    img.rotate(270)
                elif orientation == 8:
                    img.rotate(270)

        except (AttributeError, TypeError, AttributeError):
            pass

        if thumbnail:
            # If the GIF was known to be animated then save the animated
            # then cycle through the frames, transforming them and save the
            # output
            if animated_gif:
                animated_image = Image()
                for frame in img.sequence:
                    frame.transform(resize='{0}x{1}>'.format(*image_size))
                    # Directly append the frame to the output image
                    animated_image.sequence.append(frame)

                animated_output = io.BytesIO()
                animated_output.format = 'GIF'
                animated_image.save(file=animated_output)
                animated_output.seek(0)

            img.transform(resize='{0}x{1}>'.format(*image_size))
        else:
            # Just sample the image to the correct size
            img.sample(*image_size)
            # Turn off animated GIF
            animated_gif = False

        img.format = 'PNG'
        uuid = get_uuid()
        filename = '{0}.{1}'.format(uuid, 'png')

        # Return the file pointer to the start
        img.save(file=output)
        output.seek(0)

        # Place file inside GridFS
        m.save_file(filename, output, base=collection)

        animated_filename = ''
        if animated_gif:
            animated_filename = '{0}.{1}'.format(uuid, 'gif')
            m.save_file(animated_filename, animated_output, base=collection)

        return filename, animated_filename
    except (IOError, MissingDelegateError):
        # File will not have been uploaded
        return None, None

Example 22

Project: diazo
Source File: test_diazo.py
View license
    def testAll(self):
        self.errors = BytesIO()
        config = configparser.ConfigParser()
        config.read([defaultsfn, os.path.join(self.testdir, "options.cfg")])

        themefn = None
        if config.get('diazotest', 'theme'):
            themefn = os.path.join(self.testdir, config.get('diazotest',
                                                            'theme'))
        contentfn = os.path.join(self.testdir, "content.html")
        rulesfn = os.path.join(self.testdir, "rules.xml")
        xpathsfn = os.path.join(self.testdir, "xpaths.txt")
        xslfn = os.path.join(self.testdir, "compiled.xsl")
        outputfn = os.path.join(self.testdir, "output.html")

        xsl_params = {}
        extra_params = config.get('diazotest', 'extra-params')
        if extra_params:
            for token in extra_params.split(' '):
                token_split = token.split(':')
                xsl_params[token_split[0]] = len(token_split) > 1 and \
                    token_split[1] or None

        if not os.path.exists(rulesfn):
            return

        contentdoc = etree.parse(source=contentfn, base_url=contentfn,
                                 parser=etree.HTMLParser())

        # Make a compiled version
        theme_parser = etree.HTMLParser()
        ct = diazo.compiler.compile_theme(
            rules=rulesfn,
            theme=themefn,
            parser=theme_parser,
            absolute_prefix=config.get('diazotest', 'absolute-prefix'),
            indent=config.getboolean('diazotest', 'pretty-print'),
            xsl_params=xsl_params,
        )

        # Serialize / parse the theme - this can catch problems with escaping.
        cts = etree.tostring(ct, encoding='unicode')
        parser = etree.XMLParser()
        etree.fromstring(cts, parser=parser)

        # Compare to previous version
        if os.path.exists(xslfn):
            with open(xslfn) as f:
                old = f.read()
            new = cts
            if old != new:
                if self.writefiles:
                    with open(xslfn + '.old', 'w') as f:
                        f.write(old)
                if self.warnings:
                    print("WARNING:", "compiled.xsl has CHANGED")
                    for line in difflib.unified_diff(old.split(u'\n'),
                                                     new.split(u'\n'),
                                                     xslfn, 'now'):
                        print(line)

        # Write the compiled xsl out to catch unexpected changes
        if self.writefiles:
            with open(xslfn, 'w') as f:
                f.write(cts)

        # Apply the compiled version, then test against desired output
        theme_parser.resolvers.add(diazo.run.RunResolver(self.testdir))
        processor = etree.XSLT(ct)
        params = {}
        params['path'] = "'%s'" % config.get('diazotest', 'path')

        for key in xsl_params:
            try:
                params[key] = quote_param(config.get('diazotest', key))
            except configparser.NoOptionError:
                pass

        result = processor(contentdoc, **params)

        # Read the whole thing to strip off xhtml namespace.
        # If we had xslt 2.0 then we could use xpath-default-namespace.
        self.themed_string = str(result)
        self.themed_content = etree.ElementTree(
            file=StringIO(self.themed_string), parser=etree.HTMLParser())

        # remove the extra meta content type

        metas = self.themed_content.xpath(
            "/html/head/meta[@http-equiv='Content-Type']")
        if metas:
            meta = metas[0]
            meta.getparent().remove(meta)

        if os.path.exists(xpathsfn):
            with open(xpathsfn) as f:
                for xpath in f.readlines():
                    # Read the XPaths from the file, skipping blank lines and
                    # comments
                    this_xpath = xpath.strip()
                    if not this_xpath or this_xpath[0] == '#':
                        continue
                    assert self.themed_content.xpath(this_xpath), "%s: %s" % (
                        xpathsfn, this_xpath)

        # Compare to previous version
        if os.path.exists(outputfn):
            with open(outputfn) as f:
                old = f.read()
            new = self.themed_string
            if not xml_compare(
                    etree.fromstring(old.strip()),
                    etree.fromstring(new.strip())):
                # if self.writefiles:
                #    open(outputfn + '.old', 'w').write(old)
                for line in difflib.unified_diff(old.split(u'\n'),
                                                 new.split(u'\n'),
                                                 outputfn, 'now'):
                    print(line)
                assert old == new, "output.html has CHANGED"

        # Write out the result to catch unexpected changes
        if self.writefiles:
            with open(outputfn, 'w') as f:
                f.write(self.themed_string)

Example 23

Project: powerline
Source File: run_script.py
View license
def main():
	parser = get_argparser()
	args = parser.parse_args()

	shell = args.shell or args.command[0]
	test_type = args.type or shell
	test_client = args.client or test_type

	log_file_base = '{0}.{1}.{2}'.format(shell, test_type, test_client)
	full_log_file_name = os.path.join('tests', 'shell', '{0}.full.log'.format(log_file_base))
	# postproc_log_file_name = os.path.join('tests', 'shell', '{0}.log'.format(log_file_base))

	local_paths = [
		os.path.abspath(os.path.join('tests', 'shell', 'path')),
		os.path.abspath('scripts'),
	]

	if test_type == 'fish':
		local_paths += ['/usr/bin', '/bin']

	python_paths = os.environ.get('PYTHONPATH', '')
	if python_paths:
		python_paths = ':' + python_paths
	python_paths = os.path.abspath('.') + python_paths

	environ = {
		'LANG': 'en_US.UTF-8',
		'PATH': os.pathsep.join(local_paths),
		'TERM': 'screen-256color',
		'DIR1': os.environ['DIR1'],
		'DIR2': os.environ['DIR2'],
		'XDG_CONFIG_HOME': os.path.abspath(os.path.join('tests', 'shell', 'fish_home')),
		'IPYTHONDIR': os.path.abspath(os.path.join('tests', 'shell', 'ipython_home')),
		'PYTHONPATH': python_paths,
		'POWERLINE_CONFIG_OVERRIDES': os.environ.get('POWERLINE_CONFIG_OVERRIDES', ''),
		'POWERLINE_THEME_OVERRIDES': os.environ.get('POWERLINE_THEME_OVERRIDES', ''),
		'POWERLINE_CONFIG_PATHS': os.path.abspath(os.path.join('powerline', 'config_files')),
		'POWERLINE_COMMAND_ARGS': os.environ.get('POWERLINE_COMMAND_ARGS', ''),
		'POWERLINE_COMMAND': os.environ.get('POWERLINE_COMMAND', ''),
		'LD_LIBRARY_PATH': os.environ.get('LD_LIBRARY_PATH', ''),
	}

	os.environ['PATH'] = environ['PATH']

	if test_type == 'daemon':
		environ['POWERLINE_SHELL_CONTINUATION'] = '1'
		environ['POWERLINE_SHELL_SELECT'] = '1'

	if test_type != 'zpython' and shell == 'zsh':
		environ['POWERLINE_NO_ZSH_ZPYTHON'] = '1'

	sio = BytesIO()

	child = pexpect.spawn(
		args.command[0],
		args.command[1:],
		env=environ,
		logfile=sio,
		timeout=30,
	)
	child.expect(re.compile(b'.*'))
	sleep(0.5)
	child.setwinsize(1, 300)

	with open(os.path.join('tests', 'test_shells', 'input.{0}'.format(shell)), 'rb') as F:
		if not args.wait_for_echo:
			child.send(F.read())
		else:
			for line in F:
				child.send(line)
				sleep(1)
				# TODO Implement something more smart

	with open(full_log_file_name, 'wb') as LF:
		while True:
			try:
				s = child.read_nonblocking(1000)
			except pexpect.TIMEOUT:
				break
			except pexpect.EOF:
				break
			else:
				LF.write(s)

	child.close(force=True)

	check_call([
		os.path.join('tests', 'shell', 'path', 'python'),
		os.path.join('tests', 'test_shells', 'postproc.py'),
		test_type, test_client, shell
	])
	pidfile = os.path.join('tests', 'shell', '3rd', 'pid')
	if os.path.exists(pidfile):
		os.unlink(pidfile)

Example 24

Project: pretix
Source File: ticketoutput.py
View license
    def generate(self, op):
        from reportlab.graphics.shapes import Drawing
        from reportlab.pdfgen import canvas
        from reportlab.lib import pagesizes, units
        from reportlab.graphics.barcode.qr import QrCodeWidget
        from reportlab.graphics import renderPDF
        from PyPDF2 import PdfFileWriter, PdfFileReader

        order = op.order

        pagesize = self.settings.get('pagesize', default='A4')
        if hasattr(pagesizes, pagesize):
            pagesize = getattr(pagesizes, pagesize)
        else:
            pagesize = pagesizes.A4
        orientation = self.settings.get('orientation', default='portrait')
        if hasattr(pagesizes, orientation):
            pagesize = getattr(pagesizes, orientation)(pagesize)

        buffer = BytesIO()
        p = canvas.Canvas(buffer, pagesize=pagesize)

        event_s = self.settings.get('event_s', default=22, as_type=float)
        if event_s:
            p.setFont("Helvetica", event_s)
            event_x = self.settings.get('event_x', default=15, as_type=float)
            event_y = self.settings.get('event_y', default=235, as_type=float)
            p.drawString(event_x * units.mm, event_y * units.mm, str(self.event.name))

        order_s = self.settings.get('order_s', default=17, as_type=float)
        if order_s:
            p.setFont("Helvetica", order_s)
            order_x = self.settings.get('order_x', default=15, as_type=float)
            order_y = self.settings.get('order_y', default=220, as_type=float)
            p.drawString(order_x * units.mm, order_y * units.mm, _('Order code: {code}').format(code=order.code))

        name_s = self.settings.get('name_s', default=17, as_type=float)
        if name_s:
            p.setFont("Helvetica", name_s)
            name_x = self.settings.get('name_x', default=15, as_type=float)
            name_y = self.settings.get('name_y', default=210, as_type=float)
            item = str(op.item.name)
            if op.variation:
                item += " – " + str(op.variation)
            p.drawString(name_x * units.mm, name_y * units.mm, item)

        price_s = self.settings.get('price_s', default=17, as_type=float)
        if price_s:
            p.setFont("Helvetica", price_s)
            price_x = self.settings.get('price_x', default=15, as_type=float)
            price_y = self.settings.get('price_y', default=200, as_type=float)
            p.drawString(price_x * units.mm, price_y * units.mm, "%s %s" % (str(op.price), self.event.currency))

        qr_s = self.settings.get('qr_s', default=80, as_type=float)
        if qr_s:
            reqs = qr_s * units.mm
            qrw = QrCodeWidget(op.secret, barLevel='H')
            b = qrw.getBounds()
            w = b[2] - b[0]
            h = b[3] - b[1]
            d = Drawing(reqs, reqs, transform=[reqs / w, 0, 0, reqs / h, 0, 0])
            d.add(qrw)
            qr_x = self.settings.get('qr_x', default=10, as_type=float)
            qr_y = self.settings.get('qr_y', default=120, as_type=float)
            renderPDF.draw(d, p, qr_x * units.mm, qr_y * units.mm)

        code_s = self.settings.get('code_s', default=11, as_type=float)
        if code_s:
            p.setFont("Helvetica", code_s)
            code_x = self.settings.get('code_x', default=15, as_type=float)
            code_y = self.settings.get('code_y', default=120, as_type=float)
            p.drawString(code_x * units.mm, code_y * units.mm, op.secret)

        attendee_s = self.settings.get('attendee_s', default=0, as_type=float)
        if code_s and op.attendee_name:
            p.setFont("Helvetica", attendee_s)
            attendee_x = self.settings.get('attendee_x', default=15, as_type=float)
            attendee_y = self.settings.get('attendee_y', default=90, as_type=float)
            p.drawString(attendee_x * units.mm, attendee_y * units.mm, op.attendee_name)

        p.showPage()

        p.save()

        buffer.seek(0)
        new_pdf = PdfFileReader(buffer)
        output = PdfFileWriter()
        bg_file = self.settings.get('background', as_type=File)
        if isinstance(bg_file, File):
            bgf = default_storage.open(bg_file.name, "rb")
        else:
            bgf = open(finders.find('pretixpresale/pdf/ticket_default_a4.pdf'), "rb")
        bg_pdf = PdfFileReader(bgf)
        for page in new_pdf.pages:
            bg_page = copy.copy(bg_pdf.getPage(0))
            bg_page.mergePage(page)
            output.addPage(bg_page)

        outbuffer = BytesIO()
        output.write(outbuffer)
        outbuffer.seek(0)
        return 'order%s%s.pdf' % (self.event.slug, order.code), 'application/pdf', outbuffer.read()

Example 25

Project: seisflows
Source File: endian.py
View license
def endian(file):
    """
    Takes an open file and tries to determine the endianness of a Seismic
    Unix data file by doing some sanity checks with the unpacked header values.

    Returns False if the sanity checks failed and the endianness otherwise.

    It is assumed that the data is written as 32bit IEEE floating points in
    either little or big endian.

    The test currently can only identify SU files in which all traces have the
    same length. It basically just makes a sanity check for various fields in
    the Trace header.
    """
    pos = file.tell()
    if isinstance(file, io.BytesIO):
        file.seek(0, 2)
        size = file.tell()
        file.seek(pos, 0)
    else:
        size = os.fstat(file.fileno())[6]
    if size < 244:
        return False
    # Also has to be a multiple of 4 in length because every header is 400 long
    # and every data value 4 byte long.
    elif (size % 4) != 0:
        return False
    # Jump to the number of samples field in the trace header.
    file.seek(114, 0)
    sample_count = file.read(2)
    interval = file.read(2)
    # Jump to the beginning of the year fields.
    file.seek(156, 0)
    year = file.read(2)
    jul_day = file.read(2)
    hour = file.read(2)
    minute = file.read(2)
    second = file.read(2)
    # Jump to previous position.
    file.seek(pos, 0)
    # Unpack in little and big endian.
    le_sample_count = unpack(b'<h', sample_count)[0]
    be_sample_count = unpack(b'>h', sample_count)[0]
    # Check if both work.
    working_byteorders = []
    if le_sample_count > 0:
        length = 240 + (le_sample_count * 4)
        if (size % length) == 0:
            working_byteorders.append('<')
    if be_sample_count > 0:
        length = 240 + (be_sample_count * 4)
        if (size % length) == 0:
            working_byteorders.append('>')
    # If None works return False.
    if len(working_byteorders) == 0:
        return False
    # Check if the other header values make sense.
    still_working_byteorders = []
    for bo in working_byteorders:
        fmt = ("%sh" % bo).encode('ascii', 'strict')
        this_interval = unpack(fmt, interval)[0]
        this_year = unpack(fmt, year)[0]
        this_julday = unpack(fmt, jul_day)[0]
        this_hour = unpack(fmt, hour)[0]
        this_minute = unpack(fmt, minute)[0]
        this_second = unpack(fmt, second)[0]
        # Make a sanity check for each.
        # XXX: The arbitrary maximum of the sample interval is 10 seconds.
        if this_interval <= 0 or this_interval > 10E7:
            continue
        # Some programs write two digit years.
        if this_year != 0 and (this_year < 1930 or this_year >= 2030) and \
                (this_year < 0 or this_year >= 100):
            continue
        # 9999 is often used as a placeholder
        if (this_julday > 366 or this_julday < 0) and this_julday != 9999:
            continue
        if this_hour > 24 or this_hour < 0:
            continue
        if this_minute > 60 or this_minute < 0:
            continue
        if this_second > 60 or this_second < 0:
            continue
        still_working_byteorders.append(bo)
    length = len(still_working_byteorders)
    if not length:
        return False
    elif length == 1:
        return still_working_byteorders[0]
    else:
        # XXX: In the unlikely case both byte orders pass the sanity checks
        # something else should be checked. Currently it is not.
        msg = """
            Both possible byte orders passed all sanity checks. Please contact
            the ObsPy developers so they can implement additional tests.
            """.strip()
        raise Exception(msg)

Example 26

Project: osbs-client
Source File: http.py
View license
    def __init__(self, url, method, data=None, kerberos_auth=False,
                 allow_redirects=True, verify_ssl=True, ca=None, use_json=False,
                 headers=None, stream=False, username=None, password=None,
                 client_cert=None, client_key=None, verbose=False):
        self.finished = False  # have we read all data?
        self.closed = False    # have we destroyed curl resources?

        self.status_code = 0
        self.headers = None
        self.response_buffer = BytesIO()
        self.headers_buffer = BytesIO()
        self.response_decoder = None

        self.url = url
        headers = headers or {}
        method = method.lower()

        self.c = pycurl.Curl()
        self.curl_multi = pycurl.CurlMulti()

        if method == 'post':
            self.c.setopt(pycurl.POST, 1)
            headers["Expect"] = ""  # openshift can't handle Expect
        elif method == 'get':
            self.c.setopt(pycurl.HTTPGET, 1)
        elif method == 'put':
            # self.c.setopt(pycurl.PUT, 1)
            self.c.setopt(pycurl.CUSTOMREQUEST, b"PUT")
            headers["Expect"] = ""
        elif method == 'delete':
            self.c.setopt(pycurl.CUSTOMREQUEST, b"DELETE")
        else:
            raise RuntimeError("Unsupported method '%s' for curl call!" % method)

        self.c.setopt(pycurl.COOKIEFILE, b'')
        self.c.setopt(pycurl.URL, str(url))
        self.c.setopt(pycurl.WRITEFUNCTION, self.response_buffer.write)
        self.c.setopt(pycurl.HEADERFUNCTION, self.headers_buffer.write)
        self.c.setopt(pycurl.DEBUGFUNCTION, self._curl_debug)
        self.c.setopt(pycurl.SSL_VERIFYPEER, 1 if verify_ssl else 0)
        self.c.setopt(pycurl.SSL_VERIFYHOST, 2 if verify_ssl else 0)
        if ca:
            logger.info("Setting CAINFO to %r", ca)
            self.c.setopt(pycurl.CAINFO, ca)

        self.c.setopt(pycurl.VERBOSE, 1 if verbose else 0)
        if username and password:
            username = username.encode('utf-8')
            password = password.encode('utf-8')
            self.c.setopt(pycurl.USERPWD, username + b":" + password)

        if client_cert and client_key:
            self.c.setopt(pycurl.SSLCERTTYPE, "PEM")
            self.c.setopt(pycurl.SSLKEYTYPE, "PEM")
            self.c.setopt(pycurl.SSLCERT, client_cert)
            self.c.setopt(pycurl.SSLKEY, client_key)

        if data:
            # curl sets the method to post if one sets any POSTFIELDS (even '')
            self.c.setopt(pycurl.POSTFIELDS, data)

        if use_json:
            headers['Content-Type'] = b'application/json'

        if allow_redirects:
            self.c.setopt(pycurl.FOLLOWLOCATION, 1)

        if kerberos_auth:
            self.c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
            self.c.setopt(pycurl.USERPWD, b':')

        if stream:
            headers['Cache-Control'] = b'no-cache'

        if headers:
            header_list = []
            for header_key, header_value in headers.items():
                header_list.append(str("%s: %s" % (header_key, header_value)))
            self.c.setopt(pycurl.HTTPHEADER, header_list)

        self.curl_multi.add_handle(self.c)

        # Send request and read all headers. We have all headers once we receive some data or once
        # the response ends.
        # NOTE: HTTP response in chunked encoding can contain additional headers ("trailers") in the
        # last chunk. This is not handled here.
        while not (self.finished or self._any_data_received()):
            self._select()
            self._perform()

        self.headers = parse_headers(self.headers_buffer.getvalue())
        self.status_code = self.c.getinfo(pycurl.HTTP_CODE)
        self.response_decoder = codecs.getincrementaldecoder(self.encoding)()

Example 27

Project: sqlalchemy-media
Source File: test_file.py
View license
    def test_file(self):

        class Person(self.Base):
            __tablename__ = 'person'
            id = Column(Integer, primary_key=True)
            cv = Column(File.as_mutable(Json), nullable=True)

        session = self.create_all_and_get_session()

        # person1 = Person(name='person1')
        person1 = Person()
        self.assertIsNone(person1.cv)
        sample_content = b'Simple text.'

        with StoreManager(session):

            # First file before commit
            person1.cv = File.create_from(BytesIO(sample_content), content_type='text/plain', extension='.txt')
            self.assertIsInstance(person1.cv, File)
            self.assertEqual(person1.cv.locate(), '%s/%s?_ts=%s' % (
                self.base_url, person1.cv.path, person1.cv.timestamp))
            self.assertDictEqual(person1.cv, {
                'content_type': 'text/plain',
                'key': person1.cv.key,
                'extension': '.txt',
                'length': len(sample_content),
                'timestamp': person1.cv.timestamp
            })
            first_filename = join(self.temp_path, person1.cv.path)
            self.assertTrue(exists(first_filename))
            self.assertEqual(person1.cv.length, len(sample_content))

            # Second file before commit
            person1.cv.attach(BytesIO(sample_content), content_type='text/plain', extension='.txt')
            second_filename = join(self.temp_path, person1.cv.path)
            self.assertTrue(exists(second_filename))

            # Adding object to session, the new life-cycle of the person1 just began.
            session.add(person1)
            session.commit()
            self.assertFalse(exists(first_filename))
            self.assertTrue(exists(second_filename))

            # Loading again
            sample_content = b'Lorem ipsum dolor sit amet'
            person1 = session.query(Person).filter(Person.id == person1.id).one()
            person1.cv.attach(BytesIO(sample_content), content_type='text/plain', extension='.txt')
            self.assertIsInstance(person1.cv, File)
            self.assertDictEqual(person1.cv, {
                'content_type': 'text/plain',
                'key': person1.cv.key,
                'extension': '.txt',
                'length': len(sample_content),
                'timestamp': person1.cv.timestamp
            })
            third_filename = join(self.temp_path, person1.cv.path)
            self.assertTrue(exists(second_filename))
            self.assertTrue(exists(third_filename))

            # Committing the session, so the store must done the scheduled jobs
            session.commit()
            self.assertFalse(exists(second_filename))
            self.assertTrue(exists(third_filename))

            # Rollback
            person1.cv.attach(BytesIO(sample_content), content_type='text/plain', extension='.txt')
            forth_filename = join(self.temp_path, person1.cv.path)
            self.assertTrue(exists(forth_filename))
            session.rollback()
            self.assertTrue(exists(third_filename))
            self.assertFalse(exists(forth_filename))

            # Delete file after object deletion
            person1 = session.query(Person).filter(Person.id == person1.id).one()
            session.delete(person1)
            session.commit()
            self.assertFalse(exists(third_filename))

            # Delete file on set to null
            person1 = Person()
            self.assertIsNone(person1.cv)
            person1.cv = File()
            person1.cv.attach(BytesIO(sample_content), content_type='text/plain', extension='.txt')
            fifth_filename = join(self.temp_path, person1.cv.path)
            person1.cv = None
            session.add(person1)
            self.assertTrue(exists(fifth_filename))
            session.commit()
            # Because delete_orphan is not set.
            self.assertTrue(exists(fifth_filename))

            # storing a file on separate store:
            person1.cv = File.create_from(BytesIO(sample_content), store_id='temp_fs')
            fifth_filename = join(self.sys_temp_path, person1.cv.path)
            session.commit()
            self.assertTrue(exists(fifth_filename))

Example 28

Project: python-ivi
Source File: test_agilent34401A.py
View license
    def __init__(self):
        self.read_buffer = io.BytesIO()
        self.tx_log = list()
        self.rx_log = list()
        self.cmd_log = list()

        self.cmds = {
            '*cls' : None,
            '*idn?' : str,
            '*rst' : None,
            '*trg' : None,
            '*tst?' : int,
            'system:error?' : str,
            'abort' : None,
            'fetch?' : float,
            'initiate' : None,
            'read?' : float,
            'sense:function' : 'qstr',
            'sense:function?' : 'qstr',
            'volt:dc:range' : float,
            'volt:dc:range?' : float,
            'volt:ac:range' : float,
            'volt:ac:range?' : float,
            'curr:dc:range' : float,
            'curr:dc:range?' : float,
            'curr:ac:range' : float,
            'curr:ac:range?' : float,
            'res:range' : float,
            'res:range?' : float,
            'fres:range' : float,
            'fres:range?' : float,
            'freq:range:lower' : float,
            'freq:range:lower?' : float,
            'per:range:lower' : float,
            'per:range:lower?' : float,
            'cap:range' : float,
            'cap:range?' : float,
            'volt:dc:range:auto' : int,
            'volt:dc:range:auto?' : int,
            'volt:ac:range:auto' : int,
            'volt:ac:range:auto?' : int,
            'curr:dc:range:auto' : int,
            'curr:dc:range:auto?' : int,
            'curr:ac:range:auto' : int,
            'curr:ac:range:auto?' : int,
            'res:range:auto' : int,
            'res:range:auto?' : int,
            'fres:range:auto' : int,
            'fres:range:auto?' : int,
            'freq:range:auto' : int,
            'freq:range:auto?' : int,
            'per:range:auto' : int,
            'per:range:auto?' : int,
            'cap:range:auto' : int,
            'cap:range:auto?' : int,
            'volt:dc:resolution' : float,
            'volt:dc:resolution?' : float,
            'volt:ac:resolution' : float,
            'volt:ac:resolution?' : float,
            'curr:dc:resolution' : float,
            'curr:dc:resolution?' : float,
            'curr:ac:resolution' : float,
            'curr:ac:resolution?' : float,
            'res:resolution' : float,
            'res:resolution?' : float,
            'fres:resolution' : float,
            'fres:resolution?' : float,
            'trigger:delay' : float,
            'trigger:delay?' : float,
            'trigger:delay:auto' : int,
            'trigger:delay:auto?' : int,
            'trigger:source' : str,
            'trigger:source?' : str,
            'sample:count' : int,
            'sample:count?' : int,
            'trigger:count' : int,
            'trigger:count?' : int,
        }
        self.vals = {
            '*idn' : 'HEWLETT-PACKARD,34401A,0,1.7-5.0-1.0',
            '*tst' : 0,
            'system:error' : '+0,"No error"',
            'fetch?' : 1.0,
            'read?' : 1.0,
            'sense:function' : 'dc_volts',
            'volt:dc:range' : 1.0,
            'volt:ac:range' : 1.0,
            'curr:dc:range' : 1.0,
            'curr:ac:range' : 1.0,
            'res:range' : 1.0,
            'fres:range' : 1.0,
            'freq:range:lower' : 1.0,
            'per:range:lower' : 1.0,
            'cap:range' : 1.0,
            'volt:dc:range:auto' : 1,
            'volt:ac:range:auto' : 1,
            'curr:dc:range:auto' : 1,
            'curr:ac:range:auto' : 1,
            'res:range:auto' : 1,
            'fres:range:auto' : 1,
            'freq:range:auto' : 1,
            'per:range:auto' : 1,
            'cap:range:auto' : 1,
            'volt:dc:resolution' : 0.001,
            'volt:ac:resolution' : 0.001,
            'curr:dc:resolution' : 0.001,
            'curr:ac:resolution' : 0.001,
            'res:resolution' : 0.001,
            'fres:resolution' : 0.001,
            'trigger:delay' : 0.01,
            'trigger:delay:auto' : 1,
            'trigger:source' : 'imm',
            'sample:count' : 1,
            'trigger:count' : 1,
        }

        for n in range(4):
            ch = 'output%d' % (n+1)
            self.cmds[ch+':voltage'] = float
            self.cmds[ch+':voltage?'] = float
            self.vals[ch+':voltage'] = 0.0

Example 29

Project: python-future
Source File: message.py
View license
    def get_payload(self, i=None, decode=False):
        """Return a reference to the payload.

        The payload will either be a list object or a string.  If you mutate
        the list object, you modify the message's payload in place.  Optional
        i returns that index into the payload.

        Optional decode is a flag indicating whether the payload should be
        decoded or not, according to the Content-Transfer-Encoding header
        (default is False).

        When True and the message is not a multipart, the payload will be
        decoded if this header's value is `quoted-printable' or `base64'.  If
        some other encoding is used, or the header is missing, or if the
        payload has bogus data (i.e. bogus base64 or uuencoded data), the
        payload is returned as-is.

        If the message is a multipart and the decode flag is True, then None
        is returned.
        """
        # Here is the logic table for this code, based on the email5.0.0 code:
        #   i     decode  is_multipart  result
        # ------  ------  ------------  ------------------------------
        #  None   True    True          None
        #   i     True    True          None
        #  None   False   True          _payload (a list)
        #   i     False   True          _payload element i (a Message)
        #   i     False   False         error (not a list)
        #   i     True    False         error (not a list)
        #  None   False   False         _payload
        #  None   True    False         _payload decoded (bytes)
        # Note that Barry planned to factor out the 'decode' case, but that
        # isn't so easy now that we handle the 8 bit data, which needs to be
        # converted in both the decode and non-decode path.
        if self.is_multipart():
            if decode:
                return None
            if i is None:
                return self._payload
            else:
                return self._payload[i]
        # For backward compatibility, Use isinstance and this error message
        # instead of the more logical is_multipart test.
        if i is not None and not isinstance(self._payload, list):
            raise TypeError('Expected list, got %s' % type(self._payload))
        payload = self._payload
        # cte might be a Header, so for now stringify it.
        cte = str(self.get('content-transfer-encoding', '')).lower()
        # payload may be bytes here.
        if isinstance(payload, str):
            payload = str(payload)    # for Python-Future, so surrogateescape works
            if utils._has_surrogates(payload):
                bpayload = payload.encode('ascii', 'surrogateescape')
                if not decode:
                    try:
                        payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace')
                    except LookupError:
                        payload = bpayload.decode('ascii', 'replace')
            elif decode:
                try:
                    bpayload = payload.encode('ascii')
                except UnicodeError:
                    # This won't happen for RFC compliant messages (messages
                    # containing only ASCII codepoints in the unicode input).
                    # If it does happen, turn the string into bytes in a way
                    # guaranteed not to fail.
                    bpayload = payload.encode('raw-unicode-escape')
        if not decode:
            return payload
        if cte == 'quoted-printable':
            return utils._qdecode(bpayload)
        elif cte == 'base64':
            # XXX: this is a bit of a hack; decode_b should probably be factored
            # out somewhere, but I haven't figured out where yet.
            value, defects = decode_b(b''.join(bpayload.splitlines()))
            for defect in defects:
                self.policy.handle_defect(self, defect)
            return value
        elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'):
            in_file = BytesIO(bpayload)
            out_file = BytesIO()
            try:
                uu.decode(in_file, out_file, quiet=True)
                return out_file.getvalue()
            except uu.Error:
                # Some decoding problem
                return bpayload
        if isinstance(payload, str):
            return bpayload
        return payload

Example 30

Project: wotdecoder
Source File: wotdecoder.py
View license
def replay(filename, to_decode):
# filename= name of .wotreplay file
# to_decode= bitmask of chunks you want decoded.
# We do not just count blocks as they are in replay files. Instead we always decode
# Bit 0 = first Json block, starting player list
# Bit 1 = second Json block, simplified frag count
# Bit 2 = pickle, proper battle result with damage numbers
# 7(binary 111) means decode all three. 5(binary 101) means decode first Json and pikle.
#
# returns decoded_chunks[0:3], chunks bitmask, decoder status

  while True:
    wot_replay_magic_number = "12323411"
    blocks = 0
    first_chunk_decoded = {}
    second_chunk_decoded = {}
    third_chunk_decoded = {}
    chunks_bitmask = 0
    filesize = os.path.getsize(filename)
    if filesize<12: processing =10; break
    f = open(filename, "rb")
    if f.read(4)!=bytes.fromhex(wot_replay_magic_number): processing =11; break  
    blocks = struct.unpack("i",f.read(4))[0]

# 8.1 Adds new unencrypted Python pickle block containing your match stats
# Before 8.1 (< 20121101)
#  Json + binary = 1 = incomplete.
#  Json + Json + binary = 2 = complete.
# After  8.1 (>=20121101)
#  Json + binary = 1 = incomplete.
#  Json + pickle + binary = 2 = incomplete, but you looked at 'Battle Result' screen and replay got updated.
#  Json + Json + pickle + binary = 3 = complete.
# Some oddities:
#  Json + Json + ~8 bytes = 2 = incomplete, game crashed somewhere, second Json has game result, but we are missing Pickle
#
# Proper way to detect replay version is to decrypt and decompress binary part, but that is too slow.
# Instead I am using Date to estimate version in a very crude way. It is only accurade down to a day and doesnt take into
# consideration player timezone so I need to double check replays saved at 20121101. Still faster than decrypting and
# unzipping 1MB files.


    first_size = struct.unpack("i",f.read(4))[0]
#    print (first_size, filename)

    if filesize < (12+first_size+4): processing =10; break

    if (blocks == 1) and (not (to_decode&1)): processing =1; break

    first_chunk = f.read(first_size)
    if first_chunk[0:1] != b'{': processing =13; break
    first_chunk_decoded = json.loads(first_chunk.decode('utf-8'))
    chunks_bitmask = 1

    if blocks == 1: processing =1; break
    if ((blocks!=2) and (blocks!=3)): processing =16; break

    replaydate = datetime.strptime(first_chunk_decoded['dateTime'][0:10], "%d.%m.%Y")

    second_size = struct.unpack("i",f.read(4))[0]
    if filesize < (12+first_size+4+second_size): processing =10; break
    second_chunk = f.read(second_size)

# <20121101 and blocks==2 means Complete (pre 8.1). Second block should be Json.
    if (replaydate < datetime(2012, 11, 1)) and blocks==2:
      if second_chunk[0:2] == b'[{':
# Complete (pre 8.1).
        if to_decode&2:
          second_chunk_decoded = json.loads(second_chunk.decode('utf-8'))
          chunks_bitmask = chunks_bitmask|2
        processing =3; break
      else: processing =14; break

# =20121101 and blocks==2 can go both ways, need to autodetect second block.
# >20121101 and blocks==2 can contain broken replay
    elif (replaydate >= datetime(2012, 11, 1)) and blocks==2:
      if second_chunk[0:2] == b'(d':
# Incomplete (past 8.1), with 'Battle Result' pickle.
        if to_decode&4:
          third_chunk_decoded = _Unpickler(io.BytesIO(second_chunk)).load()
          chunks_bitmask = chunks_bitmask|4
          for b in third_chunk_decoded['vehicles']:
            third_chunk_decoded['vehicles'][b]['details']= _Decoder.decode_details(third_chunk_decoded['vehicles'][b]['details'].encode('raw_unicode_escape'))
            third_chunk_decoded['players'][ third_chunk_decoded['vehicles'][b]['accountDBID'] ]["vehicleid"]=b
        processing =2; break
      elif second_chunk[0:2] == b'[{':
        if to_decode&2:
          second_chunk_decoded = json.loads(second_chunk.decode('utf-8'))
          chunks_bitmask = chunks_bitmask|2
        if replaydate == datetime(2012, 11, 1):
# Complete (pre 8.1).
          processing =3; break
        else:
# Bugged (past 8.1). Game crashed somewhere, second Json has game result.
          processing =6; break

# >=20121101 and blocks==3 means Complete (past 8.1).
    elif (replaydate >= datetime(2012, 11, 1)) and blocks==3:
      if second_chunk[0:2] == b'[{':
        if to_decode&2:
          second_chunk_decoded = json.loads(second_chunk.decode('utf-8'))
          chunks_bitmask = chunks_bitmask|2
        if filesize<(12+first_size+4+second_size+4): processing =10; break
        third_size = struct.unpack("i",f.read(4))[0]
        if filesize<(12+first_size+4+second_size+4+third_size): processing =10; break
        third_chunk = f.read(third_size)
        if third_chunk[0:2] == b'(d':
          if to_decode&4:
            third_chunk_decoded = _Unpickler(io.BytesIO(third_chunk)).load()
            chunks_bitmask = chunks_bitmask|4
            for b in third_chunk_decoded['vehicles']:
              third_chunk_decoded['vehicles'][b]['details']= _Decoder.decode_details(third_chunk_decoded['vehicles'][b]['details'].encode('raw_unicode_escape'))
              third_chunk_decoded['players'][ third_chunk_decoded['vehicles'][b]['accountDBID'] ]["vehicleid"]=b
          processing =4; break
        else: processing =15; break
      else: processing =14; break


# All states that we can handle broke out of the While loop at this point.
# Unhandled cases trigger this.
    processing =20; break

  f.close()

  if chunks_bitmask&5 ==5:
# lets check if pickle belongs to this replay
# this is weak check, we only compare map and game mode, It can still pass some corrupted ones
    if maps[ third_chunk_decoded['common']['arenaTypeID'] & 65535 ][0] !=first_chunk_decoded['mapName'] or \
       gameplayid[ third_chunk_decoded['common']['arenaTypeID'] >>16] != first_chunk_decoded['gameplayID']:
#      print("EERRRROOOORRRrrrrrr!!!one77")
#      print("json:  ", first_chunk_decoded['mapName'])
#      print("pickle:", maps[ third_chunk_decoded['common']['arenaTypeID'] & 65535 ])
#      print("json:  ", first_chunk_decoded['gameplayID'])
#      print("pickle:", gameplayid[ third_chunk_decoded['common']['arenaTypeID'] >>16])
      processing =8
#      chunks_bitmask = chunks_bitmask^4
#      print(datetime.strptime(chunks[0]['dateTime'], '%d.%m.%Y %H:%M:%S'))
#      print( datetime.fromtimestamp(chunks[2]['common']['arenaCreateTime']))
#      print( mapidname[ chunks[2]['common']['arenaTypeID'] & 65535 ])

#guesstimating version, reliable only since 8.6 because WG added version string, earlier ones can be ~guessed by counting data or comparing dates
  if chunks_bitmask&1 ==1:
   if "clientVersionFromExe" in first_chunk_decoded:
    version = int(first_chunk_decoded["clientVersionFromExe"].replace(', ',''))
#    print (first_chunk_decoded["clientVersionFromExe"], version)
   else:
    
#8.7
#July 29, 2013
#8.4
#05 Mar 2013
#8.3
#16 Jan 2013
#8.2
#12 Dec 2012
#8.1
#Mar 13 2013
#8.0
#Sep 24 2012 
#7.5
#04.08.2012
#7.4
#20.06.2012
#7.3
#11.05.2012
#7.2
#30.03.2012
#7.1
#05.01.2012
#7.0
#19.12.2011
#6.7
#15.09.2011
#6.6
#10.08.2011
#6.5
#Jun 14 2011
#6.4
#Mar 12 2011
#6.3.11
#Apr 12 2011
#6.3.10
#Apr 07 2011
#6.3.9
#Mar 22 2011
#6.3
#Jan 15 2011
#6.2.8
#Dec 28 2010
#6.2.7
#Dec 23 2010
#6.2
#Dec 01 2010
#6.1.5
#Sep 28 2010
#5.5
#Oct 21 2010
#5.4.1
#Jul 16 2010    

    version = 830 # no clue, lets default to safe 8.3
  else:
   version = 0 #no first chunk = no version

# returns decoded_chunk[0:3], bitmap of available chunks, decoder status, ~version
  return (first_chunk_decoded, second_chunk_decoded, third_chunk_decoded), chunks_bitmask, processing, version

Example 31

Project: wotdecoder
Source File: wotdecoder.py
View license
def battle_result(filename):
# filename= name of .dat battle_results file
#
# returns decoded_chunk
# Will throw ValueError if filename not a pickle.

  battle_result_magic_number = "80024b01"
  filesize = os.path.getsize(filename)
  f = open(filename, "rb")
  chunk = f.read(filesize)
  f.close()

  if chunk[0:4] != bytes.fromhex(battle_result_magic_number):
    raise ValueError(filename, "Bad pickle magic_number")

  pre_preliminary = _Unpickler(io.BytesIO(chunk)).load()
  pre_preliminary = pre_preliminary[1]
#  print(pre_preliminary[1])

  preliminary = _Unpickler(io.BytesIO(pre_preliminary[2])).load()
#  print(preliminary[2])
#  print(preliminary, pre_preliminary[0])

  common_to_decode = preliminary[0]
#  for deind, val in enumerate(common_to_decode):
#   print(deind, val)
  common_decoded = {}
  common_decoded["arenaTypeID"]= common_to_decode[0] # & 65535 makes more sense, but original replay keeps this number intact
  common_decoded["arenaCreateTime"]= common_to_decode[1]
  common_decoded["winnerTeam"]= common_to_decode[2]
  common_decoded["finishReason"]= common_to_decode[3]
  common_decoded["duration"]= common_to_decode[4]
  common_decoded["bonusType"]= common_to_decode[5]
  common_decoded["guiType"]= common_to_decode[6]
  common_decoded["vehLockMode"]= common_to_decode[7]


#  print(common_to_decode)

# Some additional variables Phalynx's www.vbaddict.net/wot uses.
#  common_decoded["guiType"]= common_to_decode[6]
#  common_decoded["gameplayID"]= common_to_decode[0] >>16 #this doesnt exist in replays, safer to always use (arenaTypeID >>16)
#  common_decoded["arenaCreateTimeH"]= #datetime.datetime.fromtimestamp(common_to_decode[1]).strftime("%Y-%m-%d %H:%M:%S")
#  common_decoded["arenaTypeIcon"]= # who cares about the icon blah
#  common_decoded["arenaTypeName"]= # maps[arenaTypeID & 65535]
#  common_decoded["bonusTypeName"]= #1=public 2=training 3=tankcompany 5=cw
#  common_decoded["finishReasonName"]= #1=extermination, 2=base, 3=timeout
#  common_decoded["gameplayName"]= # (arenaTypeID >>16) 0=random/ctf, 1=encounter 2=assault
#  common_decoded["gameplayTypeID"]= #probably created because Phalynx parses gameplayID wrong way :)


  personal_to_decode = pre_preliminary[1]
  personal_decoded = {}
#  for deind, val in enumerate(personal_to_decode):
#   print(deind, val)
#  print(len(personal_to_decode))
#  print(personal_to_decode)
#  print(personal_decoded)
  personal_decoded, version = _Decoder.decode_vehicle(personal_to_decode)
#  pprint(personal_decoded)

#  personal_decoded["xpPenalty"]= personal_to_decode[30]
#  personal_decoded["creditsPenalty"]= personal_to_decode[31]
#  personal_decoded["creditsContributionIn"]= personal_to_decode[32]
#  personal_decoded["creditsContributionOut"]= personal_to_decode[33]
#  personal_decoded["tmenXP"]= personal_to_decode[34]
#  personal_decoded["eventCredits"]= personal_to_decode[35]
#  personal_decoded["eventGold"]= personal_to_decode[36]
#  personal_decoded["eventXP"]= personal_to_decode[37]
#  personal_decoded["eventFreeXP"]= personal_to_decode[38]
#  personal_decoded["eventTMenXP"]= personal_to_decode[39]
#  personal_decoded["autoRepairCost"]= personal_to_decode[40]
#  personal_decoded["autoLoadCost"]= list(personal_to_decode[41])    
#  personal_decoded["autoEquipCost"]= list(personal_to_decode[42])
#  personal_decoded["isPremium"]= personal_to_decode[43]
#  personal_decoded["premiumXPFactor10"]= personal_to_decode[44]
#  personal_decoded["premiumCreditsFactor10"]= personal_to_decode[45]
#  personal_decoded["dailyXPFactor10"]= personal_to_decode[46]
#  personal_decoded["aogasFactor10"]= personal_to_decode[47]
#  personal_decoded["markOfMastery"]= personal_to_decode[48]
# personal_decoded["dossierPopUps"]= personal_to_decode[49]

# Some additional variables Phalynx's www.vbaddict.net/wot uses.
#  personal_decoded["won"]=

  players_to_decode = preliminary[1]
  players_decoded = {}
  for player in players_to_decode:
#    for deind, val in enumerate(players_to_decode[player]):
#     print(deind, val)
    player_decoded = {}
    player_decoded["name"]= players_to_decode[player][0].decode('unicode_escape')
    player_decoded["clanDBID"]= players_to_decode[player][1]
    player_decoded["clanAbbrev"]= players_to_decode[player][2].decode('unicode_escape')
    player_decoded["prebattleID"]= players_to_decode[player][3] #this is platoonID
    player_decoded["team"]= players_to_decode[player][4]
    players_decoded[str(player)]= player_decoded
# Some additional variables Phalynx's www.vbaddict.net/wot uses.
#  player_decoded["platoonID"]= #prebattleID
#  player_decoded["vehicleid"]= #we will fill that up while decoding vehicles


  vehicles_to_decode = preliminary[2]
  vehicles_decoded = {}
  for vehicle in vehicles_to_decode:
#  for deind, val in enumerate(vehicles_to_decode[vehicle]):
#  print(deind, val)
    vehicles_decoded[str(vehicle)], v= _Decoder.decode_vehicle(vehicles_to_decode[vehicle])
#    players_decoded[str(vehicles_decoded[vehicle]['accountDBID'])]["vehicleid"]=vehicle
  whole_thing = {}
  whole_thing['arenaUniqueID']= pre_preliminary[0]
  whole_thing['common']= common_decoded
  whole_thing['personal']= personal_decoded
  whole_thing['players']= players_decoded
  whole_thing['vehicles']= vehicles_decoded

  return whole_thing, version

Example 32

Project: rdflib
Source File: test_dawg.py
View license
@nottest  # gets called by generator
def query_test(t):
    uri, name, comment, data, graphdata, query, resfile, syntax = t

    # the query-eval tests refer to graphs to load by resolvable filenames
    rdflib_sparql_module.SPARQL_LOAD_GRAPHS = True

    if uri in skiptests:
        raise SkipTest()

    def skip(reason='(none)'):
        print "Skipping %s from now on." % uri
        with bopen("skiptests.list", "a") as f:
            f.write("%s\t%s\n" % (uri, reason))

    try:
        g = Dataset()
        if data:
            g.default_context.load(data, format=_fmt(data))

        if graphdata:
            for x in graphdata:
                g.load(x, format=_fmt(x))

        if not resfile:
            # no result - syntax test

            if syntax:
                translateQuery(parseQuery(
                    bopen_read_close(query[7:])), base=urljoin(query, '.'))
            else:
                # negative syntax test
                try:
                    translateQuery(parseQuery(
                        bopen_read_close(query[7:])), base=urljoin(query, '.'))

                    assert False, 'Query should not have parsed!'
                except:
                    pass  # it's fine - the query should not parse
            return

        # eval test - carry out query
        res2 = g.query(bopen_read_close(query[7:]), base=urljoin(query, '.'))

        if resfile.endswith('ttl'):
            resg = Graph()
            resg.load(resfile, format='turtle', publicID=resfile)
            res = RDFResultParser().parse(resg)
        elif resfile.endswith('rdf'):
            resg = Graph()
            resg.load(resfile, publicID=resfile)
            res = RDFResultParser().parse(resg)
        else:
            with bopen(resfile[7:]) as f:
                if resfile.endswith('srj'):
                    res = Result.parse(f, format='json')
                elif resfile.endswith('tsv'):
                    res = Result.parse(f, format='tsv')

                elif resfile.endswith('csv'):
                    res = Result.parse(f, format='csv')

                    # CSV is lossy, round-trip our own resultset to
                    # lose the same info :)

                    # write bytes, read strings...
                    s = BytesIO()
                    res2.serialize(s, format='csv')
                    s.seek(0)
                    res2 = Result.parse(s, format='csv')
                    s.close()

                else:
                    res = Result.parse(f, format='xml')

        if not DETAILEDASSERT:
            eq(res.type, res2.type, 'Types do not match')
            if res.type == 'SELECT':
                eq(set(res.vars), set(res2.vars), 'Vars do not match')
                comp = bindingsCompatible(
                    set(res),
                    set(res2)
                )
                assert comp, 'Bindings do not match'
            elif res.type == 'ASK':
                eq(res.askAnswer, res2.askAnswer, 'Ask answer does not match')
            elif res.type in ('DESCRIBE', 'CONSTRUCT'):
                assert isomorphic(
                    res.graph, res2.graph), 'graphs are not isomorphic!'
            else:
                raise Exception('Unknown result type: %s' % res.type)
        else:
            eq(res.type, res2.type,
               'Types do not match: %r != %r' % (res.type, res2.type))
            if res.type == 'SELECT':
                eq(set(res.vars),
                   set(res2.vars), 'Vars do not match: %r != %r' % (
                   set(res.vars), set(res2.vars)))
                assert bindingsCompatible(
                    set(res),
                    set(res2)
                ), 'Bindings do not match: \n%s\n!=\n%s' % (
                    res.serialize(format='txt', namespace_manager=g.namespace_manager),
                    res2.serialize(format='txt', namespace_manager=g.namespace_manager))
            elif res.type == 'ASK':
                eq(res.askAnswer,
                   res2.askAnswer, "Ask answer does not match: %r != %r" % (
                   res.askAnswer, res2.askAnswer))
            elif res.type in ('DESCRIBE', 'CONSTRUCT'):
                assert isomorphic(
                    res.graph, res2.graph), 'graphs are not isomorphic!'
            else:
                raise Exception('Unknown result type: %s' % res.type)

    except Exception, e:

        if isinstance(e, AssertionError):
            failed_tests.append(uri)
            fails[str(e)] += 1
        else:
            error_tests.append(uri)
            errors[str(e)] += 1

        if DEBUG_ERROR and not isinstance(e, AssertionError) or DEBUG_FAIL:
            print "======================================"
            print uri
            print name
            print comment

            if not resfile:
                if syntax:
                    print "Positive syntax test"
                else:
                    print "Negative syntax test"

            if data:
                print "----------------- DATA --------------------"
                print ">>>", data
                print bopen_read_close(data[7:])
            if graphdata:
                print "----------------- GRAPHDATA --------------------"
                for x in graphdata:
                    print ">>>", x
                    print bopen_read_close(x[7:])

            print "----------------- Query -------------------"
            print ">>>", query
            print bopen_read_close(query[7:])
            if resfile:
                print "----------------- Res -------------------"
                print ">>>", resfile
                print bopen_read_close(resfile[7:])

            try:
                pq = parseQuery(bopen_read_close(query[7:]))
                print "----------------- Parsed ------------------"
                pprintAlgebra(translateQuery(pq, base=urljoin(query, '.')))
            except:
                print "(parser error)"

            print decodeStringEscape(unicode(e))

            import pdb
            pdb.post_mortem(sys.exc_info()[2])
            # pdb.set_trace()
            # nose.tools.set_trace()
        raise

Example 33

Project: youtube-dl
Source File: swfinterp.py
View license
    def __init__(self, file_contents):
        self._patched_functions = {
            (TimerClass, 'addEventListener'): lambda params: undefined,
        }
        code_tag = next(tag
                        for tag_code, tag in _extract_tags(file_contents)
                        if tag_code == 82)
        p = code_tag.index(b'\0', 4) + 1
        code_reader = io.BytesIO(code_tag[p:])

        # Parse ABC (AVM2 ByteCode)

        # Define a couple convenience methods
        u30 = lambda *args: _u30(*args, reader=code_reader)
        s32 = lambda *args: _s32(*args, reader=code_reader)
        u32 = lambda *args: _u32(*args, reader=code_reader)
        read_bytes = lambda *args: _read_bytes(*args, reader=code_reader)
        read_byte = lambda *args: _read_byte(*args, reader=code_reader)

        # minor_version + major_version
        read_bytes(2 + 2)

        # Constant pool
        int_count = u30()
        self.constant_ints = [0]
        for _c in range(1, int_count):
            self.constant_ints.append(s32())
        self.constant_uints = [0]
        uint_count = u30()
        for _c in range(1, uint_count):
            self.constant_uints.append(u32())
        double_count = u30()
        read_bytes(max(0, (double_count - 1)) * 8)
        string_count = u30()
        self.constant_strings = ['']
        for _c in range(1, string_count):
            s = _read_string(code_reader)
            self.constant_strings.append(s)
        namespace_count = u30()
        for _c in range(1, namespace_count):
            read_bytes(1)  # kind
            u30()  # name
        ns_set_count = u30()
        for _c in range(1, ns_set_count):
            count = u30()
            for _c2 in range(count):
                u30()
        multiname_count = u30()
        MULTINAME_SIZES = {
            0x07: 2,  # QName
            0x0d: 2,  # QNameA
            0x0f: 1,  # RTQName
            0x10: 1,  # RTQNameA
            0x11: 0,  # RTQNameL
            0x12: 0,  # RTQNameLA
            0x09: 2,  # Multiname
            0x0e: 2,  # MultinameA
            0x1b: 1,  # MultinameL
            0x1c: 1,  # MultinameLA
        }
        self.multinames = ['']
        for _c in range(1, multiname_count):
            kind = u30()
            assert kind in MULTINAME_SIZES, 'Invalid multiname kind %r' % kind
            if kind == 0x07:
                u30()  # namespace_idx
                name_idx = u30()
                self.multinames.append(self.constant_strings[name_idx])
            elif kind == 0x09:
                name_idx = u30()
                u30()
                self.multinames.append(self.constant_strings[name_idx])
            else:
                self.multinames.append(_Multiname(kind))
                for _c2 in range(MULTINAME_SIZES[kind]):
                    u30()

        # Methods
        method_count = u30()
        MethodInfo = collections.namedtuple(
            'MethodInfo',
            ['NEED_ARGUMENTS', 'NEED_REST'])
        method_infos = []
        for method_id in range(method_count):
            param_count = u30()
            u30()  # return type
            for _ in range(param_count):
                u30()  # param type
            u30()  # name index (always 0 for youtube)
            flags = read_byte()
            if flags & 0x08 != 0:
                # Options present
                option_count = u30()
                for c in range(option_count):
                    u30()  # val
                    read_bytes(1)  # kind
            if flags & 0x80 != 0:
                # Param names present
                for _ in range(param_count):
                    u30()  # param name
            mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
            method_infos.append(mi)

        # Metadata
        metadata_count = u30()
        for _c in range(metadata_count):
            u30()  # name
            item_count = u30()
            for _c2 in range(item_count):
                u30()  # key
                u30()  # value

        def parse_traits_info():
            trait_name_idx = u30()
            kind_full = read_byte()
            kind = kind_full & 0x0f
            attrs = kind_full >> 4
            methods = {}
            constants = None
            if kind == 0x00:  # Slot
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                if vindex != 0:
                    read_byte()  # vkind
            elif kind == 0x06:  # Const
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                vkind = 'any'
                if vindex != 0:
                    vkind = read_byte()
                if vkind == 0x03:  # Constant_Int
                    value = self.constant_ints[vindex]
                elif vkind == 0x04:  # Constant_UInt
                    value = self.constant_uints[vindex]
                else:
                    return {}, None  # Ignore silently for now
                constants = {self.multinames[trait_name_idx]: value}
            elif kind in (0x01, 0x02, 0x03):  # Method / Getter / Setter
                u30()  # disp_id
                method_idx = u30()
                methods[self.multinames[trait_name_idx]] = method_idx
            elif kind == 0x04:  # Class
                u30()  # slot_id
                u30()  # classi
            elif kind == 0x05:  # Function
                u30()  # slot_id
                function_idx = u30()
                methods[function_idx] = self.multinames[trait_name_idx]
            else:
                raise ExtractorError('Unsupported trait kind %d' % kind)

            if attrs & 0x4 != 0:  # Metadata present
                metadata_count = u30()
                for _c3 in range(metadata_count):
                    u30()  # metadata index

            return methods, constants

        # Classes
        class_count = u30()
        classes = []
        for class_id in range(class_count):
            name_idx = u30()

            cname = self.multinames[name_idx]
            avm_class = _AVMClass(name_idx, cname)
            classes.append(avm_class)

            u30()  # super_name idx
            flags = read_byte()
            if flags & 0x08 != 0:  # Protected namespace is present
                u30()  # protected_ns_idx
            intrf_count = u30()
            for _c2 in range(intrf_count):
                u30()
            u30()  # iinit
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        assert len(classes) == class_count
        self._classes_by_name = dict((c.name, c) for c in classes)

        for avm_class in classes:
            avm_class.cinit_idx = u30()
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        # Scripts
        script_count = u30()
        for _c in range(script_count):
            u30()  # init
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        # Method bodies
        method_body_count = u30()
        Method = collections.namedtuple('Method', ['code', 'local_count'])
        self._all_methods = []
        for _c in range(method_body_count):
            method_idx = u30()
            u30()  # max_stack
            local_count = u30()
            u30()  # init_scope_depth
            u30()  # max_scope_depth
            code_length = u30()
            code = read_bytes(code_length)
            m = Method(code, local_count)
            self._all_methods.append(m)
            for avm_class in classes:
                if method_idx in avm_class.method_idxs:
                    avm_class.methods[avm_class.method_idxs[method_idx]] = m
            exception_count = u30()
            for _c2 in range(exception_count):
                u30()  # from
                u30()  # to
                u30()  # target
                u30()  # exc_type
                u30()  # var_name
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        assert p + code_reader.tell() == len(code_tag)

Example 34

Project: youtube-dl
Source File: swfinterp.py
View license
    def extract_function(self, avm_class, func_name):
        p = self._patched_functions.get((avm_class, func_name))
        if p:
            return p
        if func_name in avm_class.method_pyfunctions:
            return avm_class.method_pyfunctions[func_name]
        if func_name in self._classes_by_name:
            return self._classes_by_name[func_name].make_object()
        if func_name not in avm_class.methods:
            raise ExtractorError('Cannot find function %s.%s' % (
                avm_class.name, func_name))
        m = avm_class.methods[func_name]

        def resfunc(args):
            # Helper functions
            coder = io.BytesIO(m.code)
            s24 = lambda: _s24(coder)
            u30 = lambda: _u30(coder)

            registers = [avm_class.variables] + list(args) + [None] * m.local_count
            stack = []
            scopes = collections.deque([
                self._classes_by_name, avm_class.constants, avm_class.variables])
            while True:
                opcode = _read_byte(coder)
                if opcode == 9:  # label
                    pass  # Spec says: "Do nothing."
                elif opcode == 16:  # jump
                    offset = s24()
                    coder.seek(coder.tell() + offset)
                elif opcode == 17:  # iftrue
                    offset = s24()
                    value = stack.pop()
                    if value:
                        coder.seek(coder.tell() + offset)
                elif opcode == 18:  # iffalse
                    offset = s24()
                    value = stack.pop()
                    if not value:
                        coder.seek(coder.tell() + offset)
                elif opcode == 19:  # ifeq
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value2 == value1:
                        coder.seek(coder.tell() + offset)
                elif opcode == 20:  # ifne
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value2 != value1:
                        coder.seek(coder.tell() + offset)
                elif opcode == 21:  # iflt
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value1 < value2:
                        coder.seek(coder.tell() + offset)
                elif opcode == 32:  # pushnull
                    stack.append(None)
                elif opcode == 33:  # pushundefined
                    stack.append(undefined)
                elif opcode == 36:  # pushbyte
                    v = _read_byte(coder)
                    stack.append(v)
                elif opcode == 37:  # pushshort
                    v = u30()
                    stack.append(v)
                elif opcode == 38:  # pushtrue
                    stack.append(True)
                elif opcode == 39:  # pushfalse
                    stack.append(False)
                elif opcode == 40:  # pushnan
                    stack.append(float('NaN'))
                elif opcode == 42:  # dup
                    value = stack[-1]
                    stack.append(value)
                elif opcode == 44:  # pushstring
                    idx = u30()
                    stack.append(self.constant_strings[idx])
                elif opcode == 48:  # pushscope
                    new_scope = stack.pop()
                    scopes.append(new_scope)
                elif opcode == 66:  # construct
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                    res = obj.avm_class.make_object()
                    stack.append(res)
                elif opcode == 70:  # callproperty
                    index = u30()
                    mname = self.multinames[index]
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()

                    if obj == StringClass:
                        if mname == 'String':
                            assert len(args) == 1
                            assert isinstance(args[0], (
                                int, compat_str, _Undefined))
                            if args[0] == undefined:
                                res = 'undefined'
                            else:
                                res = compat_str(args[0])
                            stack.append(res)
                            continue
                        else:
                            raise NotImplementedError(
                                'Function String.%s is not yet implemented'
                                % mname)
                    elif isinstance(obj, _AVMClass_Object):
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        stack.append(res)
                        continue
                    elif isinstance(obj, _AVMClass):
                        func = self.extract_function(obj, mname)
                        res = func(args)
                        stack.append(res)
                        continue
                    elif isinstance(obj, _ScopeDict):
                        if mname in obj.avm_class.method_names:
                            func = self.extract_function(obj.avm_class, mname)
                            res = func(args)
                        else:
                            res = obj[mname]
                        stack.append(res)
                        continue
                    elif isinstance(obj, compat_str):
                        if mname == 'split':
                            assert len(args) == 1
                            assert isinstance(args[0], compat_str)
                            if args[0] == '':
                                res = list(obj)
                            else:
                                res = obj.split(args[0])
                            stack.append(res)
                            continue
                        elif mname == 'charCodeAt':
                            assert len(args) <= 1
                            idx = 0 if len(args) == 0 else args[0]
                            assert isinstance(idx, int)
                            res = ord(obj[idx])
                            stack.append(res)
                            continue
                    elif isinstance(obj, list):
                        if mname == 'slice':
                            assert len(args) == 1
                            assert isinstance(args[0], int)
                            res = obj[args[0]:]
                            stack.append(res)
                            continue
                        elif mname == 'join':
                            assert len(args) == 1
                            assert isinstance(args[0], compat_str)
                            res = args[0].join(obj)
                            stack.append(res)
                            continue
                    raise NotImplementedError(
                        'Unsupported property %r on %r'
                        % (mname, obj))
                elif opcode == 71:  # returnvoid
                    res = undefined
                    return res
                elif opcode == 72:  # returnvalue
                    res = stack.pop()
                    return res
                elif opcode == 73:  # constructsuper
                    # Not yet implemented, just hope it works without it
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                elif opcode == 74:  # constructproperty
                    index = u30()
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()

                    mname = self.multinames[index]
                    assert isinstance(obj, _AVMClass)

                    # We do not actually call the constructor for now;
                    # we just pretend it does nothing
                    stack.append(obj.make_object())
                elif opcode == 79:  # callpropvoid
                    index = u30()
                    mname = self.multinames[index]
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                    if isinstance(obj, _AVMClass_Object):
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        assert res is undefined
                        continue
                    if isinstance(obj, _ScopeDict):
                        assert mname in obj.avm_class.method_names
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        assert res is undefined
                        continue
                    if mname == 'reverse':
                        assert isinstance(obj, list)
                        obj.reverse()
                    else:
                        raise NotImplementedError(
                            'Unsupported (void) property %r on %r'
                            % (mname, obj))
                elif opcode == 86:  # newarray
                    arg_count = u30()
                    arr = []
                    for i in range(arg_count):
                        arr.append(stack.pop())
                    arr = arr[::-1]
                    stack.append(arr)
                elif opcode == 93:  # findpropstrict
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            res = s
                            break
                    else:
                        res = scopes[0]
                    if mname not in res and mname in _builtin_classes:
                        stack.append(_builtin_classes[mname])
                    else:
                        stack.append(res[mname])
                elif opcode == 94:  # findproperty
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            res = s
                            break
                    else:
                        res = avm_class.variables
                    stack.append(res)
                elif opcode == 96:  # getlex
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            scope = s
                            break
                    else:
                        scope = avm_class.variables

                    if mname in scope:
                        res = scope[mname]
                    elif mname in _builtin_classes:
                        res = _builtin_classes[mname]
                    else:
                        # Assume uninitialized
                        # TODO warn here
                        res = undefined
                    stack.append(res)
                elif opcode == 97:  # setproperty
                    index = u30()
                    value = stack.pop()
                    idx = self.multinames[index]
                    if isinstance(idx, _Multiname):
                        idx = stack.pop()
                    obj = stack.pop()
                    obj[idx] = value
                elif opcode == 98:  # getlocal
                    index = u30()
                    stack.append(registers[index])
                elif opcode == 99:  # setlocal
                    index = u30()
                    value = stack.pop()
                    registers[index] = value
                elif opcode == 102:  # getproperty
                    index = u30()
                    pname = self.multinames[index]
                    if pname == 'length':
                        obj = stack.pop()
                        assert isinstance(obj, (compat_str, list))
                        stack.append(len(obj))
                    elif isinstance(pname, compat_str):  # Member access
                        obj = stack.pop()
                        if isinstance(obj, _AVMClass):
                            res = obj.static_properties[pname]
                            stack.append(res)
                            continue

                        assert isinstance(obj, (dict, _ScopeDict)),\
                            'Accessing member %r on %r' % (pname, obj)
                        res = obj.get(pname, undefined)
                        stack.append(res)
                    else:  # Assume attribute access
                        idx = stack.pop()
                        assert isinstance(idx, int)
                        obj = stack.pop()
                        assert isinstance(obj, list)
                        stack.append(obj[idx])
                elif opcode == 104:  # initproperty
                    index = u30()
                    value = stack.pop()
                    idx = self.multinames[index]
                    if isinstance(idx, _Multiname):
                        idx = stack.pop()
                    obj = stack.pop()
                    obj[idx] = value
                elif opcode == 115:  # convert_
                    value = stack.pop()
                    intvalue = int(value)
                    stack.append(intvalue)
                elif opcode == 128:  # coerce
                    u30()
                elif opcode == 130:  # coerce_a
                    value = stack.pop()
                    # um, yes, it's any value
                    stack.append(value)
                elif opcode == 133:  # coerce_s
                    assert isinstance(stack[-1], (type(None), compat_str))
                elif opcode == 147:  # decrement
                    value = stack.pop()
                    assert isinstance(value, int)
                    stack.append(value - 1)
                elif opcode == 149:  # typeof
                    value = stack.pop()
                    return {
                        _Undefined: 'undefined',
                        compat_str: 'String',
                        int: 'Number',
                        float: 'Number',
                    }[type(value)]
                elif opcode == 160:  # add
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 + value2
                    stack.append(res)
                elif opcode == 161:  # subtract
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 - value2
                    stack.append(res)
                elif opcode == 162:  # multiply
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 * value2
                    stack.append(res)
                elif opcode == 164:  # modulo
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 % value2
                    stack.append(res)
                elif opcode == 168:  # bitand
                    value2 = stack.pop()
                    value1 = stack.pop()
                    assert isinstance(value1, int)
                    assert isinstance(value2, int)
                    res = value1 & value2
                    stack.append(res)
                elif opcode == 171:  # equals
                    value2 = stack.pop()
                    value1 = stack.pop()
                    result = value1 == value2
                    stack.append(result)
                elif opcode == 175:  # greaterequals
                    value2 = stack.pop()
                    value1 = stack.pop()
                    result = value1 >= value2
                    stack.append(result)
                elif opcode == 192:  # increment_i
                    value = stack.pop()
                    assert isinstance(value, int)
                    stack.append(value + 1)
                elif opcode == 208:  # getlocal_0
                    stack.append(registers[0])
                elif opcode == 209:  # getlocal_1
                    stack.append(registers[1])
                elif opcode == 210:  # getlocal_2
                    stack.append(registers[2])
                elif opcode == 211:  # getlocal_3
                    stack.append(registers[3])
                elif opcode == 212:  # setlocal_0
                    registers[0] = stack.pop()
                elif opcode == 213:  # setlocal_1
                    registers[1] = stack.pop()
                elif opcode == 214:  # setlocal_2
                    registers[2] = stack.pop()
                elif opcode == 215:  # setlocal_3
                    registers[3] = stack.pop()
                else:
                    raise NotImplementedError(
                        'Unsupported opcode %d' % opcode)

        avm_class.method_pyfunctions[func_name] = resfunc
        return resfunc

Example 35

Project: pycoin
Source File: tx.py
View license
def parse_context(args, parser):
    # defaults

    txs = []
    spendables = []
    payables = []

    key_iters = []

    TX_ID_RE = re.compile(r"^[0-9a-fA-F]{64}$")

    # there are a few warnings we might optionally print out, but only if
    # they are relevant. We don't want to print them out multiple times, so we
    # collect them here and print them at the end if they ever kick in.

    warning_tx_cache = None
    warning_tx_for_tx_hash = None
    warning_spendables = None

    if args.private_key_file:
        wif_re = re.compile(r"[1-9a-km-zA-LMNP-Z]{51,111}")
        # address_re = re.compile(r"[1-9a-kmnp-zA-KMNP-Z]{27-31}")
        for f in args.private_key_file:
            if f.name.endswith(".gpg"):
                gpg_args = ["gpg", "-d"]
                if args.gpg_argument:
                    gpg_args.extend(args.gpg_argument.split())
                gpg_args.append(f.name)
                popen = subprocess.Popen(gpg_args, stdout=subprocess.PIPE)
                f = popen.stdout
            for line in f.readlines():
                # decode
                if isinstance(line, bytes):
                    line = line.decode("utf8")
                # look for WIFs
                possible_keys = wif_re.findall(line)

                def make_key(x):
                    try:
                        return Key.from_text(x)
                    except Exception:
                        return None

                keys = [make_key(x) for x in possible_keys]
                for key in keys:
                    if key:
                        key_iters.append((k.wif() for k in key.subkeys("")))

                # if len(keys) == 1 and key.hierarchical_wallet() is None:
                #    # we have exactly 1 WIF. Let's look for an address
                #   potential_addresses = address_re.findall(line)

    # update p2sh_lookup
    p2sh_lookup = {}
    if args.pay_to_script:
        for p2s in args.pay_to_script:
            try:
                script = h2b(p2s)
                p2sh_lookup[hash160(script)] = script
            except Exception:
                print("warning: error parsing pay-to-script value %s" % p2s)

    if args.pay_to_script_file:
        hex_re = re.compile(r"[0-9a-fA-F]+")
        for f in args.pay_to_script_file:
            count = 0
            for l in f:
                try:
                    m = hex_re.search(l)
                    if m:
                        p2s = m.group(0)
                        script = h2b(p2s)
                        p2sh_lookup[hash160(script)] = script
                        count += 1
                except Exception:
                    print("warning: error parsing pay-to-script file %s" % f.name)
            if count == 0:
                print("warning: no scripts found in %s" % f.name)

    # we create the tx_db lazily
    tx_db = None

    for arg in args.argument:

        # hex transaction id
        if TX_ID_RE.match(arg):
            if tx_db is None:
                warning_tx_cache = message_about_tx_cache_env()
                warning_tx_for_tx_hash = message_about_tx_for_tx_hash_env(args.network)
                tx_db = get_tx_db(args.network)
            tx = tx_db.get(h2b_rev(arg))
            if not tx:
                for m in [warning_tx_cache, warning_tx_for_tx_hash, warning_spendables]:
                    if m:
                        print("warning: %s" % m, file=sys.stderr)
                parser.error("can't find Tx with id %s" % arg)
            txs.append(tx)
            continue

        # hex transaction data
        try:
            tx = Tx.from_hex(arg)
            txs.append(tx)
            continue
        except Exception:
            pass

        is_valid = is_address_valid(arg, allowable_netcodes=[args.network])
        if is_valid:
            payables.append((arg, 0))
            continue

        try:
            key = Key.from_text(arg)
            # TODO: check network
            if key.wif() is None:
                payables.append((key.address(), 0))
                continue
            # TODO: support paths to subkeys
            key_iters.append((k.wif() for k in key.subkeys("")))
            continue
        except Exception:
            pass

        if os.path.exists(arg):
            try:
                with open(arg, "rb") as f:
                    if f.name.endswith("hex"):
                        f = io.BytesIO(codecs.getreader("hex_codec")(f).read())
                    tx = Tx.parse(f)
                    txs.append(tx)
                    try:
                        tx.parse_unspents(f)
                    except Exception as ex:
                        pass
                    continue
            except Exception:
                pass

        parts = arg.split("/")
        if len(parts) == 4:
            # spendable
            try:
                spendables.append(Spendable.from_text(arg))
                continue
            except Exception:
                pass

        if len(parts) == 2 and is_address_valid(parts[0], allowable_netcodes=[args.network]):
            try:
                payables.append(parts)
                continue
            except ValueError:
                pass

        parser.error("can't parse %s" % arg)

    if args.fetch_spendables:
        warning_spendables = message_about_spendables_for_address_env(args.network)
        for address in args.fetch_spendables:
            spendables.extend(spendables_for_address(address))

    for tx in txs:
        if tx.missing_unspents() and args.augment:
            if tx_db is None:
                warning_tx_cache = message_about_tx_cache_env()
                warning_tx_for_tx_hash = message_about_tx_for_tx_hash_env(args.network)
                tx_db = get_tx_db(args.network)
            tx.unspents_from_db(tx_db, ignore_missing=True)

    return (txs, spendables, payables, key_iters, p2sh_lookup, tx_db, warning_tx_cache,
            warning_tx_for_tx_hash, warning_spendables)

Example 36

Project: fb2mobi
Source File: mobi_split.py
View license
    def __init__(self, infile, document_id, remove_personal_label, format):
        if format == 'mobi':
            datain = b''
            with open(pathof(infile), 'rb') as f:
                datain = f.read()
            datain_rec0 = readsection(datain, 0)
            ver = getint(datain_rec0, mobi_version)
            self.combo = (ver != 8)
            if not self.combo:
                return
            exth121 = read_exth(datain_rec0, 121)
            if len(exth121) == 0:
                self.combo = False
                return
            else:
                # only pay attention to first exth121
                # (there should only be one)
                datain_kf8, = struct.unpack_from(b'>L', exth121[0], 0)
                if datain_kf8 == 0xffffffff:
                    self.combo = False
                    return
            datain_kfrec0 = readsection(datain, datain_kf8)

            self.result_file = bytearray(datain)

            # check if there are SRCS records and reduce them
            srcs = getint(datain_rec0, srcs_index)
            num_srcs = getint(datain_rec0, srcs_count)
            if srcs != 0xffffffff and num_srcs > 0:
                for i in range(srcs, srcs + num_srcs):
                    self.result_file = nullsection(self.result_file, i)
                datain_rec0 = writeint(datain_rec0, srcs_index, 0xffffffff)
                datain_rec0 = writeint(datain_rec0, srcs_count, 0)

            if remove_personal_label:
                datain_rec0 = add_exth(datain_rec0, exth_cdetype, b"EBOK");

                exth = read_exth(datain_rec0, exth_asin)
                if len(exth) == 0:
                    datain_rec0 = add_exth(datain_rec0, exth_asin, bytes(to_base(document_id.int, base=32, min_num_digits=10), 'ascii'))

            self.result_file = writesection(self.result_file, 0, datain_rec0)

            firstimage = getint(datain_rec0, first_resc_record)

            # Only keep the correct EXTH 116 StartOffset, KG 2.5 carries over the one from the mobi7 part, which then points at garbage in the mobi8 part, and confuses FW 3.4
            kf8starts = read_exth(datain_kfrec0, 116)
            # If we have multiple StartOffset, keep only the last one
            kf8start_count = len(kf8starts)
            while kf8start_count > 1:
                kf8start_count -= 1
                datain_kfrec0 = del_exth(datain_kfrec0, 116)

            exth_cover = read_exth(datain_kfrec0, exth_cover_offset)
            if len(exth_cover) > 0:
                cover_index, = struct.unpack_from('>L', exth_cover[0], 0)
                cover_index += firstimage
            else:
                cover_index = 0xffffffff

            exth_thumb = read_exth(datain_kfrec0, exth_thumb_offset)
            if len(exth_thumb) > 0:
                thumb_index, = struct.unpack_from('>L', exth_thumb[0], 0)
                thumb_index += firstimage
            else:
                thumb_index = 0xffffffff

            if cover_index != 0xffffffff:
                if thumb_index != 0xffffffff:
                    # make sure embedded thumbnail has the right size
                    cover_image = readsection(datain, cover_index)
                    thumb = BytesIO()
                    im = Image.open(BytesIO(cover_image))
                    im.thumbnail((330, 470), Image.ANTIALIAS)
                    im.save(thumb, format=im.format)
                    self.result_file = writesection(self.result_file, thumb_index, thumb.getvalue())
                else:
                    # if nothing works - fall back to the old trick, set thumbnail to the cover image
                    datain_kfrec0 = add_exth(datain_kfrec0, exth_thumb_offset, exth_cover[0])
                    thumb_index = cover_index

                exth = read_exth(datain_kfrec0, exth_thumbnail_uri)
                if len(exth) > 0:
                    datain_kfrec0 = del_exth(datain_kfrec0, exth_thumbnail_uri)
                datain_kfrec0 = add_exth(datain_kfrec0, exth_thumbnail_uri, bytes('kindle:embed:%s' % (to_base(thumb_index - firstimage, base=32, min_num_digits=4)), 'ascii'))

            if remove_personal_label:
                datain_kfrec0 = add_exth(datain_kfrec0, exth_cdetype, b"EBOK");

                exth = read_exth(datain_kfrec0, exth_asin)
                if len(exth) == 0:
                    datain_kfrec0 = add_exth(datain_kfrec0, exth_asin, bytes(to_base(document_id.int, base=32, min_num_digits=10), 'ascii'))

            self.result_file = writesection(self.result_file, datain_kf8, datain_kfrec0)

        elif format == 'azw3':
            datain = b''
            with open(pathof(infile), 'rb') as f:
                datain = f.read()
            datain_rec0 = readsection(datain, 0)
            ver = getint(datain_rec0, mobi_version)
            self.combo = (ver != 8)
            if not self.combo:
                return
            exth121 = read_exth(datain_rec0, 121)
            if len(exth121) == 0:
                self.combo = False
                return
            else:
                # only pay attention to first exth121
                # (there should only be one)
                datain_kf8, = struct.unpack_from(b'>L', exth121[0], 0)
                if datain_kf8 == 0xffffffff:
                    self.combo = False
                    return

            datain_kfrec0 = readsection(datain, datain_kf8)

            # create the standalone mobi7
            num_sec = getint(datain, number_of_pdb_records, b'H')
            # remove BOUNDARY up to but not including ELF record
            self.result_file7 = deletesectionrange(datain, datain_kf8 - 1, num_sec - 2)
            # check if there are SRCS records and delete them
            srcs = getint(datain_rec0, srcs_index)
            num_srcs = getint(datain_rec0, srcs_count)
            if srcs != 0xffffffff and num_srcs > 0:
                self.result_file7 = deletesectionrange(self.result_file7, srcs, srcs + num_srcs - 1)
                datain_rec0 = writeint(datain_rec0, srcs_index, 0xffffffff)
                datain_rec0 = writeint(datain_rec0, srcs_count, 0)
            # reset the EXTH 121 KF8 Boundary meta data to 0xffffffff
            datain_rec0 = write_exth(datain_rec0, 121, struct.pack(b'>L', 0xffffffff))
            # datain_rec0 = del_exth(datain_rec0,121)
            # datain_rec0 = del_exth(datain_rec0,534)
            # don't remove the EXTH 125 KF8 Count of Resources, seems to be present in mobi6 files as well
            # set the EXTH 129 KF8 Masthead / Cover Image string to the null string
            datain_rec0 = write_exth(datain_rec0, 129, b'')
            # don't remove the EXTH 131 KF8 Unidentified Count, seems to be present in mobi6 files as well

            # need to reset flags stored in 0x80-0x83
            # old mobi with exth: 0x50, mobi7 part with exth: 0x1850, mobi8 part with exth: 0x1050
            # Bit Flags
            # 0x1000 = Bit 12 indicates if embedded fonts are used or not
            # 0x0800 = means this Header points to *shared* images/resource/fonts ??
            # 0x0080 = unknown new flag, why is this now being set by Kindlegen 2.8?
            # 0x0040 = exth exists
            # 0x0010 = Not sure but this is always set so far
            fval, = struct.unpack_from(b'>L', datain_rec0, 0x80)
            # need to remove flag 0x0800 for KindlePreviewer 2.8 and unset Bit 12 for embedded fonts
            fval = fval & 0x07FF
            datain_rec0 = datain_rec0[:0x80] + struct.pack(b'>L', fval) + datain_rec0[0x84:]

            self.result_file7 = writesection(self.result_file7, 0, datain_rec0)

            # no need to replace kf8 style fcis with mobi 7 one
            # fcis_secnum, = struct.unpack_from(b'>L',datain_rec0, 0xc8)
            # if fcis_secnum != 0xffffffff:
            #     fcis_info = readsection(datain, fcis_secnum)
            #     text_len,  = struct.unpack_from(b'>L', fcis_info, 0x14)
            #     new_fcis = 'FCIS\x00\x00\x00\x14\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00'
            #     new_fcis += struct.pack(b'>L',text_len)
            #     new_fcis += '\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x08\x00\x01\x00\x01\x00\x00\x00\x00'
            #     self.result_file7 = writesection(self.result_file7, fcis_secnum, new_fcis)

            firstimage = getint(datain_rec0, first_resc_record)
            lastimage = getint(datain_rec0, last_content_index, b'H')
            # print("Old First Image, last Image", firstimage,lastimage)
            if lastimage == 0xffff:
                # find the lowest of the next sections and copy up to that.
                ofs_list = [(fcis_index, b'L'), (flis_index, b'L'), (datp_index, b'L'), (hufftbloff, b'L')]
                for ofs, sz in ofs_list:
                    n = getint(datain_rec0, ofs, sz)
                    # print("n",n)
                    if n > 0 and n < lastimage:
                        lastimage = n - 1
            # print("First Image, last Image", firstimage,lastimage)

            # Try to null out FONT and RES, but leave the (empty) PDB record so image refs remain valid
            for i in range(firstimage, lastimage):
                imgsec = readsection(self.result_file7, i)
                if imgsec[0:4] in [b'RESC', b'FONT']:
                    self.result_file7 = nullsection(self.result_file7, i)

            # mobi7 finished

            # create standalone mobi8
            self.result_file8 = deletesectionrange(datain, 0, datain_kf8 - 1)
            target = getint(datain_kfrec0, first_resc_record)
            self.result_file8 = insertsectionrange(datain, firstimage, lastimage, self.result_file8, target)
            datain_kfrec0 = readsection(self.result_file8, 0)

            # Only keep the correct EXTH 116 StartOffset, KG 2.5 carries over the one from the mobi7 part, which then points at garbage in the mobi8 part, and confuses FW 3.4
            kf8starts = read_exth(datain_kfrec0, 116)
            # If we have multiple StartOffset, keep only the last one
            kf8start_count = len(kf8starts)
            while kf8start_count > 1:
                kf8start_count -= 1
                datain_kfrec0 = del_exth(datain_kfrec0, 116)

            # update the EXTH 125 KF8 Count of Images/Fonts/Resources
            datain_kfrec0 = write_exth(datain_kfrec0, 125, struct.pack(b'>L', lastimage - firstimage + 1))

            # need to reset flags stored in 0x80-0x83
            # old mobi with exth: 0x50, mobi7 part with exth: 0x1850, mobi8 part with exth: 0x1050
            # standalone mobi8 with exth: 0x0050
            # Bit Flags
            # 0x1000 = Bit 12 indicates if embedded fonts are used or not
            # 0x0800 = means this Header points to *shared* images/resource/fonts ??
            # 0x0080 = unknown new flag, why is this now being set by Kindlegen 2.8?
            # 0x0040 = exth exists
            # 0x0010 = Not sure but this is always set so far
            fval, = struct.unpack_from('>L', datain_kfrec0, 0x80)
            fval = fval & 0x1FFF
            fval |= 0x0800
            datain_kfrec0 = datain_kfrec0[:0x80] + struct.pack(b'>L', fval) + datain_kfrec0[0x84:]

            # properly update other index pointers that have been shifted by the insertion of images
            ofs_list = [(kf8_fdst_index, b'L'), (fcis_index, b'L'), (flis_index, b'L'), (datp_index, b'L'), (hufftbloff, b'L')]
            for ofs, sz in ofs_list:
                n = getint(datain_kfrec0, ofs, sz)
                if n != 0xffffffff:
                    datain_kfrec0 = writeint(datain_kfrec0, ofs, n + lastimage - firstimage + 1, sz)

            exth_cover = read_exth(datain_kfrec0, exth_cover_offset)
            if len(exth_cover) > 0:
                cover_index, = struct.unpack_from('>L', exth_cover[0], 0)
                cover_index += target
            else:
                cover_index = 0xffffffff

            exth_thumb = read_exth(datain_kfrec0, exth_thumb_offset)
            if len(exth_thumb) > 0:
                thumb_index, = struct.unpack_from('>L', exth_thumb[0], 0)
                thumb_index += target
            else:
                thumb_index = 0xffffffff

            if cover_index != 0xffffffff:
                if thumb_index != 0xffffffff:
                    # make sure embedded thumbnail has the right size
                    cover_image = readsection(self.result_file8, cover_index)
                    thumb = BytesIO()
                    im = Image.open(BytesIO(cover_image))
                    im.thumbnail((330, 470), Image.ANTIALIAS)
                    im.save(thumb, format=im.format)
                    self.result_file8 = writesection(self.result_file8, thumb_index, thumb.getvalue())
                else:
                    # if nothing works - fall back to the old trick, set thumbnail to the cover image
                    datain_kfrec0 = add_exth(datain_kfrec0, exth_thumb_offset, exth_cover[0])
                    thumb_index = cover_index

                exth = read_exth(datain_kfrec0, exth_thumbnail_uri)
                if len(exth) > 0:
                    datain_kfrec0 = del_exth(datain_kfrec0, exth_thumbnail_uri)
                datain_kfrec0 = add_exth(datain_kfrec0, exth_thumbnail_uri, bytes('kindle:embed:%s' % (to_base(thumb_index - target, base=32, min_num_digits=4)), 'ascii'))

            if remove_personal_label:
                datain_kfrec0 = add_exth(datain_kfrec0, exth_cdetype, b"EBOK");

                exth = read_exth(datain_kfrec0, exth_asin)
                if len(exth) == 0:
                    datain_kfrec0 = add_exth(datain_kfrec0, exth_asin, bytes(to_base(document_id.int, base=32, min_num_digits=10), 'ascii'))

            self.result_file8 = writesection(self.result_file8, 0, datain_kfrec0)

Example 37

View license
    def __init__(self, file_contents):
        self._patched_functions = {
            (TimerClass, 'addEventListener'): lambda params: undefined,
        }
        code_tag = next(tag
                        for tag_code, tag in _extract_tags(file_contents)
                        if tag_code == 82)
        p = code_tag.index(b'\0', 4) + 1
        code_reader = io.BytesIO(code_tag[p:])

        # Parse ABC (AVM2 ByteCode)

        # Define a couple convenience methods
        u30 = lambda *args: _u30(*args, reader=code_reader)
        s32 = lambda *args: _s32(*args, reader=code_reader)
        u32 = lambda *args: _u32(*args, reader=code_reader)
        read_bytes = lambda *args: _read_bytes(*args, reader=code_reader)
        read_byte = lambda *args: _read_byte(*args, reader=code_reader)

        # minor_version + major_version
        read_bytes(2 + 2)

        # Constant pool
        int_count = u30()
        self.constant_ints = [0]
        for _c in range(1, int_count):
            self.constant_ints.append(s32())
        self.constant_uints = [0]
        uint_count = u30()
        for _c in range(1, uint_count):
            self.constant_uints.append(u32())
        double_count = u30()
        read_bytes(max(0, (double_count - 1)) * 8)
        string_count = u30()
        self.constant_strings = ['']
        for _c in range(1, string_count):
            s = _read_string(code_reader)
            self.constant_strings.append(s)
        namespace_count = u30()
        for _c in range(1, namespace_count):
            read_bytes(1)  # kind
            u30()  # name
        ns_set_count = u30()
        for _c in range(1, ns_set_count):
            count = u30()
            for _c2 in range(count):
                u30()
        multiname_count = u30()
        MULTINAME_SIZES = {
            0x07: 2,  # QName
            0x0d: 2,  # QNameA
            0x0f: 1,  # RTQName
            0x10: 1,  # RTQNameA
            0x11: 0,  # RTQNameL
            0x12: 0,  # RTQNameLA
            0x09: 2,  # Multiname
            0x0e: 2,  # MultinameA
            0x1b: 1,  # MultinameL
            0x1c: 1,  # MultinameLA
        }
        self.multinames = ['']
        for _c in range(1, multiname_count):
            kind = u30()
            assert kind in MULTINAME_SIZES, 'Invalid multiname kind %r' % kind
            if kind == 0x07:
                u30()  # namespace_idx
                name_idx = u30()
                self.multinames.append(self.constant_strings[name_idx])
            elif kind == 0x09:
                name_idx = u30()
                u30()
                self.multinames.append(self.constant_strings[name_idx])
            else:
                self.multinames.append(_Multiname(kind))
                for _c2 in range(MULTINAME_SIZES[kind]):
                    u30()

        # Methods
        method_count = u30()
        MethodInfo = collections.namedtuple(
            'MethodInfo',
            ['NEED_ARGUMENTS', 'NEED_REST'])
        method_infos = []
        for method_id in range(method_count):
            param_count = u30()
            u30()  # return type
            for _ in range(param_count):
                u30()  # param type
            u30()  # name index (always 0 for youtube)
            flags = read_byte()
            if flags & 0x08 != 0:
                # Options present
                option_count = u30()
                for c in range(option_count):
                    u30()  # val
                    read_bytes(1)  # kind
            if flags & 0x80 != 0:
                # Param names present
                for _ in range(param_count):
                    u30()  # param name
            mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
            method_infos.append(mi)

        # Metadata
        metadata_count = u30()
        for _c in range(metadata_count):
            u30()  # name
            item_count = u30()
            for _c2 in range(item_count):
                u30()  # key
                u30()  # value

        def parse_traits_info():
            trait_name_idx = u30()
            kind_full = read_byte()
            kind = kind_full & 0x0f
            attrs = kind_full >> 4
            methods = {}
            constants = None
            if kind == 0x00:  # Slot
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                if vindex != 0:
                    read_byte()  # vkind
            elif kind == 0x06:  # Const
                u30()  # Slot id
                u30()  # type_name_idx
                vindex = u30()
                vkind = 'any'
                if vindex != 0:
                    vkind = read_byte()
                if vkind == 0x03:  # Constant_Int
                    value = self.constant_ints[vindex]
                elif vkind == 0x04:  # Constant_UInt
                    value = self.constant_uints[vindex]
                else:
                    return {}, None  # Ignore silently for now
                constants = {self.multinames[trait_name_idx]: value}
            elif kind in (0x01, 0x02, 0x03):  # Method / Getter / Setter
                u30()  # disp_id
                method_idx = u30()
                methods[self.multinames[trait_name_idx]] = method_idx
            elif kind == 0x04:  # Class
                u30()  # slot_id
                u30()  # classi
            elif kind == 0x05:  # Function
                u30()  # slot_id
                function_idx = u30()
                methods[function_idx] = self.multinames[trait_name_idx]
            else:
                raise ExtractorError('Unsupported trait kind %d' % kind)

            if attrs & 0x4 != 0:  # Metadata present
                metadata_count = u30()
                for _c3 in range(metadata_count):
                    u30()  # metadata index

            return methods, constants

        # Classes
        class_count = u30()
        classes = []
        for class_id in range(class_count):
            name_idx = u30()

            cname = self.multinames[name_idx]
            avm_class = _AVMClass(name_idx, cname)
            classes.append(avm_class)

            u30()  # super_name idx
            flags = read_byte()
            if flags & 0x08 != 0:  # Protected namespace is present
                u30()  # protected_ns_idx
            intrf_count = u30()
            for _c2 in range(intrf_count):
                u30()
            u30()  # iinit
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        assert len(classes) == class_count
        self._classes_by_name = dict((c.name, c) for c in classes)

        for avm_class in classes:
            avm_class.cinit_idx = u30()
            trait_count = u30()
            for _c2 in range(trait_count):
                trait_methods, trait_constants = parse_traits_info()
                avm_class.register_methods(trait_methods)
                if trait_constants:
                    avm_class.constants.update(trait_constants)

        # Scripts
        script_count = u30()
        for _c in range(script_count):
            u30()  # init
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        # Method bodies
        method_body_count = u30()
        Method = collections.namedtuple('Method', ['code', 'local_count'])
        self._all_methods = []
        for _c in range(method_body_count):
            method_idx = u30()
            u30()  # max_stack
            local_count = u30()
            u30()  # init_scope_depth
            u30()  # max_scope_depth
            code_length = u30()
            code = read_bytes(code_length)
            m = Method(code, local_count)
            self._all_methods.append(m)
            for avm_class in classes:
                if method_idx in avm_class.method_idxs:
                    avm_class.methods[avm_class.method_idxs[method_idx]] = m
            exception_count = u30()
            for _c2 in range(exception_count):
                u30()  # from
                u30()  # to
                u30()  # target
                u30()  # exc_type
                u30()  # var_name
            trait_count = u30()
            for _c2 in range(trait_count):
                parse_traits_info()

        assert p + code_reader.tell() == len(code_tag)

Example 38

View license
    def extract_function(self, avm_class, func_name):
        p = self._patched_functions.get((avm_class, func_name))
        if p:
            return p
        if func_name in avm_class.method_pyfunctions:
            return avm_class.method_pyfunctions[func_name]
        if func_name in self._classes_by_name:
            return self._classes_by_name[func_name].make_object()
        if func_name not in avm_class.methods:
            raise ExtractorError('Cannot find function %s.%s' % (
                avm_class.name, func_name))
        m = avm_class.methods[func_name]

        def resfunc(args):
            # Helper functions
            coder = io.BytesIO(m.code)
            s24 = lambda: _s24(coder)
            u30 = lambda: _u30(coder)

            registers = [avm_class.variables] + list(args) + [None] * m.local_count
            stack = []
            scopes = collections.deque([
                self._classes_by_name, avm_class.constants, avm_class.variables])
            while True:
                opcode = _read_byte(coder)
                if opcode == 9:  # label
                    pass  # Spec says: "Do nothing."
                elif opcode == 16:  # jump
                    offset = s24()
                    coder.seek(coder.tell() + offset)
                elif opcode == 17:  # iftrue
                    offset = s24()
                    value = stack.pop()
                    if value:
                        coder.seek(coder.tell() + offset)
                elif opcode == 18:  # iffalse
                    offset = s24()
                    value = stack.pop()
                    if not value:
                        coder.seek(coder.tell() + offset)
                elif opcode == 19:  # ifeq
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value2 == value1:
                        coder.seek(coder.tell() + offset)
                elif opcode == 20:  # ifne
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value2 != value1:
                        coder.seek(coder.tell() + offset)
                elif opcode == 21:  # iflt
                    offset = s24()
                    value2 = stack.pop()
                    value1 = stack.pop()
                    if value1 < value2:
                        coder.seek(coder.tell() + offset)
                elif opcode == 32:  # pushnull
                    stack.append(None)
                elif opcode == 33:  # pushundefined
                    stack.append(undefined)
                elif opcode == 36:  # pushbyte
                    v = _read_byte(coder)
                    stack.append(v)
                elif opcode == 37:  # pushshort
                    v = u30()
                    stack.append(v)
                elif opcode == 38:  # pushtrue
                    stack.append(True)
                elif opcode == 39:  # pushfalse
                    stack.append(False)
                elif opcode == 40:  # pushnan
                    stack.append(float('NaN'))
                elif opcode == 42:  # dup
                    value = stack[-1]
                    stack.append(value)
                elif opcode == 44:  # pushstring
                    idx = u30()
                    stack.append(self.constant_strings[idx])
                elif opcode == 48:  # pushscope
                    new_scope = stack.pop()
                    scopes.append(new_scope)
                elif opcode == 66:  # construct
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                    res = obj.avm_class.make_object()
                    stack.append(res)
                elif opcode == 70:  # callproperty
                    index = u30()
                    mname = self.multinames[index]
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()

                    if obj == StringClass:
                        if mname == 'String':
                            assert len(args) == 1
                            assert isinstance(args[0], (
                                int, compat_str, _Undefined))
                            if args[0] == undefined:
                                res = 'undefined'
                            else:
                                res = compat_str(args[0])
                            stack.append(res)
                            continue
                        else:
                            raise NotImplementedError(
                                'Function String.%s is not yet implemented'
                                % mname)
                    elif isinstance(obj, _AVMClass_Object):
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        stack.append(res)
                        continue
                    elif isinstance(obj, _AVMClass):
                        func = self.extract_function(obj, mname)
                        res = func(args)
                        stack.append(res)
                        continue
                    elif isinstance(obj, _ScopeDict):
                        if mname in obj.avm_class.method_names:
                            func = self.extract_function(obj.avm_class, mname)
                            res = func(args)
                        else:
                            res = obj[mname]
                        stack.append(res)
                        continue
                    elif isinstance(obj, compat_str):
                        if mname == 'split':
                            assert len(args) == 1
                            assert isinstance(args[0], compat_str)
                            if args[0] == '':
                                res = list(obj)
                            else:
                                res = obj.split(args[0])
                            stack.append(res)
                            continue
                        elif mname == 'charCodeAt':
                            assert len(args) <= 1
                            idx = 0 if len(args) == 0 else args[0]
                            assert isinstance(idx, int)
                            res = ord(obj[idx])
                            stack.append(res)
                            continue
                    elif isinstance(obj, list):
                        if mname == 'slice':
                            assert len(args) == 1
                            assert isinstance(args[0], int)
                            res = obj[args[0]:]
                            stack.append(res)
                            continue
                        elif mname == 'join':
                            assert len(args) == 1
                            assert isinstance(args[0], compat_str)
                            res = args[0].join(obj)
                            stack.append(res)
                            continue
                    raise NotImplementedError(
                        'Unsupported property %r on %r'
                        % (mname, obj))
                elif opcode == 71:  # returnvoid
                    res = undefined
                    return res
                elif opcode == 72:  # returnvalue
                    res = stack.pop()
                    return res
                elif opcode == 73:  # constructsuper
                    # Not yet implemented, just hope it works without it
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                elif opcode == 74:  # constructproperty
                    index = u30()
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()

                    mname = self.multinames[index]
                    assert isinstance(obj, _AVMClass)

                    # We do not actually call the constructor for now;
                    # we just pretend it does nothing
                    stack.append(obj.make_object())
                elif opcode == 79:  # callpropvoid
                    index = u30()
                    mname = self.multinames[index]
                    arg_count = u30()
                    args = list(reversed(
                        [stack.pop() for _ in range(arg_count)]))
                    obj = stack.pop()
                    if isinstance(obj, _AVMClass_Object):
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        assert res is undefined
                        continue
                    if isinstance(obj, _ScopeDict):
                        assert mname in obj.avm_class.method_names
                        func = self.extract_function(obj.avm_class, mname)
                        res = func(args)
                        assert res is undefined
                        continue
                    if mname == 'reverse':
                        assert isinstance(obj, list)
                        obj.reverse()
                    else:
                        raise NotImplementedError(
                            'Unsupported (void) property %r on %r'
                            % (mname, obj))
                elif opcode == 86:  # newarray
                    arg_count = u30()
                    arr = []
                    for i in range(arg_count):
                        arr.append(stack.pop())
                    arr = arr[::-1]
                    stack.append(arr)
                elif opcode == 93:  # findpropstrict
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            res = s
                            break
                    else:
                        res = scopes[0]
                    if mname not in res and mname in _builtin_classes:
                        stack.append(_builtin_classes[mname])
                    else:
                        stack.append(res[mname])
                elif opcode == 94:  # findproperty
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            res = s
                            break
                    else:
                        res = avm_class.variables
                    stack.append(res)
                elif opcode == 96:  # getlex
                    index = u30()
                    mname = self.multinames[index]
                    for s in reversed(scopes):
                        if mname in s:
                            scope = s
                            break
                    else:
                        scope = avm_class.variables

                    if mname in scope:
                        res = scope[mname]
                    elif mname in _builtin_classes:
                        res = _builtin_classes[mname]
                    else:
                        # Assume uninitialized
                        # TODO warn here
                        res = undefined
                    stack.append(res)
                elif opcode == 97:  # setproperty
                    index = u30()
                    value = stack.pop()
                    idx = self.multinames[index]
                    if isinstance(idx, _Multiname):
                        idx = stack.pop()
                    obj = stack.pop()
                    obj[idx] = value
                elif opcode == 98:  # getlocal
                    index = u30()
                    stack.append(registers[index])
                elif opcode == 99:  # setlocal
                    index = u30()
                    value = stack.pop()
                    registers[index] = value
                elif opcode == 102:  # getproperty
                    index = u30()
                    pname = self.multinames[index]
                    if pname == 'length':
                        obj = stack.pop()
                        assert isinstance(obj, (compat_str, list))
                        stack.append(len(obj))
                    elif isinstance(pname, compat_str):  # Member access
                        obj = stack.pop()
                        if isinstance(obj, _AVMClass):
                            res = obj.static_properties[pname]
                            stack.append(res)
                            continue

                        assert isinstance(obj, (dict, _ScopeDict)),\
                            'Accessing member %r on %r' % (pname, obj)
                        res = obj.get(pname, undefined)
                        stack.append(res)
                    else:  # Assume attribute access
                        idx = stack.pop()
                        assert isinstance(idx, int)
                        obj = stack.pop()
                        assert isinstance(obj, list)
                        stack.append(obj[idx])
                elif opcode == 104:  # initproperty
                    index = u30()
                    value = stack.pop()
                    idx = self.multinames[index]
                    if isinstance(idx, _Multiname):
                        idx = stack.pop()
                    obj = stack.pop()
                    obj[idx] = value
                elif opcode == 115:  # convert_
                    value = stack.pop()
                    intvalue = int(value)
                    stack.append(intvalue)
                elif opcode == 128:  # coerce
                    u30()
                elif opcode == 130:  # coerce_a
                    value = stack.pop()
                    # um, yes, it's any value
                    stack.append(value)
                elif opcode == 133:  # coerce_s
                    assert isinstance(stack[-1], (type(None), compat_str))
                elif opcode == 147:  # decrement
                    value = stack.pop()
                    assert isinstance(value, int)
                    stack.append(value - 1)
                elif opcode == 149:  # typeof
                    value = stack.pop()
                    return {
                        _Undefined: 'undefined',
                        compat_str: 'String',
                        int: 'Number',
                        float: 'Number',
                    }[type(value)]
                elif opcode == 160:  # add
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 + value2
                    stack.append(res)
                elif opcode == 161:  # subtract
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 - value2
                    stack.append(res)
                elif opcode == 162:  # multiply
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 * value2
                    stack.append(res)
                elif opcode == 164:  # modulo
                    value2 = stack.pop()
                    value1 = stack.pop()
                    res = value1 % value2
                    stack.append(res)
                elif opcode == 168:  # bitand
                    value2 = stack.pop()
                    value1 = stack.pop()
                    assert isinstance(value1, int)
                    assert isinstance(value2, int)
                    res = value1 & value2
                    stack.append(res)
                elif opcode == 171:  # equals
                    value2 = stack.pop()
                    value1 = stack.pop()
                    result = value1 == value2
                    stack.append(result)
                elif opcode == 175:  # greaterequals
                    value2 = stack.pop()
                    value1 = stack.pop()
                    result = value1 >= value2
                    stack.append(result)
                elif opcode == 192:  # increment_i
                    value = stack.pop()
                    assert isinstance(value, int)
                    stack.append(value + 1)
                elif opcode == 208:  # getlocal_0
                    stack.append(registers[0])
                elif opcode == 209:  # getlocal_1
                    stack.append(registers[1])
                elif opcode == 210:  # getlocal_2
                    stack.append(registers[2])
                elif opcode == 211:  # getlocal_3
                    stack.append(registers[3])
                elif opcode == 212:  # setlocal_0
                    registers[0] = stack.pop()
                elif opcode == 213:  # setlocal_1
                    registers[1] = stack.pop()
                elif opcode == 214:  # setlocal_2
                    registers[2] = stack.pop()
                elif opcode == 215:  # setlocal_3
                    registers[3] = stack.pop()
                else:
                    raise NotImplementedError(
                        'Unsupported opcode %d' % opcode)

        avm_class.method_pyfunctions[func_name] = resfunc
        return resfunc

Example 39

Project: saxo
Source File: web.py
View license
def request(url, query=None, data=None, method="GET",
        limit=None, follow=True, headers=None, modern=False):
    if url.startswith("file:"):
        raise ValueError("file: scheme is not allowed")
    headers = {} if (headers is None) else headers

    response = {}
    if "User-Agent" not in headers:
        modern = modern is True
        headers["User-Agent"] = modern_user_agent if modern else user_agent
    response["request-headers"] = headers

    parts = list(urllib.parse.urlparse(url))
    try: parts[1].encode("ascii")
    except UnicodeEncodeError:
        parts[1] = parts[1].encode("idna").decode("ascii")
        url = urllib.parse.urlunparse(tuple(parts))

    safe = "".join(chr(i) for i in range(0x01, 0x80))
    base = urllib.parse.quote(url, safe=safe)
    if query:
        query = urllib.parse.urlencode(query)
        response["request-url"] = "?".join((base, query))
    else:
        response["request-url"] = base

    class ErrorHandler(urllib.request.HTTPDefaultErrorHandler):
        def http_error_default(self, req, fp, code, msg, hdrs):
            return fp

    handlers = [ErrorHandler()]
    if follow:
        handlers.append(urllib.request.HTTPRedirectHandler())

    opener = urllib.request.build_opener(*handlers)
    urllib.request.install_opener(opener)

    params = {
        "url": response["request-url"],
        "headers": response["request-headers"]
    }

    if data is not None:
        if isinstance(data, dict):
            data = urllib.parse.urlencode(data)
            params["data"] = data.encode("utf-8", "replace")
        elif isinstance(data, bytes):
            params["data"] = data
        elif isinstance(data, str):
            params["data"] = data.encode("utf-8", "replace")
        else:
            raise Exception("Unknown data type: %s" % type(data))

    req = urllib.request.Request(**params)
    with urllib.request.urlopen(req) as res:
        response["url"] = res.url
        response["status"] = res.status # int
        response["info"] = res.info()
        response["headers"] = {
            a.lower(): b for (a, b) in response["info"].items()
        }

        if method in {"GET", "POST"}:
            if limit is None:
                response["octets"] = res.read()
            else:
                response["octets"] = res.read(limit)

    if "Content-Encoding" in response["info"]:
        if response["info"]["Content-Encoding"] == "gzip":
            from gzip import GzipFile
            from io import BytesIO
            sio = BytesIO(response["octets"])
            gz = GzipFile(fileobj=sio)
            try: response["octets"] = gz.read()
            except OSError:
                # e.g. not gzip encoded, despite the site saying it is
                ...

    mime, encoding = content_type(response["info"])
    if mime:
        response["mime"] = mime
    if encoding:
        response["encoding"] = encoding
        response["encoding-source"] = "Content-Type"

    if mime and ("octets" in response):
        if ("/html" in mime) or ("/xhtml" in mime):
            search = _regex_charset.search(response["octets"])
            if search:
                html_encoding = search.group(1).lower()
                html_encoding = html_encoding.decode("ascii", "replace")

                if encoding and (encoding == html_encoding):
                        response["encoding-source"] += ", HTML"
                else: # TODO: Precedence check
                    response["encoding"] = html_encoding
                    response["encoding-source"] = "HTML"

    if "octets" in response:
        def guess_encoding(response):
            try: response["text"] = response["octets"].decode("utf-8")
            except UnicodeDecodeError:
                response["text"] = response["octets"].decode("iso-8859-1")
                response["encoding"] = "iso-8859-1"
            else:
                response["encoding"] = "utf-8"
            response["encoding-source"] = "heuristic"

        if "encoding" in response:
            encoding = response["encoding"]
            try: response["text"] = response["octets"].decode(encoding)
            except (UnicodeDecodeError, LookupError):
                guess_encoding(response)
        else:
            guess_encoding(response)

    if mime and ("text" in response):
        if ("/html" in mime) or ("/xhtml" in mime):
                response["html"] = response["text"]
                response["text"] = decode_entities(response["text"])
                response["decoded-entities"] = True

    return response

Example 40

Project: scikit-learn
Source File: kddcup99.py
View license
def _fetch_brute_kddcup99(subset=None, data_home=None,
                          download_if_missing=True, random_state=None,
                          shuffle=False, percent10=False):

    """Load the kddcup99 dataset, downloading it if necessary.

    Parameters
    ----------
    subset : None, 'SA', 'SF', 'http', 'smtp'
        To return the corresponding classical subsets of kddcup 99.
        If None, return the entire kddcup 99 dataset.

    data_home : string, optional
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    download_if_missing : boolean, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    random_state : int, RandomState instance or None, optional (default=None)
        Random state for shuffling the dataset.
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    shuffle : bool, default=False
        Whether to shuffle dataset.

    percent10 : bool, default=False
        Whether to load only 10 percent of the data.

    Returns
    -------
    dataset : dict-like object with the following attributes:
        dataset.data : numpy array of shape (494021, 41)
            Each row corresponds to the 41 features in the dataset.
        dataset.target : numpy array of shape (494021,)
            Each value corresponds to one of the 21 attack types or to the
            label 'normal.'.
        dataset.DESCR : string
            Description of the kddcup99 dataset.

    """

    data_home = get_data_home(data_home=data_home)
    if sys.version_info[0] == 3:
        # The zlib compression format use by joblib is not compatible when
        # switching from Python 2 to Python 3, let us use a separate folder
        # under Python 3:
        dir_suffix = "-py3"
    else:
        # Backward compat for Python 2 users
        dir_suffix = ""
    if percent10:
        kddcup_dir = join(data_home, "kddcup99_10" + dir_suffix)
    else:
        kddcup_dir = join(data_home, "kddcup99" + dir_suffix)
    samples_path = join(kddcup_dir, "samples")
    targets_path = join(kddcup_dir, "targets")
    available = exists(samples_path)

    if download_if_missing and not available:
        _mkdirp(kddcup_dir)
        URL_ = URL10 if percent10 else URL
        logger.warning("Downloading %s" % URL_)
        f = BytesIO(urlopen(URL_).read())

        dt = [('duration', int),
              ('protocol_type', 'S4'),
              ('service', 'S11'),
              ('flag', 'S6'),
              ('src_bytes', int),
              ('dst_bytes', int),
              ('land', int),
              ('wrong_fragment', int),
              ('urgent', int),
              ('hot', int),
              ('num_failed_logins', int),
              ('logged_in', int),
              ('num_compromised', int),
              ('root_shell', int),
              ('su_attempted', int),
              ('num_root', int),
              ('num_file_creations', int),
              ('num_shells', int),
              ('num_access_files', int),
              ('num_outbound_cmds', int),
              ('is_host_login', int),
              ('is_guest_login', int),
              ('count', int),
              ('srv_count', int),
              ('serror_rate', float),
              ('srv_serror_rate', float),
              ('rerror_rate', float),
              ('srv_rerror_rate', float),
              ('same_srv_rate', float),
              ('diff_srv_rate', float),
              ('srv_diff_host_rate', float),
              ('dst_host_count', int),
              ('dst_host_srv_count', int),
              ('dst_host_same_srv_rate', float),
              ('dst_host_diff_srv_rate', float),
              ('dst_host_same_src_port_rate', float),
              ('dst_host_srv_diff_host_rate', float),
              ('dst_host_serror_rate', float),
              ('dst_host_srv_serror_rate', float),
              ('dst_host_rerror_rate', float),
              ('dst_host_srv_rerror_rate', float),
              ('labels', 'S16')]
        DT = np.dtype(dt)

        file_ = GzipFile(fileobj=f, mode='r')
        Xy = []
        for line in file_.readlines():
            if six.PY3:
                line = line.decode()
            Xy.append(line.replace('\n', '').split(','))
        file_.close()
        print('extraction done')
        Xy = np.asarray(Xy, dtype=object)
        for j in range(42):
            Xy[:, j] = Xy[:, j].astype(DT[j])

        X = Xy[:, :-1]
        y = Xy[:, -1]
        # XXX bug when compress!=0:
        # (error: 'Incorrect data length while decompressing[...] the file
        #  could be corrupted.')

        joblib.dump(X, samples_path, compress=0)
        joblib.dump(y, targets_path, compress=0)

    try:
        X, y
    except NameError:
        X = joblib.load(samples_path)
        y = joblib.load(targets_path)

    if shuffle:
        X, y = shuffle_method(X, y, random_state=random_state)

    return Bunch(data=X, target=y, DESCR=__doc__)

Example 41

Project: scikit-learn
Source File: rcv1.py
View license
def fetch_rcv1(data_home=None, subset='all', download_if_missing=True,
               random_state=None, shuffle=False):
    """Load the RCV1 multilabel dataset, downloading it if necessary.

    Version: RCV1-v2, vectors, full sets, topics multilabels.

    ==============     =====================
    Classes                              103
    Samples total                     804414
    Dimensionality                     47236
    Features           real, between 0 and 1
    ==============     =====================

    Read more in the :ref:`User Guide <datasets>`.

    .. versionadded:: 0.17

    Parameters
    ----------
    data_home : string, optional
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    subset: string, 'train', 'test', or 'all', default='all'
        Select the dataset to load: 'train' for the training set
        (23149 samples), 'test' for the test set (781265 samples),
        'all' for both, with the training samples first if shuffle is False.
        This follows the official LYRL2004 chronological split.

    download_if_missing : boolean, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    random_state : int, RandomState instance or None, optional (default=None)
        Random state for shuffling the dataset.
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    shuffle : bool, default=False
        Whether to shuffle dataset.

    Returns
    -------
    dataset : dict-like object with the following attributes:

    dataset.data : scipy csr array, dtype np.float64, shape (804414, 47236)
        The array has 0.16% of non zero values.

    dataset.target : scipy csr array, dtype np.uint8, shape (804414, 103)
        Each sample has a value of 1 in its categories, and 0 in others.
        The array has 3.15% of non zero values.

    dataset.sample_id : numpy array, dtype np.uint32, shape (804414,)
        Identification number of each sample, as ordered in dataset.data.

    dataset.target_names : numpy array, dtype object, length (103)
        Names of each target (RCV1 topics), as ordered in dataset.target.

    dataset.DESCR : string
        Description of the RCV1 dataset.

    References
    ----------
    Lewis, D. D., Yang, Y., Rose, T. G., & Li, F. (2004). RCV1: A new
    benchmark collection for text categorization research. The Journal of
    Machine Learning Research, 5, 361-397.

    """
    N_SAMPLES = 804414
    N_FEATURES = 47236
    N_CATEGORIES = 103
    N_TRAIN = 23149

    data_home = get_data_home(data_home=data_home)
    rcv1_dir = join(data_home, "RCV1")
    if download_if_missing:
        makedirs(rcv1_dir, exist_ok=True)

    samples_path = _pkl_filepath(rcv1_dir, "samples.pkl")
    sample_id_path = _pkl_filepath(rcv1_dir, "sample_id.pkl")
    sample_topics_path = _pkl_filepath(rcv1_dir, "sample_topics.pkl")
    topics_path = _pkl_filepath(rcv1_dir, "topics_names.pkl")

    # load data (X) and sample_id
    if download_if_missing and (not exists(samples_path) or
                                not exists(sample_id_path)):
        file_urls = ["%s_test_pt%d.dat.gz" % (URL, i) for i in range(4)]
        file_urls.append("%s_train.dat.gz" % URL)
        files = []
        for file_url in file_urls:
            logger.warning("Downloading %s" % file_url)
            with closing(urlopen(file_url)) as online_file:
                # buffer the full file in memory to make possible to Gzip to
                # work correctly
                f = BytesIO(online_file.read())
            files.append(GzipFile(fileobj=f))

        Xy = load_svmlight_files(files, n_features=N_FEATURES)

        # Training data is before testing data
        X = sp.vstack([Xy[8], Xy[0], Xy[2], Xy[4], Xy[6]]).tocsr()
        sample_id = np.hstack((Xy[9], Xy[1], Xy[3], Xy[5], Xy[7]))
        sample_id = sample_id.astype(np.uint32)

        joblib.dump(X, samples_path, compress=9)
        joblib.dump(sample_id, sample_id_path, compress=9)

    else:
        X = joblib.load(samples_path)
        sample_id = joblib.load(sample_id_path)

    # load target (y), categories, and sample_id_bis
    if download_if_missing and (not exists(sample_topics_path) or
                                not exists(topics_path)):
        logger.warning("Downloading %s" % URL_topics)
        with closing(urlopen(URL_topics)) as online_topics:
            f = BytesIO(online_topics.read())

        # parse the target file
        n_cat = -1
        n_doc = -1
        doc_previous = -1
        y = np.zeros((N_SAMPLES, N_CATEGORIES), dtype=np.uint8)
        sample_id_bis = np.zeros(N_SAMPLES, dtype=np.int32)
        category_names = {}
        for line in GzipFile(fileobj=f, mode='rb'):
            line_components = line.decode("ascii").split(u" ")
            if len(line_components) == 3:
                cat, doc, _ = line_components
                if cat not in category_names:
                    n_cat += 1
                    category_names[cat] = n_cat

                doc = int(doc)
                if doc != doc_previous:
                    doc_previous = doc
                    n_doc += 1
                    sample_id_bis[n_doc] = doc
                y[n_doc, category_names[cat]] = 1

        # Samples in X are ordered with sample_id,
        # whereas in y, they are ordered with sample_id_bis.
        permutation = _find_permutation(sample_id_bis, sample_id)
        y = y[permutation, :]

        # save category names in a list, with same order than y
        categories = np.empty(N_CATEGORIES, dtype=object)
        for k in category_names.keys():
            categories[category_names[k]] = k

        # reorder categories in lexicographic order
        order = np.argsort(categories)
        categories = categories[order]
        y = sp.csr_matrix(y[:, order])

        joblib.dump(y, sample_topics_path, compress=9)
        joblib.dump(categories, topics_path, compress=9)

    else:
        y = joblib.load(sample_topics_path)
        categories = joblib.load(topics_path)

    if subset == 'all':
        pass
    elif subset == 'train':
        X = X[:N_TRAIN, :]
        y = y[:N_TRAIN, :]
        sample_id = sample_id[:N_TRAIN]
    elif subset == 'test':
        X = X[N_TRAIN:, :]
        y = y[N_TRAIN:, :]
        sample_id = sample_id[N_TRAIN:]
    else:
        raise ValueError("Unknown subset parameter. Got '%s' instead of one"
                         " of ('all', 'train', test')" % subset)

    if shuffle:
        X, y, sample_id = shuffle_(X, y, sample_id, random_state=random_state)

    return Bunch(data=X, target=y, sample_id=sample_id,
                 target_names=categories, DESCR=__doc__)

Example 42

View license
def fetch_species_distributions(data_home=None,
                                download_if_missing=True):
    """Loader for species distribution dataset from Phillips et. al. (2006)

    Read more in the :ref:`User Guide <datasets>`.

    Parameters
    ----------
    data_home : optional, default: None
        Specify another download and cache folder for the datasets. By default
        all scikit learn data is stored in '~/scikit_learn_data' subfolders.

    download_if_missing : optional, True by default
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    --------
    The data is returned as a Bunch object with the following attributes:

    coverages : array, shape = [14, 1592, 1212]
        These represent the 14 features measured at each point of the map grid.
        The latitude/longitude values for the grid are discussed below.
        Missing data is represented by the value -9999.

    train : record array, shape = (1623,)
        The training points for the data.  Each point has three fields:

        - train['species'] is the species name
        - train['dd long'] is the longitude, in degrees
        - train['dd lat'] is the latitude, in degrees

    test : record array, shape = (619,)
        The test points for the data.  Same format as the training data.

    Nx, Ny : integers
        The number of longitudes (x) and latitudes (y) in the grid

    x_left_lower_corner, y_left_lower_corner : floats
        The (x,y) position of the lower-left corner, in degrees

    grid_size : float
        The spacing between points of the grid, in degrees

    Notes
    ------

    This dataset represents the geographic distribution of species.
    The dataset is provided by Phillips et. al. (2006).

    The two species are:

    - `"Bradypus variegatus"
      <http://www.iucnredlist.org/details/3038/0>`_ ,
      the Brown-throated Sloth.

    - `"Microryzomys minutus"
      <http://www.iucnredlist.org/details/13408/0>`_ ,
      also known as the Forest Small Rice Rat, a rodent that lives in Peru,
      Colombia, Ecuador, Peru, and Venezuela.

    References
    ----------

    * `"Maximum entropy modeling of species geographic distributions"
      <http://www.cs.princeton.edu/~schapire/papers/ecolmod.pdf>`_
      S. J. Phillips, R. P. Anderson, R. E. Schapire - Ecological Modelling,
      190:231-259, 2006.

    Notes
    -----

    * See examples/applications/plot_species_distribution_modeling.py
      for an example of using this dataset with scikit-learn

    """
    data_home = get_data_home(data_home)
    if not exists(data_home):
        makedirs(data_home)

    # Define parameters for the data files.  These should not be changed
    # unless the data model changes.  They will be saved in the npz file
    # with the downloaded data.
    extra_params = dict(x_left_lower_corner=-94.8,
                        Nx=1212,
                        y_left_lower_corner=-56.05,
                        Ny=1592,
                        grid_size=0.05)
    dtype = np.int16

    archive_path = _pkl_filepath(data_home, DATA_ARCHIVE_NAME)

    if not exists(archive_path):
        print('Downloading species data from %s to %s' % (SAMPLES_URL,
                                                          data_home))
        X = np.load(BytesIO(urlopen(SAMPLES_URL).read()))

        for f in X.files:
            fhandle = BytesIO(X[f])
            if 'train' in f:
                train = _load_csv(fhandle)
            if 'test' in f:
                test = _load_csv(fhandle)

        print('Downloading coverage data from %s to %s' % (COVERAGES_URL,
                                                           data_home))

        X = np.load(BytesIO(urlopen(COVERAGES_URL).read()))

        coverages = []
        for f in X.files:
            fhandle = BytesIO(X[f])
            print(' - converting', f)
            coverages.append(_load_coverage(fhandle))
        coverages = np.asarray(coverages, dtype=dtype)

        bunch = Bunch(coverages=coverages,
                      test=test,
                      train=train,
                      **extra_params)
        joblib.dump(bunch, archive_path, compress=9)
    else:
        bunch = joblib.load(archive_path)

    return bunch

Example 43

Project: cartopy
Source File: ogc_clients.py
View license
    def _wmts_images(self, wmts, layer, matrix_set_name, extent,
                     max_pixel_span):
        """
        Add images from the specified WMTS layer and matrix set to cover
        the specified extent at an appropriate resolution.

        The zoom level (aka. tile matrix) is chosen to give the lowest
        possible resolution which still provides the requested quality.
        If insufficient resolution is available, the highest available
        resolution is used.

        Args:

            * wmts - The owslib.wmts.WebMapTileService providing the tiles.
            * layer - The owslib.wmts.ContentMetadata (aka. layer) to draw.
            * matrix_set_name - The name of the matrix set to use.
            * extent - Tuple of (left, right, bottom, top) in Axes coordinates.
            * max_pixel_span - Preferred maximum pixel width or height
                               in Axes coordinates.

        """

        # Find which tile matrix has the appropriate resolution.
        tile_matrix_set = wmts.tilematrixsets[matrix_set_name]
        tile_matrices = tile_matrix_set.tilematrix.values()
        meters_per_unit = METERS_PER_UNIT[tile_matrix_set.crs]
        tile_matrix = self._choose_matrix(tile_matrices, meters_per_unit,
                                          max_pixel_span)

        # Determine which tiles are required to cover the requested extent.
        tile_span_x, tile_span_y = self._tile_span(tile_matrix,
                                                   meters_per_unit)
        tile_matrix_set_links = getattr(layer, 'tilematrixsetlinks', None)
        if tile_matrix_set_links is None:
            tile_matrix_limits = None
        else:
            tile_matrix_set_link = tile_matrix_set_links[matrix_set_name]
            tile_matrix_limits = tile_matrix_set_link.tilematrixlimits.get(
                tile_matrix.identifier)
        min_col, max_col, min_row, max_row = self._select_tiles(
            tile_matrix, tile_matrix_limits, tile_span_x, tile_span_y, extent)

        # Find the relevant section of the image cache.
        tile_matrix_id = tile_matrix.identifier
        cache_by_wmts = WMTSRasterSource._shared_image_cache
        cache_by_layer_matrix = cache_by_wmts.setdefault(wmts, {})
        image_cache = cache_by_layer_matrix.setdefault((layer.id,
                                                        tile_matrix_id), {})

        # To avoid nasty seams between the individual tiles, we
        # accumulate the tile images into a single image.
        big_img = None
        n_rows = 1 + max_row - min_row
        n_cols = 1 + max_col - min_col
        # Ignore out-of-range errors if the current version of OWSLib
        # doesn't provide the regional information.
        ignore_out_of_range = tile_matrix_set_links is None
        for row in range(min_row, max_row + 1):
            for col in range(min_col, max_col + 1):
                # Get the tile's Image from the cache if possible.
                img_key = (row, col)
                img = image_cache.get(img_key)
                if img is None:
                    try:
                        tile = wmts.gettile(
                            layer=layer.id,
                            tilematrixset=matrix_set_name,
                            tilematrix=tile_matrix_id,
                            row=row, column=col,
                            **self.gettile_extra_kwargs)
                    except owslib.util.ServiceException as exception:
                        if ('TileOutOfRange' in exception.message and
                                ignore_out_of_range):
                            continue
                        raise exception
                    img = Image.open(io.BytesIO(tile.read()))
                    image_cache[img_key] = img
                if big_img is None:
                    size = (img.size[0] * n_cols, img.size[1] * n_rows)
                    big_img = Image.new('RGBA', size, (255, 255, 255, 255))
                top = (row - min_row) * tile_matrix.tileheight
                left = (col - min_col) * tile_matrix.tilewidth
                big_img.paste(img, (left, top))

        if big_img is None:
            img_extent = None
        else:
            matrix_min_x, matrix_max_y = tile_matrix.topleftcorner
            min_img_x = matrix_min_x + tile_span_x * min_col
            max_img_y = matrix_max_y - tile_span_y * min_row
            img_extent = (min_img_x, min_img_x + n_cols * tile_span_x,
                          max_img_y - n_rows * tile_span_y, max_img_y)
        return big_img, img_extent

Example 44

Project: iris
Source File: __init__.py
View license
    def _assert_graphic(self, tol=_HAMMING_DISTANCE):
        """
        Check the hash of the current matplotlib figure matches the expected
        image hash for the current graphic test.

        To create missing image test results, set the IRIS_TEST_CREATE_MISSING
        environment variable before running the tests. This will result in new
        and appropriately "<hash>.png" image files being generated in the image
        output directory, and the imagerepo.json file being updated.

        """
        import imagehash
        from PIL import Image

        dev_mode = os.environ.get('IRIS_TEST_CREATE_MISSING')
        unique_id = self._unique_id()
        repo_fname = os.path.join(_RESULT_PATH, 'imagerepo.json')
        with open(repo_fname, 'rb') as fi:
            repo = json.load(codecs.getreader('utf-8')(fi))

        try:
            #: The path where the images generated by the tests should go.
            image_output_directory = os.path.join(os.path.dirname(__file__),
                                                  'result_image_comparison')
            if not os.access(image_output_directory, os.W_OK):
                if not os.access(os.getcwd(), os.W_OK):
                    raise IOError('Write access to a local disk is required '
                                  'to run image tests.  Run the tests from a '
                                  'current working directory you have write '
                                  'access to to avoid this issue.')
                else:
                    image_output_directory = os.path.join(
                        os.getcwd(), 'iris_image_test_output')
            result_fname = os.path.join(image_output_directory,
                                        'result-' + unique_id + '.png')

            if not os.path.isdir(image_output_directory):
                # Handle race-condition where the directories are
                # created sometime between the check above and the
                # creation attempt below.
                try:
                    os.makedirs(image_output_directory)
                except OSError as err:
                    # Don't care about "File exists"
                    if err.errno != 17:
                        raise

            def _create_missing():
                fname = '{}.png'.format(phash)
                base_uri = ('https://scitools.github.io/test-iris-imagehash/'
                            'images/{}')
                uri = base_uri.format(fname)
                hash_fname = os.path.join(image_output_directory, fname)
                uris = repo.setdefault(unique_id, [])
                uris.append(uri)
                print('Creating image file: {}'.format(hash_fname))
                figure.savefig(hash_fname)
                msg = 'Creating imagerepo entry: {} -> {}'
                print(msg.format(unique_id, uri))
                with open(repo_fname, 'wb') as fo:
                    json.dump(repo, codecs.getwriter('utf-8')(fo), indent=4,
                              sort_keys=True)

            # TBD: Push this fix to imagehash (done!)
            # See https://github.com/JohannesBuchner/imagehash/pull/31
            # Now need this imagehash/master pushed to pypi ...
            def _hex_to_hash(hexstr, hash_size=_HASH_SIZE):
                l = []
                count = hash_size * (hash_size // 4)
                if len(hexstr) != count:
                    emsg = 'Expected hex string size of {}.'
                    raise ValueError(emsg.format(count))
                for i in range(count // 2):
                    h = hexstr[i*2:i*2+2]
                    v = int("0x" + h, 16)
                    l.append([v & 2**i > 0 for i in range(8)])
                return imagehash.ImageHash(np.array(l))

            # Calculate the test result perceptual image hash.
            buffer = io.BytesIO()
            figure = plt.gcf()
            figure.savefig(buffer, format='png')
            buffer.seek(0)
            phash = imagehash.phash(Image.open(buffer), hash_size=_HASH_SIZE)

            if unique_id not in repo:
                if dev_mode:
                    _create_missing()
                else:
                    figure.savefig(result_fname)
                    emsg = 'Missing image test result: {}.'
                    raise ValueError(emsg.format(unique_id))
            else:
                uris = repo[unique_id]
                # Create the expected perceptual image hashes from the uris.
                expected = [_hex_to_hash(os.path.splitext(os.path.basename(uri))[0])
                            for uri in uris]

                # Calculate the hamming distance vector for the result hash.
                distances = [e - phash for e in expected]

                if np.all([hd > tol for hd in distances]):
                    if dev_mode:
                        _create_missing()
                    else:
                        figure.savefig(result_fname)
                        msg = ('Bad phash {} with hamming distance {} '
                               'for test {}.')
                        msg = msg.format(phash, distances, unique_id)
                        if _DISPLAY_FIGURES:
                            emsg = 'Image comparion would have failed: {}'
                            print(emsg.format(msg))
                        else:
                            emsg = 'Image comparison failed: {}'
                            raise ValueError(emsg.format(msg))

            if _DISPLAY_FIGURES:
                plt.show()

        finally:
            plt.close()

Example 45

Project: frontera
Source File: hbase.py
View license
    def get_next_requests(self, max_n_requests, partition_id, **kwargs):
        """
        Tries to get new batch from priority queue. It makes self.GET_RETRIES tries and stops, trying to fit all
        parameters. Every new iteration evaluates a deeper batch. After batch is requested it is removed from the queue.

        :param max_n_requests: maximum number of requests
        :param partition_id: partition id to get batch from
        :param min_requests: minimum number of requests
        :param min_hosts: minimum number of hosts
        :param max_requests_per_host: maximum number of requests per host
        :return: list of :class:`Request <frontera.core.models.Request>` objects.
        """
        min_requests = kwargs.pop('min_requests')
        min_hosts = kwargs.pop('min_hosts')
        max_requests_per_host = kwargs.pop('max_requests_per_host')
        assert(max_n_requests > min_requests)
        table = self.connection.table(self.table_name)

        meta_map = {}
        queue = {}
        limit = min_requests
        tries = 0
        count = 0
        prefix = '%d_' % partition_id
        now_ts = int(time())
        filter = "PrefixFilter ('%s') AND SingleColumnValueFilter ('f', 't', <=, 'binary:%d')" % (prefix, now_ts)
        while tries < self.GET_RETRIES:
            tries += 1
            limit *= 5.5 if tries > 1 else 1.0
            self.logger.debug("Try %d, limit %d, last attempt: requests %d, hosts %d",
                              tries, limit, count, len(queue.keys()))
            meta_map.clear()
            queue.clear()
            count = 0
            for rk, data in table.scan(limit=int(limit), batch_size=256, filter=filter):
                for cq, buf in six.iteritems(data):
                    if cq == b'f:t':
                        continue
                    stream = BytesIO(buf)
                    unpacker = Unpacker(stream)
                    for item in unpacker:
                        fprint, host_crc32, _, _ = item
                        if host_crc32 not in queue:
                            queue[host_crc32] = []
                        if max_requests_per_host is not None and len(queue[host_crc32]) > max_requests_per_host:
                            continue
                        queue[host_crc32].append(fprint)
                        count += 1

                        if fprint not in meta_map:
                            meta_map[fprint] = []
                        meta_map[fprint].append((rk, item))
                if count > max_n_requests:
                    break

            if min_hosts is not None and len(queue.keys()) < min_hosts:
                continue

            if count < min_requests:
                continue
            break

        self.logger.debug("Finished: tries %d, hosts %d, requests %d", tries, len(queue.keys()), count)

        # For every fingerprint collect it's row keys and return all fingerprints from them
        fprint_map = {}
        for fprint, meta_list in six.iteritems(meta_map):
            for rk, _ in meta_list:
                fprint_map.setdefault(rk, []).append(fprint)

        results = []
        trash_can = set()

        for _, fprints in six.iteritems(queue):
            for fprint in fprints:
                for rk, _ in meta_map[fprint]:
                    if rk in trash_can:
                        continue
                    for rk_fprint in fprint_map[rk]:
                        _, item = meta_map[rk_fprint][0]
                        _, _, encoded, score = item
                        request = self.decoder.decode_request(encoded)
                        request.meta[b'score'] = score
                        results.append(request)
                    trash_can.add(rk)

        with table.batch(transaction=True) as b:
            for rk in trash_can:
                b.delete(rk)
        self.logger.debug("%d row keys removed", len(trash_can))
        return results

Example 46

Project: SickGear
Source File: curl_httpclient.py
View license
    def _curl_setup_request(self, curl, request, buffer, headers):
        curl.setopt(pycurl.URL, native_str(request.url))

        # libcurl's magic "Expect: 100-continue" behavior causes delays
        # with servers that don't support it (which include, among others,
        # Google's OpenID endpoint).  Additionally, this behavior has
        # a bug in conjunction with the curl_multi_socket_action API
        # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
        # which increases the delays.  It's more trouble than it's worth,
        # so just turn off the feature (yes, setting Expect: to an empty
        # value is the official way to disable this)
        if "Expect" not in request.headers:
            request.headers["Expect"] = ""

        # libcurl adds Pragma: no-cache by default; disable that too
        if "Pragma" not in request.headers:
            request.headers["Pragma"] = ""

        curl.setopt(pycurl.HTTPHEADER,
                    ["%s: %s" % (native_str(k), native_str(v))
                     for k, v in request.headers.get_all()])

        curl.setopt(pycurl.HEADERFUNCTION,
                    functools.partial(self._curl_header_callback,
                                      headers, request.header_callback))
        if request.streaming_callback:
            def write_function(chunk):
                self.io_loop.add_callback(request.streaming_callback, chunk)
        else:
            write_function = buffer.write
        if bytes is str:  # py2
            curl.setopt(pycurl.WRITEFUNCTION, write_function)
        else:  # py3
            # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
            # a fork/port.  That version has a bug in which it passes unicode
            # strings instead of bytes to the WRITEFUNCTION.  This means that
            # if you use a WRITEFUNCTION (which tornado always does), you cannot
            # download arbitrary binary data.  This needs to be fixed in the
            # ported pycurl package, but in the meantime this lambda will
            # make it work for downloading (utf8) text.
            curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s)))
        curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
        curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
        curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout))
        curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout))
        if request.user_agent:
            curl.setopt(pycurl.USERAGENT, native_str(request.user_agent))
        else:
            curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
        if request.network_interface:
            curl.setopt(pycurl.INTERFACE, request.network_interface)
        if request.decompress_response:
            curl.setopt(pycurl.ENCODING, "gzip,deflate")
        else:
            curl.setopt(pycurl.ENCODING, "none")
        if request.proxy_host and request.proxy_port:
            curl.setopt(pycurl.PROXY, request.proxy_host)
            curl.setopt(pycurl.PROXYPORT, request.proxy_port)
            if request.proxy_username:
                credentials = '%s:%s' % (request.proxy_username,
                                         request.proxy_password)
                curl.setopt(pycurl.PROXYUSERPWD, credentials)
        else:
            curl.setopt(pycurl.PROXY, '')
            curl.unsetopt(pycurl.PROXYUSERPWD)
        if request.validate_cert:
            curl.setopt(pycurl.SSL_VERIFYPEER, 1)
            curl.setopt(pycurl.SSL_VERIFYHOST, 2)
        else:
            curl.setopt(pycurl.SSL_VERIFYPEER, 0)
            curl.setopt(pycurl.SSL_VERIFYHOST, 0)
        if request.ca_certs is not None:
            curl.setopt(pycurl.CAINFO, request.ca_certs)
        else:
            # There is no way to restore pycurl.CAINFO to its default value
            # (Using unsetopt makes it reject all certificates).
            # I don't see any way to read the default value from python so it
            # can be restored later.  We'll have to just leave CAINFO untouched
            # if no ca_certs file was specified, and require that if any
            # request uses a custom ca_certs file, they all must.
            pass

        if request.allow_ipv6 is False:
            # Curl behaves reasonably when DNS resolution gives an ipv6 address
            # that we can't reach, so allow ipv6 unless the user asks to disable.
            curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
        else:
            curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)

        # Set the request method through curl's irritating interface which makes
        # up names for almost every single method
        curl_options = {
            "GET": pycurl.HTTPGET,
            "POST": pycurl.POST,
            "PUT": pycurl.UPLOAD,
            "HEAD": pycurl.NOBODY,
        }
        custom_methods = set(["DELETE", "OPTIONS", "PATCH"])
        for o in curl_options.values():
            curl.setopt(o, False)
        if request.method in curl_options:
            curl.unsetopt(pycurl.CUSTOMREQUEST)
            curl.setopt(curl_options[request.method], True)
        elif request.allow_nonstandard_methods or request.method in custom_methods:
            curl.setopt(pycurl.CUSTOMREQUEST, request.method)
        else:
            raise KeyError('unknown method ' + request.method)

        # Handle curl's cryptic options for every individual HTTP method
        if request.method == "GET":
            if request.body is not None:
                raise ValueError('Body must be None for GET request')
        elif request.method in ("POST", "PUT") or request.body:
            if request.body is None:
                raise ValueError(
                    'Body must not be None for "%s" request'
                    % request.method)

            request_buffer = BytesIO(utf8(request.body))

            def ioctl(cmd):
                if cmd == curl.IOCMD_RESTARTREAD:
                    request_buffer.seek(0)
            curl.setopt(pycurl.READFUNCTION, request_buffer.read)
            curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
            if request.method == "POST":
                curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
            else:
                curl.setopt(pycurl.UPLOAD, True)
                curl.setopt(pycurl.INFILESIZE, len(request.body))

        if request.auth_username is not None:
            userpwd = "%s:%s" % (request.auth_username, request.auth_password or '')

            if request.auth_mode is None or request.auth_mode == "basic":
                curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
            elif request.auth_mode == "digest":
                curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
            else:
                raise ValueError("Unsupported auth_mode %s" % request.auth_mode)

            curl.setopt(pycurl.USERPWD, native_str(userpwd))
            curl_log.debug("%s %s (username: %r)", request.method, request.url,
                           request.auth_username)
        else:
            curl.unsetopt(pycurl.USERPWD)
            curl_log.debug("%s %s", request.method, request.url)

        if request.client_cert is not None:
            curl.setopt(pycurl.SSLCERT, request.client_cert)

        if request.client_key is not None:
            curl.setopt(pycurl.SSLKEY, request.client_key)

        if request.ssl_options is not None:
            raise ValueError("ssl_options not supported in curl_httpclient")

        if threading.activeCount() > 1:
            # libcurl/pycurl is not thread-safe by default.  When multiple threads
            # are used, signals should be disabled.  This has the side effect
            # of disabling DNS timeouts in some environments (when libcurl is
            # not linked against ares), so we don't do it when there is only one
            # thread.  Applications that use many short-lived threads may need
            # to set NOSIGNAL manually in a prepare_curl_callback since
            # there may not be any other threads running at the time we call
            # threading.activeCount.
            curl.setopt(pycurl.NOSIGNAL, 1)
        if request.prepare_curl_callback is not None:
            request.prepare_curl_callback(curl)

Example 47

Project: SickRage
Source File: curl_httpclient.py
View license
    def _curl_setup_request(self, curl, request, buffer, headers):
        curl.setopt(pycurl.URL, native_str(request.url))

        # libcurl's magic "Expect: 100-continue" behavior causes delays
        # with servers that don't support it (which include, among others,
        # Google's OpenID endpoint).  Additionally, this behavior has
        # a bug in conjunction with the curl_multi_socket_action API
        # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
        # which increases the delays.  It's more trouble than it's worth,
        # so just turn off the feature (yes, setting Expect: to an empty
        # value is the official way to disable this)
        if "Expect" not in request.headers:
            request.headers["Expect"] = ""

        # libcurl adds Pragma: no-cache by default; disable that too
        if "Pragma" not in request.headers:
            request.headers["Pragma"] = ""

        curl.setopt(pycurl.HTTPHEADER,
                    ["%s: %s" % (native_str(k), native_str(v))
                     for k, v in request.headers.get_all()])

        curl.setopt(pycurl.HEADERFUNCTION,
                    functools.partial(self._curl_header_callback,
                                      headers, request.header_callback))
        if request.streaming_callback:
            def write_function(chunk):
                self.io_loop.add_callback(request.streaming_callback, chunk)
        else:
            write_function = buffer.write
        if bytes is str:  # py2
            curl.setopt(pycurl.WRITEFUNCTION, write_function)
        else:  # py3
            # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
            # a fork/port.  That version has a bug in which it passes unicode
            # strings instead of bytes to the WRITEFUNCTION.  This means that
            # if you use a WRITEFUNCTION (which tornado always does), you cannot
            # download arbitrary binary data.  This needs to be fixed in the
            # ported pycurl package, but in the meantime this lambda will
            # make it work for downloading (utf8) text.
            curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s)))
        curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
        curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
        curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout))
        curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout))
        if request.user_agent:
            curl.setopt(pycurl.USERAGENT, native_str(request.user_agent))
        else:
            curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
        if request.network_interface:
            curl.setopt(pycurl.INTERFACE, request.network_interface)
        if request.decompress_response:
            curl.setopt(pycurl.ENCODING, "gzip,deflate")
        else:
            curl.setopt(pycurl.ENCODING, "none")
        if request.proxy_host and request.proxy_port:
            curl.setopt(pycurl.PROXY, request.proxy_host)
            curl.setopt(pycurl.PROXYPORT, request.proxy_port)
            if request.proxy_username:
                credentials = '%s:%s' % (request.proxy_username,
                                         request.proxy_password)
                curl.setopt(pycurl.PROXYUSERPWD, credentials)
        else:
            curl.setopt(pycurl.PROXY, '')
            curl.unsetopt(pycurl.PROXYUSERPWD)
        if request.validate_cert:
            curl.setopt(pycurl.SSL_VERIFYPEER, 1)
            curl.setopt(pycurl.SSL_VERIFYHOST, 2)
        else:
            curl.setopt(pycurl.SSL_VERIFYPEER, 0)
            curl.setopt(pycurl.SSL_VERIFYHOST, 0)
        if request.ca_certs is not None:
            curl.setopt(pycurl.CAINFO, request.ca_certs)
        else:
            # There is no way to restore pycurl.CAINFO to its default value
            # (Using unsetopt makes it reject all certificates).
            # I don't see any way to read the default value from python so it
            # can be restored later.  We'll have to just leave CAINFO untouched
            # if no ca_certs file was specified, and require that if any
            # request uses a custom ca_certs file, they all must.
            pass

        if request.allow_ipv6 is False:
            # Curl behaves reasonably when DNS resolution gives an ipv6 address
            # that we can't reach, so allow ipv6 unless the user asks to disable.
            curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
        else:
            curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)

        # Set the request method through curl's irritating interface which makes
        # up names for almost every single method
        curl_options = {
            "GET": pycurl.HTTPGET,
            "POST": pycurl.POST,
            "PUT": pycurl.UPLOAD,
            "HEAD": pycurl.NOBODY,
        }
        custom_methods = set(["DELETE", "OPTIONS", "PATCH"])
        for o in curl_options.values():
            curl.setopt(o, False)
        if request.method in curl_options:
            curl.unsetopt(pycurl.CUSTOMREQUEST)
            curl.setopt(curl_options[request.method], True)
        elif request.allow_nonstandard_methods or request.method in custom_methods:
            curl.setopt(pycurl.CUSTOMREQUEST, request.method)
        else:
            raise KeyError('unknown method ' + request.method)

        body_expected = request.method in ("POST", "PATCH", "PUT")
        body_present = request.body is not None
        if not request.allow_nonstandard_methods:
            # Some HTTP methods nearly always have bodies while others
            # almost never do. Fail in this case unless the user has
            # opted out of sanity checks with allow_nonstandard_methods.
            if ((body_expected and not body_present) or
                    (body_present and not body_expected)):
                raise ValueError(
                    'Body must %sbe None for method %s (unless '
                    'allow_nonstandard_methods is true)' %
                    ('not ' if body_expected else '', request.method))

        if body_expected or body_present:
            if request.method == "GET":
                # Even with `allow_nonstandard_methods` we disallow
                # GET with a body (because libcurl doesn't allow it
                # unless we use CUSTOMREQUEST). While the spec doesn't
                # forbid clients from sending a body, it arguably
                # disallows the server from doing anything with them.
                raise ValueError('Body must be None for GET request')
            request_buffer = BytesIO(utf8(request.body or ''))

            def ioctl(cmd):
                if cmd == curl.IOCMD_RESTARTREAD:
                    request_buffer.seek(0)
            curl.setopt(pycurl.READFUNCTION, request_buffer.read)
            curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
            if request.method == "POST":
                curl.setopt(pycurl.POSTFIELDSIZE, len(request.body or ''))
            else:
                curl.setopt(pycurl.UPLOAD, True)
                curl.setopt(pycurl.INFILESIZE, len(request.body or ''))

        if request.auth_username is not None:
            userpwd = "%s:%s" % (request.auth_username, request.auth_password or '')

            if request.auth_mode is None or request.auth_mode == "basic":
                curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
            elif request.auth_mode == "digest":
                curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
            else:
                raise ValueError("Unsupported auth_mode %s" % request.auth_mode)

            curl.setopt(pycurl.USERPWD, native_str(userpwd))
            curl_log.debug("%s %s (username: %r)", request.method, request.url,
                           request.auth_username)
        else:
            curl.unsetopt(pycurl.USERPWD)
            curl_log.debug("%s %s", request.method, request.url)

        if request.client_cert is not None:
            curl.setopt(pycurl.SSLCERT, request.client_cert)

        if request.client_key is not None:
            curl.setopt(pycurl.SSLKEY, request.client_key)

        if request.ssl_options is not None:
            raise ValueError("ssl_options not supported in curl_httpclient")

        if threading.activeCount() > 1:
            # libcurl/pycurl is not thread-safe by default.  When multiple threads
            # are used, signals should be disabled.  This has the side effect
            # of disabling DNS timeouts in some environments (when libcurl is
            # not linked against ares), so we don't do it when there is only one
            # thread.  Applications that use many short-lived threads may need
            # to set NOSIGNAL manually in a prepare_curl_callback since
            # there may not be any other threads running at the time we call
            # threading.activeCount.
            curl.setopt(pycurl.NOSIGNAL, 1)
        if request.prepare_curl_callback is not None:
            request.prepare_curl_callback(curl)

Example 48

Project: SickRage
Source File: itasa.py
View license
    def query(self, series, season, episode, video_format, resolution, country=None):

        # To make queries you need to be logged in
        if not self.logged_in:  # pragma: no cover
            raise ConfigurationError('Cannot query if not logged in')

        # get the show id
        show_id = self.get_show_id(series, country)
        if show_id is None:
            logger.error('No show id found for %r ', series)
            return []

        # get the page of the season of the show
        logger.info('Getting the subtitle of show id %d, season %d episode %d, format %r', show_id,
                    season, episode, video_format)
        subtitles = []

        # Default format is SDTV
        if not video_format or video_format.lower() == 'hdtv':
            if resolution in ('1080i', '1080p', '720p'):
                sub_format = resolution
            else:
                sub_format = 'normale'
        else:
            sub_format = video_format.lower()

        # Look for year
        params = {
            'apikey': self.apikey
        }
        r = self.session.get(self.server_url + 'shows/' + str(show_id), params=params, timeout=30)
        r.raise_for_status()
        root = etree.fromstring(r.content)

        year = root.find('data/show/started').text
        if year:
            year = int(year.split('-', 1)[0])
        tvdb_id = root.find('data/show/id_tvdb').text
        if tvdb_id:
            tvdb_id = int(tvdb_id)

        params = {
            'apikey': self.apikey,
            'show_id': show_id,
            'q': '%dx%02d' % (season, episode),
            'version': sub_format
            }
        r = self.session.get(self.server_url + 'subtitles/search', params=params, timeout=30)
        r.raise_for_status()
        root = etree.fromstring(r.content)

        if int(root.find('data/count').text) == 0:
            logger.warning('Subtitles not found,  try with rip suffix')

            params['version'] = sub_format + 'rip'
            r = self.session.get(self.server_url + 'subtitles/search', params=params, timeout=30)
            r.raise_for_status()
            root = etree.fromstring(r.content)
            if int(root.find('data/count').text) == 0:
                logger.warning('Subtitles not found, go season mode')

                # If no subtitle are found for single episode try to download all season zip
                subs = self._get_season_subtitles(show_id, season, sub_format)
                if subs:
                    for subtitle in subs:
                        subtitle.format = video_format
                        subtitle.year = year
                        subtitle.tvdb_id = tvdb_id

                    return subs
                else:
                    return []

        # Looking for subtitles in first page
        for subtitle in root.findall('data/subtitles/subtitle'):
            if '%dx%02d' % (season, episode) in subtitle.find('name').text.lower():

                logger.debug('Found subtitle id %d - %r - %r',
                             int(subtitle.find('id').text),
                             subtitle.find('name').text,
                             subtitle.find('version').text)

                sub = ItaSASubtitle(
                        int(subtitle.find('id').text),
                        subtitle.find('show_name').text,
                        season,
                        episode,
                        video_format,
                        year,
                        tvdb_id,
                        subtitle.find('name').text)

                subtitles.append(sub)

        # Not in the first page of result try next (if any)
        next_page = root.find('data/next')
        while next_page.text is not None:   # pragma: no cover

            r = self.session.get(next_page.text, timeout=30)
            r.raise_for_status()
            root = etree.fromstring(r.content)

            logger.info('Loading subtitles page %r', root.data.page.text)

            # Looking for show in following pages
            for subtitle in root.findall('data/subtitles/subtitle'):
                if '%dx%02d' % (season, episode) in subtitle.find('name').text.lower():

                    logger.debug('Found subtitle id %d - %r - %r',
                                 int(subtitle.find('id').text),
                                 subtitle.find('name').text,
                                 subtitle.find('version').text)

                    sub = ItaSASubtitle(
                        int(subtitle.find('id').text),
                        subtitle.find('show_name').text,
                        season,
                        episode,
                        video_format,
                        year,
                        tvdb_id,
                        subtitle.find('name').text)

                    subtitles.append(sub)

            next_page = root.find('data/next')

        # Download the subs found, can be more than one in zip
        additional_subs = []
        for sub in subtitles:

            # open the zip
            content = self._download_zip(sub.sub_id)
            if not is_zipfile(io.BytesIO(content)):   # pragma: no cover
                if 'limite di download' in content:
                    raise TooManyRequests()
                else:
                    raise ConfigurationError('Not a zip file: %r' % content)

            with ZipFile(io.BytesIO(content)) as zf:
                if len(zf.namelist()) > 1:   # pragma: no cover

                    for index, name in enumerate(zf.namelist()):

                        if index == 0:
                            # First element
                            sub.content = fix_line_ending(zf.read(name))
                            sub.full_data = name
                        else:
                            add_sub = copy.deepcopy(sub)
                            add_sub.content = fix_line_ending(zf.read(name))
                            add_sub.full_data = name
                            additional_subs.append(add_sub)
                else:
                    sub.content = fix_line_ending(zf.read(zf.namelist()[0]))
                    sub.full_data = zf.namelist()[0]

        return subtitles + additional_subs

Example 49

Project: TYOS
Source File: app.py
View license
    def display(self):
        while True:
            if self.mode == 'gallery':                        
                self.screen.blit(self.image_in_view, (0,0))
                self.screen.blit(self.left, (20, 410))
                self.screen.blit(self.right, (240, 410))
                self.screen.blit(self.home, (125, 400))
                self.screen.blit(self.delete, (5, 5))
                self.screen.blit(self.upload, (40, 5))
                if self.deleted:
                    self.screen.blit(self.deleted_image, (79, 200))
                    if time.time() - self.delete_time > 3:
                        self.deleted = False
                if self.uploading:
                    self.screen.blit(self.uploading_image, (79, 200))
                    if time.time() - self.uploading_time > 6:
                        self.uploading = False
                
            if self.mode == 'capture':
                #Get camera stream
                self.stream = io.BytesIO() # Capture into in-memory stream
                self.camera.capture(self.stream, use_video_port=True, format='raw')
                self.stream.seek(0)
                self.stream.readinto(self.yuv)  # stream -> YUV buffer
                self.stream.close()
                yuv2rgb.convert(self.yuv, self.rgb, 320, 480)
            
                #Create pygame image from screen and blit it
                img = pygame.image.frombuffer(self.rgb[0:(320 * 480 * 3)], (320, 480), 'RGB')
                self.screen.blit(img, (0,0))
            
                #Blit buttons
                self.screen.blit(self.capture, (125, 400))
                self.screen.blit(self.gallery, (20, 415))
                self.screen.blit(self.door, (240, 410))

                if self.no_files:
                    self.screen.blit(self.no_files_image, (79, 200))
                    if time.time() - self.files_time > 3:
                        self.no_files = False
            
            pygame.display.update()
            
            #Handle events
            for event in pygame.event.get():
                if event.type == pygame.MOUSEBUTTONUP:
                    if self.mode == 'gallery':
                        if event.pos[1] < 40 and event.pos[0] < 35:
                            self.deleted = True
                            self.delete_time = time.time()
                            os.remove('/home/pi/Photos/' + self.images[self.current_image])
                            self.current_image = 0
                            self.images = os.listdir('/home/pi/Photos/')
                            if len(self.images) == 0:
                                self.mode = 'capture'
                                self.no_files = True
                                self.files_time = time.time()
                            
                        if event.pos[1] < 40 and event.pos[0] > 35 and event.pos[0] < 75:
                            self.uploading = True
                            self.uploading_time = time.time()
                            cam = Popen(['/home/pi/Dropbox-Uploader/./dropbox_uploader.sh', 'upload', '/home/pi/Photos/' +
                                         self.images[self.current_image], self.images[self.current_image]])
                            
                    if event.pos[1] > 400 and event.pos[1] < 470:
                        if event.pos[0] > 125 and event.pos[0] < 195:
                            if self.mode == 'capture':
                                self.camera.capture('/home/pi/Photos/' + str(self.index) + '.jpg')
                                self.index += 1
                            if self.mode == 'gallery':
                                self.mode = 'capture'
            
                        if event.pos[0] < 70:
                            if self.mode == 'capture':
                                self.mode = 'gallery'
                                self.current_image = 0
                                self.images = os.listdir('/home/pi/Photos/')
                                if len(self.images) == 0:
                                    self.mode = 'capture'
                                    self.no_files = True
                                    self.files_time = time.time()
                                else:
                                    self.image_in_view = pygame.image.load('/home/pi/Photos/' + self.images[self.current_image])
                                
                            if self.mode == 'gallery':
                                self.current_image -= 1
                                if self.current_image == -1:
                                    self.current_image = len(self.images) - 1
                                self.image_in_view = pygame.image.load('/home/pi/Photos/' + self.images[self.current_image])
                                
                        if event.pos[0] > 255:
                            if self.mode == 'capture':
                                print 'exiting...'
                                os.remove('/home/pi/index.dat')
                                new = open('/home/pi/index.dat', 'w+')
                                new.write(str(self.index))
                                new.close()
                                cam = Popen(['sudo', 'python', '/home/pi/tyos/src/main.py'])
                                pygame.quit()
                                sys.exit()


                            if self.mode == 'gallery':
                                if self.current_image == len(self.images) - 1:
                                    self.current_image = 0
                                else:
                                    self.current_image += 1
                                self.image_in_view = pygame.image.load('/home/pi/Photos/' + self.images[self.current_image])

Example 50

Project: maltrail
Source File: httpd.py
View license
def start_httpd(address=None, port=None, join=False, pem=None):
    """
    Starts HTTP server
    """

    class ThreadingServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer):
        def server_bind(self):
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            BaseHTTPServer.HTTPServer.server_bind(self)

        def finish_request(self, *args, **kwargs):
            try:
                BaseHTTPServer.HTTPServer.finish_request(self, *args, **kwargs)
            except:
                if config.SHOW_DEBUG:
                    traceback.print_exc()

    class SSLThreadingServer(ThreadingServer):
        def __init__(self, server_address, pem, HandlerClass):
            import OpenSSL  # python-openssl

            ThreadingServer.__init__(self, server_address, HandlerClass)
            ctx = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
            ctx.use_privatekey_file(pem)
            ctx.use_certificate_file(pem)
            self.socket = OpenSSL.SSL.Connection(ctx, socket.socket(self.address_family, self.socket_type))
            self.server_bind()
            self.server_activate()

        def shutdown_request(self, request):
            try:
                request.shutdown()
            except:
                if config.SHOW_DEBUG:
                    traceback.print_exc()

    class ReqHandler(BaseHTTPServer.BaseHTTPRequestHandler):
        def do_GET(self):
            path, query = self.path.split('?', 1) if '?' in self.path else (self.path, "")
            params = {}
            content = None
            skip = False

            if hasattr(self, "data"):
                params.update(urlparse.parse_qs(self.data))

            if query:
                params.update(urlparse.parse_qs(query))

            for key in params:
                if params[key]:
                    params[key] = params[key][-1]

            if path == '/':
                path = "index.html"

            path = path.strip('/')
            extension = os.path.splitext(path)[-1].lower()

            if hasattr(self, "_%s" % path):
                content = getattr(self, "_%s" % path)(params)

            else:
                path = path.replace('/', os.path.sep)
                path = os.path.abspath(os.path.join(HTML_DIR, path)).strip()

                if not os.path.isfile(path) and os.path.isfile("%s.html" % path):
                    path = "%s.html" % path

                if ".." not in os.path.relpath(path, HTML_DIR) and os.path.isfile(path) and (extension not in DISABLED_CONTENT_EXTENSIONS or os.path.split(path)[-1] in CONTENT_EXTENSIONS_EXCLUSIONS):
                    mtime = time.gmtime(os.path.getmtime(path))
                    if_modified_since = self.headers.get(HTTP_HEADER.IF_MODIFIED_SINCE)

                    if if_modified_since and extension not in (".htm", ".html"):
                        if_modified_since = [_ for _ in if_modified_since.split(';') if _.upper().endswith("GMT")][0]
                        if time.mktime(mtime) <= time.mktime(time.strptime(if_modified_since, HTTP_TIME_FORMAT)):
                            self.send_response(httplib.NOT_MODIFIED)
                            self.send_header(HTTP_HEADER.CONNECTION, "close")
                            skip = True

                    if not skip:
                        content = open(path, "rb").read()
                        last_modified = time.strftime(HTTP_TIME_FORMAT, mtime)
                        self.send_response(httplib.OK)
                        self.send_header(HTTP_HEADER.CONNECTION, "close")
                        self.send_header(HTTP_HEADER.CONTENT_TYPE, mimetypes.guess_type(path)[0] or "application/octet-stream")
                        self.send_header(HTTP_HEADER.LAST_MODIFIED, last_modified)
                        if extension not in (".htm", ".html"):
                            self.send_header(HTTP_HEADER.EXPIRES, "Sun, 17-Jan-2038 19:14:07 GMT")        # Reference: http://blog.httpwatch.com/2007/12/10/two-simple-rules-for-http-caching/
                            self.send_header(HTTP_HEADER.CACHE_CONTROL, "max-age=3600, must-revalidate")  # Reference: http://stackoverflow.com/a/5084555
                        else:
                            self.send_header(HTTP_HEADER.CACHE_CONTROL, "no-cache")

                else:
                    self.send_response(httplib.NOT_FOUND)
                    self.send_header(HTTP_HEADER.CONNECTION, "close")
                    content = '<!DOCTYPE html><html lang="en"><head><title>404 Not Found</title></head><body><h1>Not Found</h1><p>The requested URL %s was not found on this server.</p></body></html>' % self.path.split('?')[0]

            if content is not None:
                for match in re.finditer(r"<\!(\w+)\!>", content):
                    name = match.group(1)
                    _ = getattr(self, "_%s" % name.lower(), None)
                    if _:
                        content = self._format(content, **{ name: _() })

                if "gzip" in self.headers.getheader(HTTP_HEADER.ACCEPT_ENCODING, ""):
                    self.send_header(HTTP_HEADER.CONTENT_ENCODING, "gzip")
                    _ = cStringIO.StringIO()
                    compress = gzip.GzipFile("", "w+b", 9, _)
                    compress._stream = _
                    compress.write(content)
                    compress.flush()
                    compress.close()
                    content = compress._stream.getvalue()

                self.send_header(HTTP_HEADER.CONTENT_LENGTH, str(len(content)))

            self.end_headers()

            if content:
                self.wfile.write(content)

            self.wfile.flush()
            self.wfile.close()

        def do_POST(self):
            length = self.headers.getheader(HTTP_HEADER.CONTENT_LENGTH)
            data = self.rfile.read(int(length))
            data = urllib.unquote_plus(data)
            self.data = data
            self.do_GET()

        def get_session(self):
            retval = None
            cookie = self.headers.get(HTTP_HEADER.COOKIE)

            if cookie:
                match = re.search(r"%s\s*=\s*([^;]+)" % SESSION_COOKIE_NAME, cookie)
                if match:
                    session = match.group(1)
                    if session in SESSIONS:
                        if SESSIONS[session].client_ip != self.client_address[0]:
                            pass
                        elif SESSIONS[session].expiration > time.time():
                            retval = SESSIONS[session]
                        else:
                            del SESSIONS[session]

            return retval

        def delete_session(self):
            cookie = self.headers.get(HTTP_HEADER.COOKIE)

            if cookie:
                match = re.search(r"%s=(.+)" % SESSION_COOKIE_NAME, cookie)
                if match:
                    session = match.group(1)
                    if session in SESSIONS:
                        del SESSIONS[session]

        def version_string(self):
            return SERVER_HEADER

        def end_headers(self):
            if not hasattr(self, "_headers_ended"):
                BaseHTTPServer.BaseHTTPRequestHandler.end_headers(self)
                self._headers_ended = True

        def log_message(self, format, *args):
            return

        def finish(self):
            try:
                BaseHTTPServer.BaseHTTPRequestHandler.finish(self)
            except:
                if config.SHOW_DEBUG:
                    traceback.print_exc()

        def _version(self):
            return VERSION

        def _format(self, content, **params):
            if content:
                for key, value in params.items():
                    content = content.replace("<!%s!>" % key, value)

            return content

        def _login(self, params):
            valid = False

            if params.get("username") and params.get("hash") and params.get("nonce"):
                if params.get("nonce") not in DISPOSED_NONCES:
                    DISPOSED_NONCES.add(params.get("nonce"))
                    for entry in (config.USERS or []):
                        entry = re.sub(r"\s", "", entry)
                        username, stored_hash, uid, netfilter = entry.split(':')
                        if username == params.get("username"):
                            try:
                                if params.get("hash") == hashlib.sha256(stored_hash.strip() + params.get("nonce")).hexdigest():
                                    valid = True
                                    break
                            except:
                                if config.SHOW_DEBUG:
                                    traceback.print_exc()

            if valid:
                session_id = os.urandom(SESSION_ID_LENGTH).encode("hex")
                expiration = time.time() + 3600 * SESSION_EXPIRATION_HOURS

                self.send_response(httplib.OK)
                self.send_header(HTTP_HEADER.CONNECTION, "close")
                self.send_header(HTTP_HEADER.SET_COOKIE, "%s=%s; expires=%s; path=/; HttpOnly" % (SESSION_COOKIE_NAME, session_id, time.strftime(HTTP_TIME_FORMAT, time.gmtime(expiration))))

                if netfilter in ("", "0.0.0.0/0"):
                    netfilters = None
                else:
                    addresses = set()
                    netmasks = set()

                    for item in set(re.split(r"[;,]", netfilter)):
                        item = item.strip()
                        if '/' in item:
                            _ = item.split('/')[-1]
                            if _.isdigit() and int(_) >= 16:
                                lower = addr_to_int(item.split('/')[0])
                                mask = make_mask(int(_))
                                upper = lower | (0xffffffff ^ mask)
                                while lower <= upper:
                                    addresses.add(int_to_addr(lower))
                                    lower += 1
                            else:
                                netmasks.add(item)
                        elif '-' in item:
                            _ = item.split('-')
                            lower, upper = addr_to_int(_[0]), addr_to_int(_[1])
                            while lower <= upper:
                                addresses.add(int_to_addr(lower))
                                lower += 1
                        elif re.search(r"\d+\.\d+\.\d+\.\d+", item):
                            addresses.add(item)

                    netfilters = netmasks
                    if addresses:
                        netfilters.add(get_regex(addresses))

                SESSIONS[session_id] = AttribDict({"username": username, "uid": uid, "netfilters": netfilters, "expiration": expiration, "client_ip": self.client_address[0]})
            else:
                time.sleep(UNAUTHORIZED_SLEEP_TIME)
                self.send_response(httplib.UNAUTHORIZED)
                self.send_header(HTTP_HEADER.CONNECTION, "close")

            self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")
            content = "Login %s" % ("success" if valid else "failed")

            if not subprocess.mswindows:
                try:
                    subprocess.check_output("logger -p auth.info -t \"%s[%d]\" \"%s password for %s from %s port %s\"" % (NAME.lower(), os.getpid(), "Accepted" if valid else "Failed", params.get("username"), self.client_address[0], self.client_address[1]), stderr=subprocess.STDOUT, shell=True)
                except Exception:
                    if config.SHOW_DEBUG:
                        traceback.print_exc()

            return content

        def _logout(self, params):
            self.delete_session()
            self.send_response(httplib.FOUND)
            self.send_header(HTTP_HEADER.CONNECTION, "close")
            self.send_header(HTTP_HEADER.LOCATION, "/")

        def _whoami(self, params):
            session = self.get_session()
            username = session.username if session else ""

            self.send_response(httplib.OK)
            self.send_header(HTTP_HEADER.CONNECTION, "close")
            self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")

            return username

        def _check_ip(self, params):
            session = self.get_session()

            if session is None:
                self.send_response(httplib.UNAUTHORIZED)
                self.send_header(HTTP_HEADER.CONNECTION, "close")
                return None

            self.send_response(httplib.OK)
            self.send_header(HTTP_HEADER.CONNECTION, "close")
            self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")

            try:
                result_worst = worst_asns(params.get("address"))
                if result_worst:
                    result_ipcat = result_worst
                else:
                    _ = (ipcat_lookup(params.get("address")) or "").lower().split(' ')
                    result_ipcat = _[1] if _[0] == 'the' else _[0]
                return ("%s" if not params.get("callback") else "%s(%%s)" % params.get("callback")) % json.dumps({"ipcat": result_ipcat, "worst_asns": str(result_worst is not None).lower()})
            except:
                if config.SHOW_DEBUG:
                    traceback.print_exc()

        def _trails(self, params):
            self.send_response(httplib.OK)
            self.send_header(HTTP_HEADER.CONNECTION, "close")
            self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")

            return open(TRAILS_FILE, "rb").read()

        def _ping(self, params):
            self.send_response(httplib.OK)
            self.send_header(HTTP_HEADER.CONNECTION, "close")
            self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")

            return PING_RESPONSE

        def _events(self, params):
            session = self.get_session()

            if session is None:
                self.send_response(httplib.UNAUTHORIZED)
                self.send_header(HTTP_HEADER.CONNECTION, "close")
                return None

            start, end, size, total = None, None, -1, None
            content = None
            log_exists = False
            dates = params.get("date", "")

            if ".." in dates:
                pass
            elif '_' not in dates:
                try:
                    date = datetime.datetime.strptime(dates, "%Y-%m-%d").strftime("%Y-%m-%d")
                    event_log_path = os.path.join(config.LOG_DIR, "%s.log" % date)
                    if os.path.exists(event_log_path):
                        range_handle = open(event_log_path, "rb")
                        log_exists = True
                except ValueError:
                    print "[!] invalid date format in request"
                    log_exists = False
            else:
                logs_data = ""
                date_interval = dates.split("_", 1)
                try:
                    start_date = datetime.datetime.strptime(date_interval[0], "%Y-%m-%d").date()
                    end_date = datetime.datetime.strptime(date_interval[1], "%Y-%m-%d").date()
                    for i in xrange(int((end_date - start_date).days) + 1):
                        date = start_date + datetime.timedelta(i)
                        event_log_path = os.path.join(config.LOG_DIR, "%s.log" % date.strftime("%Y-%m-%d"))
                        if os.path.exists(event_log_path):
                            log_handle = open(event_log_path, "rb")
                            logs_data += log_handle.read()
                            log_handle.close()

                    range_handle = io.BytesIO(logs_data)
                    log_exists = True
                except ValueError:
                    print "[!] invalid date format in request"
                    log_exists = False

            if log_exists:
                range_handle.seek(0, 2)
                total = range_handle.tell()
                range_handle.seek(0)

                if self.headers.get(HTTP_HEADER.RANGE):
                    match = re.search(r"bytes=(\d+)-(\d+)", self.headers[HTTP_HEADER.RANGE])
                    if match:
                        start, end = int(match.group(1)), int(match.group(2))
                        max_size = end - start + 1
                        end = min(total - 1, end)
                        size = end - start + 1

                        if start == 0 or not session.range_handle:
                            session.range_handle = range_handle

                        if session.netfilters is None:
                            session.range_handle.seek(start)
                            self.send_response(httplib.PARTIAL_CONTENT)
                            self.send_header(HTTP_HEADER.CONNECTION, "close")
                            self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")
                            self.send_header(HTTP_HEADER.CONTENT_RANGE, "bytes %d-%d/%d" % (start, end, total))
                            content = session.range_handle.read(size)
                        else:
                            self.send_response(httplib.OK)
                            self.send_header(HTTP_HEADER.CONNECTION, "close")
                            self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")

                            buffer, addresses, netmasks, regex = cStringIO.StringIO(), set(), [], ""
                            for netfilter in session.netfilters:
                                if not netfilter:
                                    continue
                                if '/' in netfilter:
                                    netmasks.append(netfilter)
                                elif re.search(r"\A[\d.]+\Z", netfilter):
                                    addresses.add(netfilter)
                                elif '\.' in netfilter:
                                    regex = r"\b(%s)\b" % netfilter
                                else:
                                    print "[!] invalid network filter '%s'" % netfilter
                                    return

                            for line in session.range_handle:
                                display = False
                                ip = None

                                if regex:
                                    match = re.search(regex, line)
                                    if match:
                                        ip = match.group(1)
                                        display = True

                                if not display and (addresses or netmasks):
                                    for match in re.finditer(r"\b(\d+\.\d+\.\d+\.\d+)\b", line):
                                        if not display:
                                            ip = match.group(1)
                                        else:
                                            break

                                        if ip in addresses:
                                            display = True
                                            break
                                        elif netmasks:
                                            for _ in netmasks:
                                                prefix, mask = _.split('/')
                                                if addr_to_int(ip) & make_mask(int(mask)) == addr_to_int(prefix):
                                                    addresses.add(ip)
                                                    display = True
                                                    break

                                if display:
                                    if ",%s" % ip in line or "%s," % ip in line:
                                        line = re.sub(r" ([\d.,]+,)?%s(,[\d.,]+)? " % re.escape(ip), " %s " % ip, line)
                                    buffer.write(line)
                                    if buffer.tell() >= max_size:
                                        break

                            content = buffer.getvalue()
                            end = start + len(content) - 1
                            self.send_header(HTTP_HEADER.CONTENT_RANGE, "bytes %d-%d/%d" % (start, end, end + 1 + max_size * (len(content) >= max_size)))

                        if len(content) < max_size:
                            session.range_handle.close()
                            session.range_handle = None

                if size == -1:
                    self.send_response(httplib.OK)
                    self.send_header(HTTP_HEADER.CONNECTION, "close")
                    self.send_header(HTTP_HEADER.CONTENT_TYPE, "text/plain")
                    self.end_headers()

                    with range_handle as f:
                        while True:
                            data = f.read(io.DEFAULT_BUFFER_SIZE)
                            if not data:
                                break
                            else:
                                self.wfile.write(data)

            else:
                self.send_response(httplib.OK)  # instead of httplib.NO_CONTENT (compatibility reasons)
                self.send_header(HTTP_HEADER.CONNECTION, "close")
                if self.headers.get(HTTP_HEADER.RANGE):
                    self.send_header(HTTP_HEADER.CONTENT_RANGE, "bytes 0-0/0")

            return content

        def _counts(self, params):
            counts = {}

            session = self.get_session()

            if session is None:
                self.send_response(httplib.UNAUTHORIZED)
                self.send_header(HTTP_HEADER.CONNECTION, "close")
                return None

            self.send_response(httplib.OK)
            self.send_header(HTTP_HEADER.CONNECTION, "close")
            self.send_header(HTTP_HEADER.CONTENT_TYPE, "application/json")

            match = re.search(r"\d+\-\d+\-\d+", params.get("from", ""))
            if match:
                min_ = datetime.datetime.strptime(match.group(0), DATE_FORMAT)
            else:
                min_ = datetime.datetime.fromtimestamp(0)

            match = re.search(r"\d+\-\d+\-\d+", params.get("to", ""))
            if match:
                max_ = datetime.datetime.strptime(match.group(0), DATE_FORMAT)
            else:
                max_ = datetime.datetime.now()

            min_ = min_.replace(hour=0, minute=0, second=0, microsecond=0)
            max_ = max_.replace(hour=23, minute=59, second=59, microsecond=999999)

            for filepath in sorted(glob.glob(os.path.join(config.LOG_DIR, "*.log"))):
                filename = os.path.basename(filepath)
                if not re.search(r"\A\d{4}-\d{2}-\d{2}\.log\Z", filename):
                    continue
                try:
                    current = datetime.datetime.strptime(os.path.splitext(filename)[0], DATE_FORMAT)
                except:
                    if config.SHOW_DEBUG:
                        traceback.print_exc()
                else:
                    if min_ <= current <= max_:
                        timestamp = int(time.mktime(current.timetuple()))
                        size = os.path.getsize(filepath)
                        with open(filepath, "rb") as f:
                            content = f.read(io.DEFAULT_BUFFER_SIZE)
                            if size >= io.DEFAULT_BUFFER_SIZE:
                                total = 1.0 * content.count('\n') * size / io.DEFAULT_BUFFER_SIZE
                                counts[timestamp] = int(round(total / 100) * 100)
                            else:
                                counts[timestamp] = content.count('\n')

            return json.dumps(counts)

    class SSLReqHandler(ReqHandler):
        def setup(self):
            self.connection = self.request
            self.rfile = socket._fileobject(self.request, "rb", self.rbufsize)
            self.wfile = socket._fileobject(self.request, "wb", self.wbufsize)

    try:
        if pem:
            server = SSLThreadingServer((address or '', int(port) if str(port or "").isdigit() else 0), pem, SSLReqHandler)
        else:
            server = ThreadingServer((address or '', int(port) if str(port or "").isdigit() else 0), ReqHandler)
    except Exception as ex:
        if "Address already in use" in str(ex):
            exit("[!] another instance already running")
        elif "Name or service not known" in str(ex):
            exit("[!] invalid configuration value for 'HTTP_ADDRESS' ('%s')" % config.HTTP_ADDRESS)
        elif "Cannot assign requested address" in str(ex):
            exit("[!] can't use configuration value for 'HTTP_ADDRESS' ('%s')" % config.HTTP_ADDRESS)
        else:
            raise

    print "[i] starting HTTP%s server at 'http%s://%s:%d/'" % ('S' if pem else "", 's' if pem else "", server.server_address[0], server.server_address[1])

    print "[o] running..."

    if join:
        server.serve_forever()
    else:
        thread = threading.Thread(target=server.serve_forever)
        thread.daemon = True
        thread.start()