StringIO.StringIO

Here are the examples of the python api StringIO.StringIO taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: pypi-legacy
Source File: description_utils.py
View license
def extractPackageReadme(content, filename, filetype):
    '''Extract the README from a file and attempt to turn it into HTML.

    Return the source text and html version or empty strings in either case if
    extraction fails.
    '''
    text = ''
    if filename.endswith('.zip') or filename.endswith('.egg'):
        try:
            t = StringIO.StringIO(content)
            t.filename = filename
            zip = zipfile.ZipFile(t)
            l = zip.namelist()
        except zipfile.error:
            return '', ''
        for entry in l:
            parts = entry.split('/')
            if len(parts) != 2:
                continue
            filename = parts[-1]
            if filename.count('.') > 1:
                continue
            if filename.count('.') == 1:
                name, ext = filename.split('.')
            else:
                # just use the filename and assume a readme is plain text
                name = filename
                ext = 'txt'
            if name.upper() != 'README':
                continue
            if ext not in ('txt', 'rst', 'md'):
                return

            # grab the content and parse if it's something we might understand,
            # based on the file extension
            text = zip.open(entry).read()

            # we can only deal with UTF-8 so make it UTF-8 safe
            text = text.decode('utf-8', 'replace').encode('utf-8')

            if text:
                return text

    elif (filename.endswith('.tar.gz') or filename.endswith('.tgz') or
            filename.endswith('.tar.bz2') or filename.endswith('.tbz2')):
        # open the tar file with the appropriate compression
        ext = filename.split('.')[-1]
        if ext[-2:] == 'gz':
            file = StringIO.StringIO(content)
            file = gzip.GzipFile(filename, fileobj=file)
        else:
            file = StringIO.StringIO(bz2.decompress(content))
        try:
            tar = tarfile.TarFile(filename, 'r', file)
            l = tar.getmembers()
        except tarfile.TarError:
            return '', ''
        for entry in l:
            parts = entry.name.split('/')
            if len(parts) != 2:
                continue
            filename = parts[-1]
            if filename.count('.') > 1:
                continue
            if filename.count('.') == 1:
                name, ext = filename.split('.')
            else:
                # just use the filename and assume a readme is plain text
                name = filename
                ext = 'txt'
            if name.upper() != 'README':
                continue
            if ext not in ('txt', 'rst', 'md'):
                continue
            # grab the content and parse if it's something we might understand,
            # based on the file extension
            try:
                text = tar.extractfile(entry).read()

                # we can only deal with UTF-8 so make it UTF-8 safe
                text = text.decode('utf-8', 'replace').encode('utf-8')
            except:
                # issue 3521663: extraction may fail if entry is a symlink to
                # a non-existing file
                continue

            if text:
                return text

    return text

Example 2

View license
def wsgi_server(application):
    '''WSGI handler based on the Python wsgiref SimpleHandler.
    
    A WSGI application should return a iterable op StringTypes. 
    Any encoding must be handled by the WSGI application itself.
    '''
    
    # TODO - this wsgi handler executes the application and renders a page 
    # in memory completely before returning it as a response to the client. 
    # Thus, it does not "stream" the result back to the client. It should be 
    # possible though. The SimpleHandler accepts file-like stream objects. So, 
    # it should be just a matter of connecting 0MQ requests/response streams to 
    # the SimpleHandler requests and response streams. However, the Python API 
    # for Mongrel2 doesn't seem to support file-like stream objects for requests 
    # and responses. Unless I have missed something.
    
    while True:
        if DEBUG: print "WAITING FOR REQUEST"
        
        # receive a request
        req = conn.recv()
        if DEBUG: print "REQUEST BODY: %r\n" % req.body
        
        if req.is_disconnect():
            if DEBUG: print "DISCONNECT"
            continue #effectively ignore the disconnect from the client
        
        # Set a couple of environment attributes a.k.a. header attributes 
        # that are a must according to PEP 333
        environ = req.headers
        environ['SERVER_PROTOCOL'] = 'HTTP/1.1' # SimpleHandler expects a server_protocol, lets assume it is HTTP 1.1
        environ['REQUEST_METHOD'] = environ['METHOD']
        if ':' in environ['Host']:
            environ['SERVER_NAME'] = environ['Host'].split(':')[0]
            environ['SERVER_PORT'] = environ['Host'].split(':')[1]
        else:
            environ['SERVER_NAME'] = environ['Host']
            environ['SERVER_PORT'] = ''
        environ['SCRIPT_NAME'] = '' # empty for now
		# 26 aug 2010: Apparently Mongrel2 has started (around 1.0beta1) to quote urls and
		# apparently Django isn't expecting an already quoted string. So, I just
		# unquote the path_info here again so Django doesn't throw a "page not found" on 
		# urls with spaces and other characters in it.
        environ['PATH_INFO'] = urllib.unquote(environ['PATH'])
        if '?' in environ['URI']:
            environ['QUERY_STRING'] = environ['URI'].split('?')[1]
        else:
            environ['QUERY_STRING'] = ''
        if environ.has_key('Content-Length'):
            environ['CONTENT_LENGTH'] = environ['Content-Length'] # necessary for POST to work with Django
        environ['wsgi.input'] = req.body
        
        if DEBUG: print "ENVIRON: %r\n" % environ
        
        # SimpleHandler needs file-like stream objects for
        # requests, errors and reponses
        reqIO = StringIO.StringIO(req.body)
        errIO = StringIO.StringIO()
        respIO = StringIO.StringIO()
        
        # execute the application
        handler = SimpleHandler(reqIO, respIO, errIO, environ, multithread = False, multiprocess = False)
        handler.run(application)
        
        # Get the response and filter out the response (=data) itself,
        # the response headers, 
        # the response status code and the response status description
        response = respIO.getvalue()
        response = response.split("\r\n")
        data = response[-1]
        headers = dict([r.split(": ") for r in response[1:-2]])
        code = response[0][9:12]
        status = response[0][13:]
        
        # strip BOM's from response data
        # Especially the WSGI handler from Django seems to generate them (2 actually, huh?)
        # a BOM isn't really necessary and cause HTML parsing errors in Chrome and Safari
        # See also: http://www.xs4all.nl/~mechiel/projects/bomstrip/
        # Although I still find this a ugly hack, it does work.
        data = data.replace('\xef\xbb\xbf', '')
        
        # Get the generated errors
        errors = errIO.getvalue()
        
        # return the response
        if DEBUG: print "RESPONSE: %r\n" % response
        if errors:
            if DEBUG: print "ERRORS: %r" % errors
            data = "%s\r\n\r\n%s" % (data, errors)            
        conn.reply_http(req, data, code = code, status = status, headers = headers)

Example 3

Project: termite-data-server
Source File: anyserver.py
View license
def mongrel2_handler(application, conn, debug=False):
    """
    Based on :
    https://github.com/berry/Mongrel2-WSGI-Handler/blob/master/wsgi-handler.py

    WSGI handler based on the Python wsgiref SimpleHandler.
    A WSGI application should return a iterable op StringTypes.
    Any encoding must be handled by the WSGI application itself.
    """
    from wsgiref.handlers import SimpleHandler
    try:
        import cStringIO as StringIO
    except:
        import StringIO

    # TODO - this wsgi handler executes the application and renders a page
    # in memory completely before returning it as a response to the client.
    # Thus, it does not "stream" the result back to the client. It should be
    # possible though. The SimpleHandler accepts file-like stream objects. So,
    # it should be just a matter of connecting 0MQ requests/response streams to
    # the SimpleHandler requests and response streams. However, the Python API
    # for Mongrel2 doesn't seem to support file-like stream objects for requests
    # and responses. Unless I have missed something.

    while True:
        if debug:
            print "WAITING FOR REQUEST"

        # receive a request
        req = conn.recv()
        if debug:
            print "REQUEST BODY: %r\n" % req.body

        if req.is_disconnect():
            if debug:
                print "DISCONNECT"
            continue  # effectively ignore the disconnect from the client

        # Set a couple of environment attributes a.k.a. header attributes
        # that are a must according to PEP 333
        environ = req.headers
        environ['SERVER_PROTOCOL'] = 'HTTP/1.1'  # SimpleHandler expects a server_protocol, lets assume it is HTTP 1.1
        environ['REQUEST_METHOD'] = environ['METHOD']
        if ':' in environ['Host']:
            environ['SERVER_NAME'] = environ['Host'].split(':')[0]
            environ['SERVER_PORT'] = environ['Host'].split(':')[1]
        else:
            environ['SERVER_NAME'] = environ['Host']
            environ['SERVER_PORT'] = ''
        environ['SCRIPT_NAME'] = ''  # empty for now
        environ['PATH_INFO'] = urllib.unquote(environ['PATH'])
        if '?' in environ['URI']:
            environ['QUERY_STRING'] = environ['URI'].split('?')[1]
        else:
            environ['QUERY_STRING'] = ''
        if 'Content-Length' in environ:
            environ['CONTENT_LENGTH'] = environ[
                'Content-Length']  # necessary for POST to work with Django
        environ['wsgi.input'] = req.body

        if debug:
            print "ENVIRON: %r\n" % environ

        # SimpleHandler needs file-like stream objects for
        # requests, errors and responses
        reqIO = StringIO.StringIO(req.body)
        errIO = StringIO.StringIO()
        respIO = StringIO.StringIO()

        # execute the application
        handler = SimpleHandler(reqIO, respIO, errIO, environ,
                                multithread=False, multiprocess=False)
        handler.run(application)

        # Get the response and filter out the response (=data) itself,
        # the response headers,
        # the response status code and the response status description
        response = respIO.getvalue()
        response = response.split("\r\n")
        data = response[-1]
        headers = dict([r.split(": ") for r in response[1:-2]])
        code = response[0][9:12]
        status = response[0][13:]

        # strip BOM's from response data
        # Especially the WSGI handler from Django seems to generate them (2 actually, huh?)
        # a BOM isn't really necessary and cause HTML parsing errors in Chrome and Safari
        # See also: http://www.xs4all.nl/~mechiel/projects/bomstrip/
        # Although I still find this a ugly hack, it does work.
        data = data.replace('\xef\xbb\xbf', '')

        # Get the generated errors
        errors = errIO.getvalue()

        # return the response
        if debug:
            print "RESPONSE: %r\n" % response
        if errors:
            if debug:
                print "ERRORS: %r" % errors
            data = "%s\r\n\r\n%s" % (data, errors)
        conn.reply_http(
            req, data, code=code, status=status, headers=headers)

Example 4

Project: termite-visualizations
Source File: anyserver.py
View license
def mongrel2_handler(application, conn, debug=False):
    """
    Based on :
    https://github.com/berry/Mongrel2-WSGI-Handler/blob/master/wsgi-handler.py

    WSGI handler based on the Python wsgiref SimpleHandler.
    A WSGI application should return a iterable op StringTypes.
    Any encoding must be handled by the WSGI application itself.
    """
    from wsgiref.handlers import SimpleHandler
    try:
        import cStringIO as StringIO
    except:
        import StringIO

    # TODO - this wsgi handler executes the application and renders a page
    # in memory completely before returning it as a response to the client.
    # Thus, it does not "stream" the result back to the client. It should be
    # possible though. The SimpleHandler accepts file-like stream objects. So,
    # it should be just a matter of connecting 0MQ requests/response streams to
    # the SimpleHandler requests and response streams. However, the Python API
    # for Mongrel2 doesn't seem to support file-like stream objects for requests
    # and responses. Unless I have missed something.

    while True:
        if debug:
            print "WAITING FOR REQUEST"

        # receive a request
        req = conn.recv()
        if debug:
            print "REQUEST BODY: %r\n" % req.body

        if req.is_disconnect():
            if debug:
                print "DISCONNECT"
            continue  # effectively ignore the disconnect from the client

        # Set a couple of environment attributes a.k.a. header attributes
        # that are a must according to PEP 333
        environ = req.headers
        environ['SERVER_PROTOCOL'] = 'HTTP/1.1'  # SimpleHandler expects a server_protocol, lets assume it is HTTP 1.1
        environ['REQUEST_METHOD'] = environ['METHOD']
        if ':' in environ['Host']:
            environ['SERVER_NAME'] = environ['Host'].split(':')[0]
            environ['SERVER_PORT'] = environ['Host'].split(':')[1]
        else:
            environ['SERVER_NAME'] = environ['Host']
            environ['SERVER_PORT'] = ''
        environ['SCRIPT_NAME'] = ''  # empty for now
        environ['PATH_INFO'] = urllib.unquote(environ['PATH'])
        if '?' in environ['URI']:
            environ['QUERY_STRING'] = environ['URI'].split('?')[1]
        else:
            environ['QUERY_STRING'] = ''
        if 'Content-Length' in environ:
            environ['CONTENT_LENGTH'] = environ[
                'Content-Length']  # necessary for POST to work with Django
        environ['wsgi.input'] = req.body

        if debug:
            print "ENVIRON: %r\n" % environ

        # SimpleHandler needs file-like stream objects for
        # requests, errors and responses
        reqIO = StringIO.StringIO(req.body)
        errIO = StringIO.StringIO()
        respIO = StringIO.StringIO()

        # execute the application
        handler = SimpleHandler(reqIO, respIO, errIO, environ,
                                multithread=False, multiprocess=False)
        handler.run(application)

        # Get the response and filter out the response (=data) itself,
        # the response headers,
        # the response status code and the response status description
        response = respIO.getvalue()
        response = response.split("\r\n")
        data = response[-1]
        headers = dict([r.split(": ") for r in response[1:-2]])
        code = response[0][9:12]
        status = response[0][13:]

        # strip BOM's from response data
        # Especially the WSGI handler from Django seems to generate them (2 actually, huh?)
        # a BOM isn't really necessary and cause HTML parsing errors in Chrome and Safari
        # See also: http://www.xs4all.nl/~mechiel/projects/bomstrip/
        # Although I still find this a ugly hack, it does work.
        data = data.replace('\xef\xbb\xbf', '')

        # Get the generated errors
        errors = errIO.getvalue()

        # return the response
        if debug:
            print "RESPONSE: %r\n" % response
        if errors:
            if debug:
                print "ERRORS: %r" % errors
            data = "%s\r\n\r\n%s" % (data, errors)
        conn.reply_http(
            req, data, code=code, status=status, headers=headers)

Example 5

Project: FanFicFare
Source File: writer_epub.py
View license
    def writeStoryImpl(self, out):

        ## Python 2.5 ZipFile is rather more primative than later
        ## versions.  It can operate on a file, or on a StringIO, but
        ## not on an open stream.  OTOH, I suspect we would have had
        ## problems with closing and opening again to change the
        ## compression type anyway.
        zipio = StringIO.StringIO()

        ## mimetype must be first file and uncompressed.  Python 2.5
        ## ZipFile can't change compression type file-by-file, so we
        ## have to close and re-open
        outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
        outputepub.debug=3
        outputepub.writestr('mimetype','application/epub+zip')
        outputepub.close()

        ## Re-open file for content.
        outputepub = ZipFile(zipio, 'a', compression=ZIP_DEFLATED)
        outputepub.debug=3

        ## Create META-INF/container.xml file.  The only thing it does is
        ## point to content.opf
        containerdom = getDOMImplementation().createDocument(None, "container", None)
        containertop = containerdom.documentElement
        containertop.setAttribute("version","1.0")
        containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")
        rootfiles = containerdom.createElement("rootfiles")
        containertop.appendChild(rootfiles)
        rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf",
                                                              "media-type":"application/oebps-package+xml"}))
        outputepub.writestr("META-INF/container.xml",containerdom.toxml(encoding='utf-8'))
        containerdom.unlink()
        del containerdom

        ## Epub has two metadata files with real data.  We're putting
        ## them in content.opf (pointed to by META-INF/container.xml)
        ## and toc.ncx (pointed to by content.opf)

        ## content.opf contains metadata, a 'manifest' list of all
        ## other included files, and another 'spine' list of the items in the
        ## file

        uniqueid= 'fanficfare-uid:%s-u%s-s%s' % (
            self.getMetadata('site'),
            self.story.getList('authorId')[0],
            self.getMetadata('storyId'))

        contentdom = getDOMImplementation().createDocument(None, "package", None)
        package = contentdom.documentElement
        package.setAttribute("version","2.0")
        package.setAttribute("xmlns","http://www.idpf.org/2007/opf")
        package.setAttribute("unique-identifier","fanficfare-uid")
        metadata=newTag(contentdom,"metadata",
                        attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/",
                               "xmlns:opf":"http://www.idpf.org/2007/opf"})
        package.appendChild(metadata)

        metadata.appendChild(newTag(contentdom,"dc:identifier",
                                    text=uniqueid,
                                    attrs={"id":"fanficfare-uid"}))

        if self.getMetadata('title'):
            metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title')))

        if self.getMetadata('author'):
            if self.story.isList('author'):
                for auth in self.story.getList('author'):
                    metadata.appendChild(newTag(contentdom,"dc:creator",
                                                attrs={"opf:role":"aut"},
                                                text=auth))
            else:
                metadata.appendChild(newTag(contentdom,"dc:creator",
                                            attrs={"opf:role":"aut"},
                                            text=self.getMetadata('author')))

        metadata.appendChild(newTag(contentdom,"dc:contributor",text="FanFicFare [https://github.com/JimmXinu/FanFicFare]",attrs={"opf:role":"bkp"}))
        metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
        if self.story.getMetadata('langcode'):
            metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
        else:
            metadata.appendChild(newTag(contentdom,"dc:language",text='en'))

        #  published, created, updated, calibre
        #  Leave calling self.story.getMetadataRaw directly in case date format changes.
        if self.story.getMetadataRaw('datePublished'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"publication"},
                                        text=self.story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")))

        if self.story.getMetadataRaw('dateCreated'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"creation"},
                                        text=self.story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")))

        if self.story.getMetadataRaw('dateUpdated'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"modification"},
                                        text=self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%d")))
            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"name":"calibre:timestamp",
                                               "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))

        if self.getMetadata('description'):
            metadata.appendChild(newTag(contentdom,"dc:description",text=
                                        self.getMetadata('description')))

        for subject in self.story.getSubjectTags():
            metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))


        if self.getMetadata('site'):
            metadata.appendChild(newTag(contentdom,"dc:publisher",
                                        text=self.getMetadata('site')))

        if self.getMetadata('storyUrl'):
            metadata.appendChild(newTag(contentdom,"dc:identifier",
                                        attrs={"opf:scheme":"URL"},
                                        text=self.getMetadata('storyUrl')))
            metadata.appendChild(newTag(contentdom,"dc:source",
                                        text=self.getMetadata('storyUrl')))

        ## end of metadata, create manifest.
        items = [] # list of (id, href, type, title) tuples(all strings)
        itemrefs = [] # list of strings -- idrefs from .opfs' spines
        items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file,
                                                                   ## but it needs to be in the items manifest.

        guide = None
        coverIO = None

        coverimgid = "image0000"
        if not self.story.cover and self.story.oldcover:
            logger.debug("writer_epub: no new cover, has old cover, write image.")
            (oldcoverhtmlhref,
             oldcoverhtmltype,
             oldcoverhtmldata,
             oldcoverimghref,
             oldcoverimgtype,
             oldcoverimgdata) = self.story.oldcover
            outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
            outputepub.writestr(oldcoverimghref,oldcoverimgdata)

            coverimgid = "image0"
            items.append((coverimgid,
                          oldcoverimghref,
                          oldcoverimgtype,
                          None))
            items.append(("cover",oldcoverhtmlhref,oldcoverhtmltype,None))
            itemrefs.append("cover")
            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0",
                                                           "name":"cover"}))
            guide = newTag(contentdom,"guide")
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                                   "title":"Cover",
                                                                   "href":oldcoverhtmlhref}))



        if self.getConfig('include_images'):
            imgcount=0
            for imgmap in self.story.getImgUrls():
                imgfile = "OEBPS/"+imgmap['newsrc']
                outputepub.writestr(imgfile,imgmap['data'])
                items.append(("image%04d"%imgcount,
                              imgfile,
                              imgmap['mime'],
                              None))
                imgcount+=1
                if 'cover' in imgfile:
                    # make sure coverimgid is set to the cover, not
                    # just the first image.
                    coverimgid = items[-1][0]


        items.append(("style","OEBPS/stylesheet.css","text/css",None))

        if self.story.cover:
            # Note that the id of the cover xhmtl *must* be 'cover'
            # for it to work on Nook.
            items.append(("cover","OEBPS/cover.xhtml","application/xhtml+xml",None))
            itemrefs.append("cover")
            #
            # <meta name="cover" content="cover.jpg"/>
            metadata.appendChild(newTag(contentdom,"meta",{"content":coverimgid,
                                                           "name":"cover"}))
            # cover stuff for later:
            # at end of <package>:
            # <guide>
            # <reference type="cover" title="Cover" href="Text/cover.xhtml"/>
            # </guide>
            guide = newTag(contentdom,"guide")
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                       "title":"Cover",
                                                       "href":"OEBPS/cover.xhtml"}))

            if self.hasConfig("cover_content"):
                COVER = string.Template(self.getConfig("cover_content"))
            else:
                COVER = self.EPUB_COVER
            coverIO = StringIO.StringIO()
            coverIO.write(COVER.substitute(dict(self.story.getAllMetadata().items()+{'coverimg':self.story.cover}.items())))

        if self.getConfig("include_titlepage"):
            items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
            itemrefs.append("title_page")
        if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
            items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
            itemrefs.append("toc_page")

        ## save where to insert logpage.
        logpage_indices = (len(items),len(itemrefs))

        dologpage = ( self.getConfig("include_logpage") == "smart" and \
                          (self.story.logfile or self.story.getMetadataRaw("status") == "In-Progress") )  \
                     or self.getConfig("include_logpage") == "true"

        ## collect chapter urls and file names for internalize_text_links option.
        chapurlmap = {}
        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
            if chap.html:
                i=index+1
                items.append(("file%04d"%i,
                              "OEBPS/file%04d.xhtml"%i,
                              "application/xhtml+xml",
                              chap.title))
                itemrefs.append("file%04d"%i)
                chapurlmap[chap.url]="file%04d.xhtml"%i # url -> relative epub file name.

        if dologpage:
            if self.getConfig("logpage_at_end") == "true":
                ## insert logpage after chapters.
                logpage_indices = (len(items),len(itemrefs))
            items.insert(logpage_indices[0],("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
            itemrefs.insert(logpage_indices[1],"log_page")

        manifest = contentdom.createElement("manifest")
        package.appendChild(manifest)
        for item in items:
            (id,href,type,title)=item
            manifest.appendChild(newTag(contentdom,"item",
                                        attrs={'id':id,
                                               'href':href,
                                               'media-type':type}))

        spine = newTag(contentdom,"spine",attrs={"toc":"ncx"})
        package.appendChild(spine)
        for itemref in itemrefs:
            spine.appendChild(newTag(contentdom,"itemref",
                                     attrs={"idref":itemref,
                                            "linear":"yes"}))
        # guide only exists if there's a cover.
        if guide:
            package.appendChild(guide)

        # write content.opf to zip.
        contentxml = contentdom.toxml(encoding='utf-8')

        # tweak for brain damaged Nook STR.  Nook insists on name before content.
        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%coverimgid,
                                        '<meta name="cover" content="%s"/>'%coverimgid)
        outputepub.writestr("content.opf",contentxml)

        contentdom.unlink()
        del contentdom

        ## create toc.ncx file
        tocncxdom = getDOMImplementation().createDocument(None, "ncx", None)
        ncx = tocncxdom.documentElement
        ncx.setAttribute("version","2005-1")
        ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/")
        head = tocncxdom.createElement("head")
        ncx.appendChild(head)
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:uid", "content":uniqueid}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:depth", "content":"1"}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:totalPageCount", "content":"0"}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:maxPageNumber", "content":"0"}))

        docTitle = tocncxdom.createElement("docTitle")
        docTitle.appendChild(newTag(tocncxdom,"text",text=self.getMetadata('title')))
        ncx.appendChild(docTitle)

        tocnavMap = tocncxdom.createElement("navMap")
        ncx.appendChild(tocnavMap)

        # <navPoint id="<id>" playOrder="<risingnumberfrom0>">
        #   <navLabel>
        #     <text><chapter title></text>
        #   </navLabel>
        #   <content src="<chapterfile>"/>
        # </navPoint>
        index=0
        for item in items:
            (id,href,type,title)=item
            # only items to be skipped, cover.xhtml, images, toc.ncx, stylesheet.css, should have no title.
            if title :
                navPoint = newTag(tocncxdom,"navPoint",
                                  attrs={'id':id,
                                         'playOrder':unicode(index)})
                tocnavMap.appendChild(navPoint)
                navLabel = newTag(tocncxdom,"navLabel")
                navPoint.appendChild(navLabel)
                ## the xml library will re-escape as needed.
                navLabel.appendChild(newTag(tocncxdom,"text",text=stripHTML(title)))
                navPoint.appendChild(newTag(tocncxdom,"content",attrs={"src":href}))
                index=index+1

        # write toc.ncx to zip file
        outputepub.writestr("toc.ncx",tocncxdom.toxml(encoding='utf-8'))
        tocncxdom.unlink()
        del tocncxdom

        # write stylesheet.css file.
        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata()))

        # write title page.
        if self.getConfig("titlepage_use_table"):
            TITLE_PAGE_START  = self.EPUB_TABLE_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TABLE_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TABLE_TITLE_WIDE_ENTRY
            NO_TITLE_ENTRY    = self.EPUB_TABLE_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TABLE_TITLE_PAGE_END
        else:
            TITLE_PAGE_START  = self.EPUB_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TITLE_ENTRY # same, only wide in tables.
            NO_TITLE_ENTRY    = self.EPUB_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TITLE_PAGE_END

        if coverIO:
            outputepub.writestr("OEBPS/cover.xhtml",coverIO.getvalue())
            coverIO.close()

        titlepageIO = StringIO.StringIO()
        self.writeTitlePage(out=titlepageIO,
                            START=TITLE_PAGE_START,
                            ENTRY=TITLE_ENTRY,
                            WIDE_ENTRY=WIDE_TITLE_ENTRY,
                            END=TITLE_PAGE_END,
                            NO_TITLE_ENTRY=NO_TITLE_ENTRY)
        if titlepageIO.getvalue(): # will be false if no title page.
            outputepub.writestr("OEBPS/title_page.xhtml",titlepageIO.getvalue())
        titlepageIO.close()

        # write toc page.
        tocpageIO = StringIO.StringIO()
        self.writeTOCPage(tocpageIO,
                          self.EPUB_TOC_PAGE_START,
                          self.EPUB_TOC_ENTRY,
                          self.EPUB_TOC_PAGE_END)
        if tocpageIO.getvalue(): # will be false if no toc page.
            outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
        tocpageIO.close()

        if dologpage:
            # write log page.
            logpageIO = StringIO.StringIO()
            self.writeLogPage(logpageIO)
            outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
            logpageIO.close()

        if self.hasConfig('chapter_start'):
            CHAPTER_START = string.Template(self.getConfig("chapter_start"))
        else:
            CHAPTER_START = self.EPUB_CHAPTER_START

        if self.hasConfig('chapter_end'):
            CHAPTER_END = string.Template(self.getConfig("chapter_end"))
        else:
            CHAPTER_END = self.EPUB_CHAPTER_END

        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
            if chap.html:
                chap_data = chap.html
                if self.getConfig('internalize_text_links'):
                    soup = bs4.BeautifulSoup(chap.html,'html5lib')
                    changed=False
                    for alink in soup.find_all('a'):
                        if alink.has_attr('href') and alink['href'] in chapurlmap:
                            alink['href']=chapurlmap[alink['href']]
                            changed=True
                    if changed:
                        chap_data = unicode(soup)
                        # Don't want html, head or body tags in
                        # chapter html--bs4 insists on adding them.
                        chap_data = re.sub(r"</?(html|head|body)[^>]*>\r?\n?","",chap_data)

                #logger.debug('Writing chapter text for: %s' % chap.title)
                vals={'url':removeEntities(chap.url),
                      'chapter':removeEntities(chap.title),
                      'origchapter':removeEntities(chap.origtitle),
                      'tocchapter':removeEntities(chap.toctitle),
                      'index':"%04d"%(index+1),
                      'number':index+1}
                # escape double quotes in all vals.
                for k,v in vals.items():
                    if isinstance(v,basestring): vals[k]=v.replace('"','&quot;')
                fullhtml = CHAPTER_START.substitute(vals) + \
                    chap_data.strip() + \
                    CHAPTER_END.substitute(vals)
                # strip to avoid ever growning numbers of newlines.
                # ffnet(& maybe others) gives the whole chapter text
                # as one line.  This causes problems for nook(at
                # least) when the chapter size starts getting big
                # (200k+)
                fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)

                outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                del fullhtml

        if self.story.calibrebookmark:
            outputepub.writestr("META-INF/calibre_bookmarks.txt",self.story.calibrebookmark)

	# declares all the files created by Windows.  otherwise, when
        # it runs in appengine, windows unzips the files as 000 perms.
        for zf in outputepub.filelist:
            zf.create_system = 0
        outputepub.close()
        out.write(zipio.getvalue())
        zipio.close()

Example 6

Project: FanFicFare
Source File: writer_epub.py
View license
    def writeStoryImpl(self, out):

        ## Python 2.5 ZipFile is rather more primative than later
        ## versions.  It can operate on a file, or on a StringIO, but
        ## not on an open stream.  OTOH, I suspect we would have had
        ## problems with closing and opening again to change the
        ## compression type anyway.
        zipio = StringIO.StringIO()

        ## mimetype must be first file and uncompressed.  Python 2.5
        ## ZipFile can't change compression type file-by-file, so we
        ## have to close and re-open
        outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
        outputepub.debug=3
        outputepub.writestr('mimetype','application/epub+zip')
        outputepub.close()

        ## Re-open file for content.
        outputepub = ZipFile(zipio, 'a', compression=ZIP_DEFLATED)
        outputepub.debug=3

        ## Create META-INF/container.xml file.  The only thing it does is
        ## point to content.opf
        containerdom = getDOMImplementation().createDocument(None, "container", None)
        containertop = containerdom.documentElement
        containertop.setAttribute("version","1.0")
        containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")
        rootfiles = containerdom.createElement("rootfiles")
        containertop.appendChild(rootfiles)
        rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf",
                                                              "media-type":"application/oebps-package+xml"}))
        outputepub.writestr("META-INF/container.xml",containerdom.toxml(encoding='utf-8'))
        containerdom.unlink()
        del containerdom

        ## Epub has two metadata files with real data.  We're putting
        ## them in content.opf (pointed to by META-INF/container.xml)
        ## and toc.ncx (pointed to by content.opf)

        ## content.opf contains metadata, a 'manifest' list of all
        ## other included files, and another 'spine' list of the items in the
        ## file

        uniqueid= 'fanficfare-uid:%s-u%s-s%s' % (
            self.getMetadata('site'),
            self.story.getList('authorId')[0],
            self.getMetadata('storyId'))

        contentdom = getDOMImplementation().createDocument(None, "package", None)
        package = contentdom.documentElement
        package.setAttribute("version","2.0")
        package.setAttribute("xmlns","http://www.idpf.org/2007/opf")
        package.setAttribute("unique-identifier","fanficfare-uid")
        metadata=newTag(contentdom,"metadata",
                        attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/",
                               "xmlns:opf":"http://www.idpf.org/2007/opf"})
        package.appendChild(metadata)

        metadata.appendChild(newTag(contentdom,"dc:identifier",
                                    text=uniqueid,
                                    attrs={"id":"fanficfare-uid"}))

        if self.getMetadata('title'):
            metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title')))

        if self.getMetadata('author'):
            if self.story.isList('author'):
                for auth in self.story.getList('author'):
                    metadata.appendChild(newTag(contentdom,"dc:creator",
                                                attrs={"opf:role":"aut"},
                                                text=auth))
            else:
                metadata.appendChild(newTag(contentdom,"dc:creator",
                                            attrs={"opf:role":"aut"},
                                            text=self.getMetadata('author')))

        metadata.appendChild(newTag(contentdom,"dc:contributor",text="FanFicFare [https://github.com/JimmXinu/FanFicFare]",attrs={"opf:role":"bkp"}))
        metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
        if self.story.getMetadata('langcode'):
            metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
        else:
            metadata.appendChild(newTag(contentdom,"dc:language",text='en'))

        #  published, created, updated, calibre
        #  Leave calling self.story.getMetadataRaw directly in case date format changes.
        if self.story.getMetadataRaw('datePublished'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"publication"},
                                        text=self.story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")))

        if self.story.getMetadataRaw('dateCreated'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"creation"},
                                        text=self.story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")))

        if self.story.getMetadataRaw('dateUpdated'):
            metadata.appendChild(newTag(contentdom,"dc:date",
                                        attrs={"opf:event":"modification"},
                                        text=self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%d")))
            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"name":"calibre:timestamp",
                                               "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))

        if self.getMetadata('description'):
            metadata.appendChild(newTag(contentdom,"dc:description",text=
                                        self.getMetadata('description')))

        for subject in self.story.getSubjectTags():
            metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))


        if self.getMetadata('site'):
            metadata.appendChild(newTag(contentdom,"dc:publisher",
                                        text=self.getMetadata('site')))

        if self.getMetadata('storyUrl'):
            metadata.appendChild(newTag(contentdom,"dc:identifier",
                                        attrs={"opf:scheme":"URL"},
                                        text=self.getMetadata('storyUrl')))
            metadata.appendChild(newTag(contentdom,"dc:source",
                                        text=self.getMetadata('storyUrl')))

        ## end of metadata, create manifest.
        items = [] # list of (id, href, type, title) tuples(all strings)
        itemrefs = [] # list of strings -- idrefs from .opfs' spines
        items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file,
                                                                   ## but it needs to be in the items manifest.

        guide = None
        coverIO = None

        coverimgid = "image0000"
        if not self.story.cover and self.story.oldcover:
            logger.debug("writer_epub: no new cover, has old cover, write image.")
            (oldcoverhtmlhref,
             oldcoverhtmltype,
             oldcoverhtmldata,
             oldcoverimghref,
             oldcoverimgtype,
             oldcoverimgdata) = self.story.oldcover
            outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
            outputepub.writestr(oldcoverimghref,oldcoverimgdata)

            coverimgid = "image0"
            items.append((coverimgid,
                          oldcoverimghref,
                          oldcoverimgtype,
                          None))
            items.append(("cover",oldcoverhtmlhref,oldcoverhtmltype,None))
            itemrefs.append("cover")
            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0",
                                                           "name":"cover"}))
            guide = newTag(contentdom,"guide")
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                                   "title":"Cover",
                                                                   "href":oldcoverhtmlhref}))



        if self.getConfig('include_images'):
            imgcount=0
            for imgmap in self.story.getImgUrls():
                imgfile = "OEBPS/"+imgmap['newsrc']
                outputepub.writestr(imgfile,imgmap['data'])
                items.append(("image%04d"%imgcount,
                              imgfile,
                              imgmap['mime'],
                              None))
                imgcount+=1
                if 'cover' in imgfile:
                    # make sure coverimgid is set to the cover, not
                    # just the first image.
                    coverimgid = items[-1][0]


        items.append(("style","OEBPS/stylesheet.css","text/css",None))

        if self.story.cover:
            # Note that the id of the cover xhmtl *must* be 'cover'
            # for it to work on Nook.
            items.append(("cover","OEBPS/cover.xhtml","application/xhtml+xml",None))
            itemrefs.append("cover")
            #
            # <meta name="cover" content="cover.jpg"/>
            metadata.appendChild(newTag(contentdom,"meta",{"content":coverimgid,
                                                           "name":"cover"}))
            # cover stuff for later:
            # at end of <package>:
            # <guide>
            # <reference type="cover" title="Cover" href="Text/cover.xhtml"/>
            # </guide>
            guide = newTag(contentdom,"guide")
            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
                                                       "title":"Cover",
                                                       "href":"OEBPS/cover.xhtml"}))

            if self.hasConfig("cover_content"):
                COVER = string.Template(self.getConfig("cover_content"))
            else:
                COVER = self.EPUB_COVER
            coverIO = StringIO.StringIO()
            coverIO.write(COVER.substitute(dict(self.story.getAllMetadata().items()+{'coverimg':self.story.cover}.items())))

        if self.getConfig("include_titlepage"):
            items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
            itemrefs.append("title_page")
        if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
            items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
            itemrefs.append("toc_page")

        ## save where to insert logpage.
        logpage_indices = (len(items),len(itemrefs))

        dologpage = ( self.getConfig("include_logpage") == "smart" and \
                          (self.story.logfile or self.story.getMetadataRaw("status") == "In-Progress") )  \
                     or self.getConfig("include_logpage") == "true"

        ## collect chapter urls and file names for internalize_text_links option.
        chapurlmap = {}
        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
            if chap.html:
                i=index+1
                items.append(("file%04d"%i,
                              "OEBPS/file%04d.xhtml"%i,
                              "application/xhtml+xml",
                              chap.title))
                itemrefs.append("file%04d"%i)
                chapurlmap[chap.url]="file%04d.xhtml"%i # url -> relative epub file name.

        if dologpage:
            if self.getConfig("logpage_at_end") == "true":
                ## insert logpage after chapters.
                logpage_indices = (len(items),len(itemrefs))
            items.insert(logpage_indices[0],("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
            itemrefs.insert(logpage_indices[1],"log_page")

        manifest = contentdom.createElement("manifest")
        package.appendChild(manifest)
        for item in items:
            (id,href,type,title)=item
            manifest.appendChild(newTag(contentdom,"item",
                                        attrs={'id':id,
                                               'href':href,
                                               'media-type':type}))

        spine = newTag(contentdom,"spine",attrs={"toc":"ncx"})
        package.appendChild(spine)
        for itemref in itemrefs:
            spine.appendChild(newTag(contentdom,"itemref",
                                     attrs={"idref":itemref,
                                            "linear":"yes"}))
        # guide only exists if there's a cover.
        if guide:
            package.appendChild(guide)

        # write content.opf to zip.
        contentxml = contentdom.toxml(encoding='utf-8')

        # tweak for brain damaged Nook STR.  Nook insists on name before content.
        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%coverimgid,
                                        '<meta name="cover" content="%s"/>'%coverimgid)
        outputepub.writestr("content.opf",contentxml)

        contentdom.unlink()
        del contentdom

        ## create toc.ncx file
        tocncxdom = getDOMImplementation().createDocument(None, "ncx", None)
        ncx = tocncxdom.documentElement
        ncx.setAttribute("version","2005-1")
        ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/")
        head = tocncxdom.createElement("head")
        ncx.appendChild(head)
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:uid", "content":uniqueid}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:depth", "content":"1"}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:totalPageCount", "content":"0"}))
        head.appendChild(newTag(tocncxdom,"meta",
                                attrs={"name":"dtb:maxPageNumber", "content":"0"}))

        docTitle = tocncxdom.createElement("docTitle")
        docTitle.appendChild(newTag(tocncxdom,"text",text=self.getMetadata('title')))
        ncx.appendChild(docTitle)

        tocnavMap = tocncxdom.createElement("navMap")
        ncx.appendChild(tocnavMap)

        # <navPoint id="<id>" playOrder="<risingnumberfrom0>">
        #   <navLabel>
        #     <text><chapter title></text>
        #   </navLabel>
        #   <content src="<chapterfile>"/>
        # </navPoint>
        index=0
        for item in items:
            (id,href,type,title)=item
            # only items to be skipped, cover.xhtml, images, toc.ncx, stylesheet.css, should have no title.
            if title :
                navPoint = newTag(tocncxdom,"navPoint",
                                  attrs={'id':id,
                                         'playOrder':unicode(index)})
                tocnavMap.appendChild(navPoint)
                navLabel = newTag(tocncxdom,"navLabel")
                navPoint.appendChild(navLabel)
                ## the xml library will re-escape as needed.
                navLabel.appendChild(newTag(tocncxdom,"text",text=stripHTML(title)))
                navPoint.appendChild(newTag(tocncxdom,"content",attrs={"src":href}))
                index=index+1

        # write toc.ncx to zip file
        outputepub.writestr("toc.ncx",tocncxdom.toxml(encoding='utf-8'))
        tocncxdom.unlink()
        del tocncxdom

        # write stylesheet.css file.
        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata()))

        # write title page.
        if self.getConfig("titlepage_use_table"):
            TITLE_PAGE_START  = self.EPUB_TABLE_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TABLE_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TABLE_TITLE_WIDE_ENTRY
            NO_TITLE_ENTRY    = self.EPUB_TABLE_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TABLE_TITLE_PAGE_END
        else:
            TITLE_PAGE_START  = self.EPUB_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TITLE_ENTRY # same, only wide in tables.
            NO_TITLE_ENTRY    = self.EPUB_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TITLE_PAGE_END

        if coverIO:
            outputepub.writestr("OEBPS/cover.xhtml",coverIO.getvalue())
            coverIO.close()

        titlepageIO = StringIO.StringIO()
        self.writeTitlePage(out=titlepageIO,
                            START=TITLE_PAGE_START,
                            ENTRY=TITLE_ENTRY,
                            WIDE_ENTRY=WIDE_TITLE_ENTRY,
                            END=TITLE_PAGE_END,
                            NO_TITLE_ENTRY=NO_TITLE_ENTRY)
        if titlepageIO.getvalue(): # will be false if no title page.
            outputepub.writestr("OEBPS/title_page.xhtml",titlepageIO.getvalue())
        titlepageIO.close()

        # write toc page.
        tocpageIO = StringIO.StringIO()
        self.writeTOCPage(tocpageIO,
                          self.EPUB_TOC_PAGE_START,
                          self.EPUB_TOC_ENTRY,
                          self.EPUB_TOC_PAGE_END)
        if tocpageIO.getvalue(): # will be false if no toc page.
            outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
        tocpageIO.close()

        if dologpage:
            # write log page.
            logpageIO = StringIO.StringIO()
            self.writeLogPage(logpageIO)
            outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
            logpageIO.close()

        if self.hasConfig('chapter_start'):
            CHAPTER_START = string.Template(self.getConfig("chapter_start"))
        else:
            CHAPTER_START = self.EPUB_CHAPTER_START

        if self.hasConfig('chapter_end'):
            CHAPTER_END = string.Template(self.getConfig("chapter_end"))
        else:
            CHAPTER_END = self.EPUB_CHAPTER_END

        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
            if chap.html:
                chap_data = chap.html
                if self.getConfig('internalize_text_links'):
                    soup = bs4.BeautifulSoup(chap.html,'html5lib')
                    changed=False
                    for alink in soup.find_all('a'):
                        if alink.has_attr('href') and alink['href'] in chapurlmap:
                            alink['href']=chapurlmap[alink['href']]
                            changed=True
                    if changed:
                        chap_data = unicode(soup)
                        # Don't want html, head or body tags in
                        # chapter html--bs4 insists on adding them.
                        chap_data = re.sub(r"</?(html|head|body)[^>]*>\r?\n?","",chap_data)

                #logger.debug('Writing chapter text for: %s' % chap.title)
                vals={'url':removeEntities(chap.url),
                      'chapter':removeEntities(chap.title),
                      'origchapter':removeEntities(chap.origtitle),
                      'tocchapter':removeEntities(chap.toctitle),
                      'index':"%04d"%(index+1),
                      'number':index+1}
                # escape double quotes in all vals.
                for k,v in vals.items():
                    if isinstance(v,basestring): vals[k]=v.replace('"','&quot;')
                fullhtml = CHAPTER_START.substitute(vals) + \
                    chap_data.strip() + \
                    CHAPTER_END.substitute(vals)
                # strip to avoid ever growning numbers of newlines.
                # ffnet(& maybe others) gives the whole chapter text
                # as one line.  This causes problems for nook(at
                # least) when the chapter size starts getting big
                # (200k+)
                fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)

                outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                del fullhtml

        if self.story.calibrebookmark:
            outputepub.writestr("META-INF/calibre_bookmarks.txt",self.story.calibrebookmark)

	# declares all the files created by Windows.  otherwise, when
        # it runs in appengine, windows unzips the files as 000 perms.
        for zf in outputepub.filelist:
            zf.create_system = 0
        outputepub.close()
        out.write(zipio.getvalue())
        zipio.close()

Example 7

Project: franklin
Source File: readers_test.py
View license
    @staticmethod
    def test_repr():
        'It test the repr reader'

        assert u'a (h)' == _cast_to_class("u'a (h)'")

        assert ('adios',) == _cast_to_class("('adios',)")
        assert ['arab1', 'arab2'] == _cast_to_class("['arab1', 'arab2']")
        assert ('arab1', 'arab2') == _cast_to_class("('arab1', 'arab2')")
        result = _cast_to_class("{1: 2}")
        assert {1: 2}  == result

        assert {'al': {'c': 1}, 'T': 2} ==  _cast_to_class("{'al': {'c': 1}, 'T': 2}")

        seq1 = SeqWithQuality(seq=Seq('ATCT'))
        seq2 = SeqWithQuality(seq=Seq('AAAA'))
        fcontent = repr(seq1) + '\n' + repr(seq2) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seqs = list(seqs_in_file(fhand, format='repr'))
        assert repr(seqs[0]) == repr(seq1)
        assert repr(seqs[1]) == repr(seq2)

        #with quality
        seq1 = SeqWithQuality(seq=Seq('ATCT'), qual=[10, 2, 3, 4])
        fcontent = repr(seq1) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seqs = list(seqs_in_file(fhand, format='repr'))
        assert repr(seqs[0]) == repr(seq1)

        #a seq with features
        seq1 = SeqWithQuality(seq=Seq('GAAAAGATGTG', SingleLetterAlphabet()),
                             id='seq', name='seq', description='', dbxrefs=[],
                    features=[SeqFeature(FeatureLocation(ExactPosition(478),
                                                         ExactPosition(478)),
                                         type='intron',
                                         qualifiers={'db': 'tomato'} ),
                              SeqFeature(FeatureLocation(ExactPosition(572),
                                                         ExactPosition(572)),
                                         type='intron',
                                         qualifiers={'db': 'tomato'} )],
                             annotations={}, qual=None)
        fcontent = repr(seq1) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seq0 = list(seqs_in_file(fhand, format='repr'))[0]

        assert seq0.seq == seq1.seq
        assert seq0.qual == seq1.qual
        assert seq0.description == seq1.description
        assert seq0.annotations == seq1.annotations
        feat0 = seq0.features[0]
        feat1 = seq1.features[0]
        assert feat0.type == feat1.type
        assert feat0.qualifiers == feat1.qualifiers
        assert str(feat0.location) == str(feat1.location)

        #some more qualifiers
        fhand = tempfile.NamedTemporaryFile(suffix='.repr')
        seq1 = SeqWithQuality(id='seqid', name='seqname',
                         description='seqdescription', seq=Seq('ATGAT'))
        seq1.letter_annotations["phred_quality"] = [40, 40, 38, 30, 30]
        seq1.annotations['source'] = 'ara1'

        seqfeature = SeqFeature(location=FeatureLocation(5, 8),
                                type='orthologs',
                                qualifiers={'arabidposis':['arab1', 'arab2']})
        seq1.features.append(seqfeature)

        fcontent = repr(seq1) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seq0 = list(seqs_in_file(fhand, format='repr'))[0]

        assert seq0.seq == seq1.seq
        assert seq0.qual == seq1.qual
        assert seq0.description == seq1.description
        assert seq0.annotations == seq1.annotations
        feat0 = seq0.features[0]
        feat1 = seq1.features[0]
        assert feat0.type == feat1.type
        assert feat0.qualifiers == feat1.qualifiers
        assert str(feat0.location) == str(feat1.location)

        #with snps
        repr_ = "SeqWithQuality(seq=Seq('GGGGATTTG', Alphabet()), features=[SeqFeature(FeatureLocation(ExactPosition(213),ExactPosition(213)), type='snv', qualifiers={'alleles': {('C', 3): {'read_groups': ['group1+454', 'group1+454', 'group1+454'], 'qualities': [44.0, 44.0, 44.0], 'libraries': ['group1', 'group1', 'group1'], 'read_names': ['seq1', 'seq4', 'seq7'], 'orientations': [True, True, True], 'samples': ['group1+454', 'group1+454', 'group1+454'], 'quality': 66.0, 'mapping_qualities': [149, 149, 149]}, ('T', 0): {'read_groups': ['group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454'], 'qualities': [44.0, 44.0, 44.0, 44.0, 44.0, 44.0], 'libraries': ['group1', 'group1', 'group1', 'group1', 'group1', 'group1'], 'read_names': ['seq2', 'seq3', 'seq5', 'seq6', 'seq8', 'seq9'], 'orientations': [True, True, True, True, True, True], 'samples': ['group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454'], 'quality': 66.0, 'mapping_qualities': [28, 28, 28, 28, 28, 28]}}, 'reference_allele': 'C'} )])\n"
        alleles = {('C', 3):
                     {'read_groups': ['group1+454', 'group1+454', 'group1+454'],
                      'qualities': [44.0, 44.0, 44.0],
                      'libraries': ['group1', 'group1', 'group1'],
                      'read_names': ['seq1', 'seq4', 'seq7'],
                      'orientations': [True, True, True],
                      'samples': ['group1+454', 'group1+454', 'group1+454'],
                      'quality': 66.0, 'mapping_qualities': [149, 149, 149]},
                   ('T', 0):
                     {'read_groups': ['group1+454', 'group1+454', 'group1+454',
                                      'group1+454', 'group1+454', 'group1+454'],
                      'qualities': [44.0, 44.0, 44.0, 44.0, 44.0, 44.0],
                      'libraries': ['group1', 'group1', 'group1', 'group1',
                                    'group1', 'group1'],
                      'read_names': ['seq2', 'seq3', 'seq5', 'seq6', 'seq8',
                                     'seq9'],
                      'orientations': [True, True, True, True, True, True],
                      'samples': ['group1+454', 'group1+454', 'group1+454',
                                  'group1+454', 'group1+454', 'group1+454'],
                      'quality': 66.0,
                      'mapping_qualities': [28, 28, 28, 28, 28, 28]}
                  }
        fcontent = repr_
        fhand = StringIO.StringIO(fcontent)
        seq0 = list(seqs_in_file(fhand, format='repr'))[0]
        alleles0 = seq0.features[0].qualifiers['alleles']
        assert alleles == alleles0

Example 8

Project: franklin
Source File: readers_test.py
View license
    @staticmethod
    def test_repr():
        'It test the repr reader'

        assert u'a (h)' == _cast_to_class("u'a (h)'")

        assert ('adios',) == _cast_to_class("('adios',)")
        assert ['arab1', 'arab2'] == _cast_to_class("['arab1', 'arab2']")
        assert ('arab1', 'arab2') == _cast_to_class("('arab1', 'arab2')")
        result = _cast_to_class("{1: 2}")
        assert {1: 2}  == result

        assert {'al': {'c': 1}, 'T': 2} ==  _cast_to_class("{'al': {'c': 1}, 'T': 2}")

        seq1 = SeqWithQuality(seq=Seq('ATCT'))
        seq2 = SeqWithQuality(seq=Seq('AAAA'))
        fcontent = repr(seq1) + '\n' + repr(seq2) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seqs = list(seqs_in_file(fhand, format='repr'))
        assert repr(seqs[0]) == repr(seq1)
        assert repr(seqs[1]) == repr(seq2)

        #with quality
        seq1 = SeqWithQuality(seq=Seq('ATCT'), qual=[10, 2, 3, 4])
        fcontent = repr(seq1) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seqs = list(seqs_in_file(fhand, format='repr'))
        assert repr(seqs[0]) == repr(seq1)

        #a seq with features
        seq1 = SeqWithQuality(seq=Seq('GAAAAGATGTG', SingleLetterAlphabet()),
                             id='seq', name='seq', description='', dbxrefs=[],
                    features=[SeqFeature(FeatureLocation(ExactPosition(478),
                                                         ExactPosition(478)),
                                         type='intron',
                                         qualifiers={'db': 'tomato'} ),
                              SeqFeature(FeatureLocation(ExactPosition(572),
                                                         ExactPosition(572)),
                                         type='intron',
                                         qualifiers={'db': 'tomato'} )],
                             annotations={}, qual=None)
        fcontent = repr(seq1) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seq0 = list(seqs_in_file(fhand, format='repr'))[0]

        assert seq0.seq == seq1.seq
        assert seq0.qual == seq1.qual
        assert seq0.description == seq1.description
        assert seq0.annotations == seq1.annotations
        feat0 = seq0.features[0]
        feat1 = seq1.features[0]
        assert feat0.type == feat1.type
        assert feat0.qualifiers == feat1.qualifiers
        assert str(feat0.location) == str(feat1.location)

        #some more qualifiers
        fhand = tempfile.NamedTemporaryFile(suffix='.repr')
        seq1 = SeqWithQuality(id='seqid', name='seqname',
                         description='seqdescription', seq=Seq('ATGAT'))
        seq1.letter_annotations["phred_quality"] = [40, 40, 38, 30, 30]
        seq1.annotations['source'] = 'ara1'

        seqfeature = SeqFeature(location=FeatureLocation(5, 8),
                                type='orthologs',
                                qualifiers={'arabidposis':['arab1', 'arab2']})
        seq1.features.append(seqfeature)

        fcontent = repr(seq1) + '\n'
        fhand = StringIO.StringIO(fcontent)
        seq0 = list(seqs_in_file(fhand, format='repr'))[0]

        assert seq0.seq == seq1.seq
        assert seq0.qual == seq1.qual
        assert seq0.description == seq1.description
        assert seq0.annotations == seq1.annotations
        feat0 = seq0.features[0]
        feat1 = seq1.features[0]
        assert feat0.type == feat1.type
        assert feat0.qualifiers == feat1.qualifiers
        assert str(feat0.location) == str(feat1.location)

        #with snps
        repr_ = "SeqWithQuality(seq=Seq('GGGGATTTG', Alphabet()), features=[SeqFeature(FeatureLocation(ExactPosition(213),ExactPosition(213)), type='snv', qualifiers={'alleles': {('C', 3): {'read_groups': ['group1+454', 'group1+454', 'group1+454'], 'qualities': [44.0, 44.0, 44.0], 'libraries': ['group1', 'group1', 'group1'], 'read_names': ['seq1', 'seq4', 'seq7'], 'orientations': [True, True, True], 'samples': ['group1+454', 'group1+454', 'group1+454'], 'quality': 66.0, 'mapping_qualities': [149, 149, 149]}, ('T', 0): {'read_groups': ['group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454'], 'qualities': [44.0, 44.0, 44.0, 44.0, 44.0, 44.0], 'libraries': ['group1', 'group1', 'group1', 'group1', 'group1', 'group1'], 'read_names': ['seq2', 'seq3', 'seq5', 'seq6', 'seq8', 'seq9'], 'orientations': [True, True, True, True, True, True], 'samples': ['group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454', 'group1+454'], 'quality': 66.0, 'mapping_qualities': [28, 28, 28, 28, 28, 28]}}, 'reference_allele': 'C'} )])\n"
        alleles = {('C', 3):
                     {'read_groups': ['group1+454', 'group1+454', 'group1+454'],
                      'qualities': [44.0, 44.0, 44.0],
                      'libraries': ['group1', 'group1', 'group1'],
                      'read_names': ['seq1', 'seq4', 'seq7'],
                      'orientations': [True, True, True],
                      'samples': ['group1+454', 'group1+454', 'group1+454'],
                      'quality': 66.0, 'mapping_qualities': [149, 149, 149]},
                   ('T', 0):
                     {'read_groups': ['group1+454', 'group1+454', 'group1+454',
                                      'group1+454', 'group1+454', 'group1+454'],
                      'qualities': [44.0, 44.0, 44.0, 44.0, 44.0, 44.0],
                      'libraries': ['group1', 'group1', 'group1', 'group1',
                                    'group1', 'group1'],
                      'read_names': ['seq2', 'seq3', 'seq5', 'seq6', 'seq8',
                                     'seq9'],
                      'orientations': [True, True, True, True, True, True],
                      'samples': ['group1+454', 'group1+454', 'group1+454',
                                  'group1+454', 'group1+454', 'group1+454'],
                      'quality': 66.0,
                      'mapping_qualities': [28, 28, 28, 28, 28, 28]}
                  }
        fcontent = repr_
        fhand = StringIO.StringIO(fcontent)
        seq0 = list(seqs_in_file(fhand, format='repr'))[0]
        alleles0 = seq0.features[0].qualifiers['alleles']
        assert alleles == alleles0

Example 9

Project: cpppo
Source File: get_attribute.py
View license
def main( argv=None ):
    """Get Attribute(s) Single/All the specified Instance or Attribute level address(es)

    """
    ap				= argparse.ArgumentParser(
        description = "An EtherNet/IP Get Attribute Single/All and Set Attribute Single client",
        formatter_class = argparse.RawDescriptionHelpFormatter,
        epilog = """\

One or more EtherNet/IP CIP Object/Instance Attributes may be read or
written.  The full format for specifying a tag and an operation is:

    @<Object>/<Instance>/<Attribute>[=<value>,<value>...]

The default Send Path is '@6/1', and the default Route Path is [{"link": 0,
"port":1}].  This should work with a device that can route requests to links
(eg. a *Logix Controller), with the Processor is slot 1 of the chassis.  If you
have a simpler device (ie. something that does not route requests, such as an AB
PowerFlex for example), then you may want to specify:

    --send-path='' --route-path=false

to eliminate the *Logix-style Unconnected Send (service 0x52) encapsulation
which is required to carry this Send/Route Path data. """ )

    ap.add_argument( '-a', '--address',
                     default=( "%s:%d" % enip.address ),
                     help="EtherNet/IP interface[:port] to connect to (default: %s:%d)" % (
                         enip.address[0], enip.address[1] ))
    ap.add_argument( '--no-print', action='store_false', dest='print',
                     help="Disable printing of summary of operations to stdout" )
    ap.add_argument( '--print', action='store_true',
                     default=True, # inconsistent default from client.py, for historical reasons
                     help="Printing a summary of operations to stdout (default: True)" )
    ap.add_argument( '-m', '--multiple', action='store_true',
                     help="Use Multiple Service Packet request targeting ~500 bytes (default: False)" )
    ap.add_argument( '-d', '--depth',
                     default=0,
                     help="Pipelining depth" )
    ap.add_argument( '-t', '--timeout',
                     default=5.0,
                     help="EtherNet/IP timeout (default: 5s)" )
    ap.add_argument( '-v', '--verbose', action="count",
                     default=0, 
                     help="Display logging information." )
    ap.add_argument( '-l', '--log',
                     help="Log file, if desired" )
    ap.add_argument( '--route-path',
                     default=None,
                     help="Route Path, in JSON (default: %r); 0/false to specify no/empty route_path" % (
                         str( json.dumps( client.connector.route_path_default ))))
    ap.add_argument( '--send-path',
                     default=None,
                     help="Send Path to UCMM (default: @6/1); Specify an empty string '' for no Send Path" )
    ap.add_argument( '-S', '--simple', action='store_true',
                     default=False,
                     help="Access a simple (non-routing) EtherNet/IP CIP device (eg. MicroLogix)")
    ap.add_argument( '-P', '--profile', action='store_true',
                     help="Activate profiling (default: False)" )
    ap.add_argument( 'tags', nargs="+",
                     help="Class/Instance[/Attribute] to get (- to read from stdin), eg: @2/1 @2/1/1" )

    args			= ap.parse_args( argv )

    # Set up logging level (-v...) and --log <file>
    levelmap 			= {
        0: logging.WARNING,
        1: logging.NORMAL,
        2: logging.DETAIL,
        3: logging.INFO,
        4: logging.DEBUG,
        }
    cpppo.log_cfg['level']	= ( levelmap[args.verbose] 
                                    if args.verbose in levelmap
                                    else logging.DEBUG )
    if args.log:
        cpppo.log_cfg['filename'] = args.log

    logging.basicConfig( **cpppo.log_cfg )

    addr			= args.address.split(':')
    assert 1 <= len( addr ) <= 2, "Invalid --address [<interface>]:[<port>}: %s" % args.address
    addr			= ( str( addr[0] ) if addr[0] else enip.address[0],
                                    int( addr[1] ) if len( addr ) > 1 and addr[1] else enip.address[1] )
    timeout			= float( args.timeout )
    depth			= int( args.depth )
    multiple			= 500 if args.multiple else 0
    route_path			= json.loads( args.route_path ) if args.route_path \
                                  else [] if args.simple else None # may be None/0/False/[]
    send_path			= args.send_path                if args.send_path \
                                  else '' if args.simple else None # uses '@2/1/1' by default

    if '-' in args.tags:
        # Collect tags from sys.stdin 'til EOF, at position of '-' in argument list
        minus			= args.tags.index( '-' )
        tags			= itertools.chain( args.tags[:minus], sys.stdin, args.tags[minus+1:] )
    else:
        tags			= args.tags

    profiler			= None
    if args.profile:
        import cProfile as profile
        import pstats
        import StringIO
        profiler		= profile.Profile()

    failures			= 0
    with client.connector( host=addr[0], port=addr[1], timeout=timeout, profiler=profiler ) as connection:
        idx			= -1
        start			= cpppo.timer()
        operations		= attribute_operations( tags, route_path=route_path, send_path=send_path )
        for idx,dsc,op,rpy,sts,val in connection.pipeline(
                operations=operations, depth=depth, multiple=multiple, timeout=timeout ):
            if args.print:
                print( "%s: %3d: %s == %s" % ( time.ctime(), idx, dsc, val ))
            failures	       += 1 if sts else 0
        elapsed			= cpppo.timer() - start
        log.normal( "%3d requests in %7.3fs at pipeline depth %2s; %7.3f TPS" % (
            idx+1, elapsed, args.depth, (idx+1) / elapsed ))

    if profiler:
        s			= StringIO.StringIO()
        ps			= pstats.Stats( profiler, stream=s )
        for sortby in [ 'cumulative', 'time' ]:
            ps.sort_stats( sortby )
            ps.print_stats( 25 )
        print( s.getvalue() )

    return 1 if failures else 0

Example 10

Project: vumi-go
Source File: test_go_setup_env.py
View license
    def setUp(self):
        self.vumi_helper = self.add_helper(DjangoVumiApiHelper())

        self.command = go_setup_env.Command()
        self.command.stdout = StringIO()
        self.command.stderr = StringIO()
        # do whatever setup command.handle() does manually
        self.command.setup_backend(self.vumi_helper.mk_config({}))
        self.add_cleanup(self.command.cleanup)
        self.tagpool = self.command.tagpool
        self.command.file_name_template = 'go_%(file_name)s.%(suffix)s'
        self.command.dest_dir = 'setup_env'
        self.command.config = {
            'redis_manager': {'key_prefix': 'test'},
            'riak_manager': {'bucket_prefix': 'test.'}
        }

        self.command.contact_group_info = []
        self.command.conversation_info = []
        self.command.router_info = []
        self.command.transport_names = []
        self.command.router_names = []
        self.command.application_names = []

        self.tagpool_file = tmp_yaml_file([
            'pools:',
            '  pool1:',
            '    tags: "[\'default%d\' % i for i in range(10)]"',
            '    metadata:',
            '      display_name: "Pool 1"',
            '  pool2:',
            '    tags: ["a", "b", "c"]',
            '    metadata:',
            '      display_name: "Pool 2"',
        ])

        self.workers_file = tmp_yaml_file([
            'transports:',
            '  sms_transport:',
            '    class: vumi.transports.telnet.TelnetServerTransport',
            '    config:',
            '      telnet_port: 8080',
            '  ussd_transport:',
            '    class: vumi.transports.telnet.TelnetServerTransport',
            '    config:',
            '      telnet_port: 8081',
            '',
            'routers:',
            '  keyword:',
            '    class: go.routers.keyword.vumi_app.KeywordRouter',
            '    config:',
            '      redis_manager: {}',
            '      riak_manager: {}',
            '',
            'applications:',
            '  bulk_message:',
            '    class: go.apps.bulk_message.vumi_app.BulkMessageApplication',
            '    config:',
            '      redis_manager: {}',
            '      riak_manager: {}',
        ])

        self.account_1_file = tmp_yaml_file([
            'account:',
            '  email: "[email protected]"',
            '  password: foo',
            '  first_name: First',
            '  last_name: Last',
            '  applications:',
            '    - go.apps.surveys',
            '    - go.apps.bulk_message',
            '  tagpools:',
            '    - ["pool1", 10]',
            '    - ["pool2", 15]',
            '',
            'channels:',
            '  - "pool1:default0"',
            '  - "pool1:default1"',
            '',
            'conversations:',
            '  - key: "conv1"',
            '    conversation_type: survey',
            '    name: foo',
            '    config:',
            '      foo: bar',
            '  - key: conv2',
            '    conversation_type: wikipedia',
            '    name: Wikipedia',
            '    config: {}',
            '',
            'routers:',
            '  - key: "router1"',
            '    router_type: keyword',
            '    name: foo',
            '    config:',
            '      keyword_endpoint_mapping:',
            '        foo: keyword_foo',
            '',
            'routing_entries:',
            '  - ["conv1", "default", "pool1:default0", "default"]',
            '  - ["pool1:default0", "default", "conv1", "default"]',
            '  - ["router1:INBOUND", "default", "pool1:default1", "default"]',
            '  - ["pool1:default1", "default", "router1:INBOUND", "default"]',
            '  - ["conv2", "default", "router1:OUTBOUND", "default"]',
            '  - ["router1:OUTBOUND", "default", "conv2", "default"]',
            '',
            'contact_groups:',
            '  - key: group1',
            '    name: group1',
            '    contacts_csv: contacts.csv',
        ])

        self.account_2_file = tmp_yaml_file([
            'account:',
            '  email: "[email protected]"',
            '  password: bar',
            '  applications:',
            '    - go.apps.bulk_message',
            '    - go.apps.jsbox',
            '  tagpools:',
            '    - ["pool1", null]',
            '    - ["pool2", null]',
            '',
            'conversations:',
            '  - key: "conversation-key-2"',
            '    conversation_type: bulk_message',
            '    name: oof',
            '    config:',
            '      oof: rab',
            '',
            'contact_groups:',
            '  - key: group2',
            '    name: group2',
            '    contacts_csv: contacts.csv',
        ])

Example 11

Project: auto-sklearn
Source File: plot_metafeatures.py
View license
def plot_metafeatures(metafeatures_plot_dir, metafeatures, metafeatures_times,
                      runs, method='pca', seed=1, depth=1, distance='l2'):
    """Project datasets in a 2d space and plot them.

    arguments:
      * metafeatures_plot_dir: a directory to save the generated plots
      * metafeatures: a pandas Dataframe from the MetaBase
      * runs: a dictionary of runs from the MetaBase
      * method: either pca or t-sne
      * seed: only used for t-sne
      * depth: if 1, a one-step look-ahead is performed
    """
    if type(metafeatures) != pd.DataFrame:
        raise ValueError("Argument metafeatures must be of type pd.Dataframe "
                         "but is %s" % str(type(metafeatures)))

    ############################################################################
    # Write out the datasets and their size as a TEX table
    # TODO put this in an own function
    dataset_tex = StringIO.StringIO()
    dataset_tex.write('\\begin{tabular}{lrrr}\n')
    dataset_tex.write('\\textbf{Dataset name} & '
                      '\\textbf{\#features} & '
                      '\\textbf{\#patterns} & '
                      '\\textbf{\#classes} \\\\\n')

    num_features = []
    num_instances = []
    num_classes = []

    for dataset in sorted(metafeatures.index):
        dataset_tex.write('%s & %d & %d & %d \\\\\n' % (
                        dataset.replace('larochelle_etal_2007_', '').replace(
                            '_', '-'),
                        metafeatures.loc[dataset]['number_of_features'],
                        metafeatures.loc[dataset]['number_of_instances'],
                        metafeatures.loc[dataset]['number_of_classes']))
        num_features.append(metafeatures.loc[dataset]['number_of_features'])
        num_instances.append(metafeatures.loc[dataset]['number_of_instances'])
        num_classes.append(metafeatures.loc[dataset]['number_of_classes'])

    dataset_tex.write('Minimum & %.1f & %.1f & %.1f \\\\\n' %
        (np.min(num_features), np.min(num_instances), np.min(num_classes)))
    dataset_tex.write('Maximum & %.1f & %.1f & %.1f \\\\\n' %
        (np.max(num_features), np.max(num_instances), np.max(num_classes)))
    dataset_tex.write('Mean & %.1f & %.1f & %.1f \\\\\n' %
        (np.mean(num_features), np.mean(num_instances), np.mean(num_classes)))

    dataset_tex.write('10\\%% quantile & %.1f & %.1f & %.1f \\\\\n' % (
        np.percentile(num_features, 10), np.percentile(num_instances, 10),
        np.percentile(num_classes, 10)))
    dataset_tex.write('90\\%% quantile & %.1f & %.1f & %.1f \\\\\n' % (
        np.percentile(num_features, 90), np.percentile(num_instances, 90),
        np.percentile(num_classes, 90)))
    dataset_tex.write('median & %.1f & %.1f & %.1f \\\\\n' % (
        np.percentile(num_features, 50), np.percentile(num_instances, 50),
        np.percentile(num_classes, 50)))
    dataset_tex.write('\\end{tabular}')
    dataset_tex.seek(0)

    dataset_tex_output = os.path.join(metafeatures_plot_dir, 'datasets.tex')
    with open(dataset_tex_output, 'w') as fh:
        fh.write(dataset_tex.getvalue())

    ############################################################################
    # Write out a list of metafeatures, each with the min/max/mean
    # calculation time and the min/max/mean value
    metafeatures_tex = StringIO.StringIO()
    metafeatures_tex.write('\\begin{tabular}{lrrrrrr}\n')
    metafeatures_tex.write('\\textbf{Metafeature} & '
                      '\\textbf{Minimum} & '
                      '\\textbf{Mean} & '
                      '\\textbf{Maximum} &'
                      '\\textbf{Minimum time} &'
                      '\\textbf{Mean time} &'
                      '\\textbf{Maximum time} '
                      '\\\\\n')

    for mf_name in sorted(metafeatures.columns):
        metafeatures_tex.write('%s & %.2f & %.2f & %.2f & %.2f & %.2f & %.2f \\\\\n'
                               % (mf_name.replace('_', '-'),
                                  metafeatures.loc[:,mf_name].min(),
                                  metafeatures.loc[:,mf_name].mean(),
                                  metafeatures.loc[:,mf_name].max(),
                                  metafeature_times.loc[:, mf_name].min(),
                                  metafeature_times.loc[:, mf_name].mean(),
                                  metafeature_times.loc[:, mf_name].max()))

    metafeatures_tex.write('\\end{tabular}')
    metafeatures_tex.seek(0)

    metafeatures_tex_output = os.path.join(metafeatures_plot_dir, 'metafeatures.tex')
    with open(metafeatures_tex_output, 'w') as fh:
        fh.write(metafeatures_tex.getvalue())

    # Without this scaling the transformation for visualization purposes is
    # useless
    metafeatures = metafeatures.copy()
    X_min = np.nanmin(metafeatures, axis=0)
    X_max = np.nanmax(metafeatures, axis=0)
    metafeatures = (metafeatures - X_min) / (X_max - X_min)

    # PCA
    if method == 'pca':
        pca = PCA(2)
        transformation = pca.fit_transform(metafeatures.values)

    elif method == 't-sne':
        if distance == 'l2':
            distance_matrix = sklearn.metrics.pairwise.pairwise_distances(
                metafeatures.values, metric='l2')
        elif distance == 'l1':
            distance_matrix = sklearn.metrics.pairwise.pairwise_distances(
                metafeatures.values, metric='l1')
        elif distance == 'runs':
            names_to_indices = dict()
            for metafeature in metafeatures.index:
                idx = len(names_to_indices)
                names_to_indices[metafeature] = idx

            X, Y = pyMetaLearn.metalearning.create_datasets\
                .create_predict_spearman_rank(metafeatures, runs,
                                              'combination')
            # Make a metric matrix out of Y
            distance_matrix = np.zeros((metafeatures.shape[0],
                                        metafeatures.shape[0]), dtype=np.float64)

            for idx in Y.index:
                dataset_names = idx.split("_")
                d1 = names_to_indices[dataset_names[0]]
                d2 = names_to_indices[dataset_names[1]]
                distance_matrix[d1][d2] = Y.loc[idx]
                distance_matrix[d2][d1] = Y.loc[idx]

        else:
            raise NotImplementedError()

        # For whatever reason, tsne doesn't accept l1 metric
        tsne = TSNE(random_state=seed, perplexity=50, verbose=1)
        transformation = tsne.fit_transform(distance_matrix)

    # Transform the transformation back to range [0, 1] to ease plotting
    transformation_min = np.nanmin(transformation, axis=0)
    transformation_max = np.nanmax(transformation, axis=0)
    transformation = (transformation - transformation_min) / \
                     (transformation_max - transformation_min)
    print(transformation_min, transformation_max)

    #for i, dataset in enumerate(directory_content):
    #    print dataset, meta_feature_array[i]
    fig = plt.figure(dpi=600, figsize=(12, 12))
    ax = plt.subplot(111)

    # The dataset names must be aligned at the borders of the plot in a way
    # the arrows don't cross each other. First, define the different slots
    # where the labels will be positioned and then figure out the optimal
    # order of the labels
    slots = []
    # 25 datasets on the top y-axis
    slots.extend([(-0.1 + 0.05 * i, 1.1) for i in range(25)])
    # 24 datasets on the right x-axis
    slots.extend([(1.1, 1.05 - 0.05 * i) for i in range(24)])
    # 25 datasets on the bottom y-axis
    slots.extend([(-0.1 + 0.05 * i, -0.1) for i in range(25)])
    # 24 datasets on the left x-axis
    slots.extend([(-0.1, 1.05 - 0.05 * i) for i in range(24)])

    # Align the labels on the outer axis
    labels_top = []
    labels_left = []
    labels_right = []
    labels_bottom = []

    for values in zip(metafeatures.index,
                      transformation[:, 0], transformation[:, 1]):
        label, x, y = values
        # Although all plot area goes up to 1.1, 1.1, the range of all the
        # points lies inside [0,1]
        if x >= y and x < 1.0 - y:
            labels_bottom.append((x, label))
        elif x >= y and x >= 1.0 - y:
            labels_right.append((y, label))
        elif y > x and x <= 1.0 -y:
             labels_left.append((y, label))
        else:
            labels_top.append((x, label))

    # Sort the labels according to their alignment
    labels_bottom.sort()
    labels_left.sort()
    labels_left.reverse()
    labels_right.sort()
    labels_right.reverse()
    labels_top.sort()

    # Build an index label -> x, y
    points = {}
    for values in zip(metafeatures.index,
                      transformation[:, 0], transformation[:, 1]):
        label, x, y = values
        points[label] = (x, y)

    # Find out the final positions...
    positions_top = {}
    positions_left = {}
    positions_right = {}
    positions_bottom = {}

    # Find the actual positions
    for i, values in enumerate(labels_bottom):
        y, label = values
        margin = 1.2 / len(labels_bottom)
        positions_bottom[label] = (-0.05 + i * margin, -0.1,)
    for i, values in enumerate(labels_left):
        x, label = values
        margin = 1.2 / len(labels_left)
        positions_left[label] = (-0.1, 1.1 - i * margin)
    for i, values in enumerate(labels_top):
        y, label = values
        margin = 1.2 / len(labels_top)
        positions_top[label] = (-0.05 + i * margin, 1.1)
    for i, values in enumerate(labels_right):
        y, label = values
        margin = 1.2 / len(labels_right)
        positions_right[label] = (1.1, 1.05 - i * margin)

    # Do greedy resorting if it decreases the number of intersections...
    def resort(label_positions, marker_positions, maxdepth=1):
        # TODO: are the inputs dicts or lists
        # TODO: two-step look-ahead
        def intersect(start1, end1, start2, end2):
            # Compute if there is an intersection, for the algorithm see
            # Computer Graphics by F.S.Hill

            # If one vector is just a point, it cannot intersect with a line...
            for v in [start1, start2, end1, end2]:
                if not np.isfinite(v).all():
                    return False     # Obviously there is no intersection

            def perpendicular(d):
                return np.array((-d[1], d[0]))

            d1 = end1 - start1      # denoted b
            d2 = end2 - start2      # denoted d
            d2_1 = start2 - start1  # denoted c
            d1_perp = perpendicular(d1)   # denoted by b_perp
            d2_perp = perpendicular(d2)   # denoted by d_perp

            t = np.dot(d2_1, d2_perp) / np.dot(d1, d2_perp)
            u = - np.dot(d2_1, d1_perp) / np.dot(d2, d1_perp)

            if 0 <= t <= 1 and 0 <= u <= 1:
                return True    # There is an intersection
            else:
                return False     # There is no intersection

        def number_of_intersections(label_positions, marker_positions):
            num = 0
            for key1, key2 in itertools.permutations(label_positions, r=2):
                s1 = np.array(label_positions[key1])
                e1 = np.array(marker_positions[key1])
                s2 = np.array(label_positions[key2])
                e2 = np.array(marker_positions[key2])
                if intersect(s1, e1, s2, e2):
                    num += 1
            return num

        # test if swapping two lines would decrease the number of intersections
        # TODO: if this was done with a datastructure different than dicts,
        # it could be much faster, because there is a lot of redundant
        # computing performed in the second iteration
        def swap(label_positions, marker_positions, depth=0,
                 maxdepth=maxdepth, best_found=sys.maxint):
            if len(label_positions) <= 1:
                return

            two_step_look_ahead = False
            while True:
                improvement = False
                for key1, key2 in itertools.combinations(label_positions, r=2):
                    before = number_of_intersections(label_positions, marker_positions)
                    # swap:
                    tmp = label_positions[key1]
                    label_positions[key1] = label_positions[key2]
                    label_positions[key2] = tmp
                    if depth < maxdepth and two_step_look_ahead:
                        swap(label_positions, marker_positions,
                             depth=depth+1, best_found=before)

                    after = number_of_intersections(label_positions, marker_positions)

                    if best_found > after and before > after:
                        improvement = True
                        print(before, after)
                        print("Depth %d: Swapped %s with %s" %
                              (depth, key1, key2))
                    else:       # swap back...
                        tmp = label_positions[key1]
                        label_positions[key1] = label_positions[key2]
                        label_positions[key2] = tmp

                    if after == 0:
                        break

                # If it is not yet sorted perfectly, do another pass with
                # two-step lookahead
                if before == 0:
                    print("Sorted perfectly...")
                    break
                print(depth, two_step_look_ahead)
                if two_step_look_ahead:
                    break
                if maxdepth == depth:
                    print("Reached maximum recursion depth...")
                    break
                if not improvement and depth < maxdepth:
                    print("Still %d errors, trying two-step lookahead" % before)
                    two_step_look_ahead = True

        swap(label_positions, marker_positions, maxdepth=maxdepth)

    resort(positions_bottom, points, maxdepth=depth)
    resort(positions_left, points, maxdepth=depth)
    resort(positions_right, points, maxdepth=depth)
    resort(positions_top, points, maxdepth=depth)

    # Helper function
    def plot(x, y, label_x, label_y, label, ha, va, relpos, rotation=0):
        ax.scatter(x, y, marker='o', label=label, s=80, linewidths=0.1,
                   color='blue', edgecolor='black')

        label = label.replace('larochelle_etal_2007_', '')

        x = ax.annotate(label, xy=(x, y), xytext=(label_x, label_y),
                    ha=ha, va=va, rotation=rotation,
                    bbox=dict(boxstyle='round', fc='gray', alpha=0.5),
                    arrowprops=dict(arrowstyle='->', color='black',
                                    relpos=relpos))

    # Do the plotting
    for i, key in enumerate(positions_bottom):
        x, y = positions_bottom[key]
        plot(points[key][0], points[key][1], x, y,
             key, ha='right', va='top', rotation=45, relpos=(1, 1))
    for i, key in enumerate(positions_left):
        x, y = positions_left[key]
        plot(points[key][0], points[key][1], x, y, key,
             ha='right', va='top', rotation=45, relpos=(1, 1))
    for i, key in enumerate(positions_top):
        x, y = positions_top[key]
        plot(points[key][0], points[key][1], x, y, key,
             ha='left', va='bottom', rotation=45, relpos=(0, 0))
    for i, key in enumerate(positions_right):
        x, y = positions_right[key]
        plot(points[key][0], points[key][1], x, y, key,
             ha='left', va='bottom', rotation=45, relpos=(0, 0))

    # Resize everything
    box = ax.get_position()
    remove = 0.05 * box.width
    ax.set_position([box.x0 + remove, box.y0 + remove,
                     box.width - remove*2, box.height - remove*2])

    locs_x = ax.get_xticks()
    locs_y = ax.get_yticks()
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_xlim((-0.1, 1.1))
    ax.set_ylim((-0.1, 1.1))
    plt.savefig(os.path.join(metafeatures_plot_dir, "pca.png"))
    plt.savefig(os.path.join(metafeatures_plot_dir, "pca.pdf"))
    plt.clf()

Example 12

Project: meshtool
Source File: make_atlases.py
View license
def packImages(mesh, img2texs, unique_images, image_scales):
    #if there aren't at least two images left, nothing to do
    if len(unique_images) < 2:
        return
    
    #okay, now we can start packing!
    rp = RectPack(MAX_IMAGE_DIMENSION, MAX_IMAGE_DIMENSION)
    for path, pilimg in unique_images.iteritems():
        width, height = pilimg.size
        rp.addRectangle(path, width, height)
    success = rp.pack()

    if not success:
        if len(rp.rejects) == len(unique_images):
            #this means that nothing could be packed into the max size
            # if not a single image can be packed into the max size
            # then there's no point in continuing
            return
                
        group1 = dict(( (path, pilimg) for path, pilimg in unique_images.iteritems() if path in rp.rejects ))
        group2 = dict(( (path, pilimg) for path, pilimg in unique_images.iteritems() if path not in rp.rejects ))
        
        return combinePacks(packImages(mesh, img2texs, group1, image_scales),
                    packImages(mesh, img2texs, group2, image_scales))
    
    width = rp.width
    height = rp.height
    
    print "actually making atlas of size %dx%d with %d subimages referenced by %d texcoords" % \
        (width, height, len(unique_images), sum([len(img2texs[imgpath]) for imgpath in unique_images]))
    atlasimg = Image.new('RGBA', (width, height), (0,0,0,255))
    
    to_del = {}
    for path, pilimg in unique_images.iteritems():
        x,y,w,h = rp.getPlacement(path)
        atlasimg.paste(pilimg, (x,y,x+w,y+h))
        
        x,y,w,h,width,height = (float(i) for i in (x,y,w,h,width,height))
        
        for texset in img2texs[path]:
            geom = mesh.geometries[texset.geom_id]
            prim = geom.primitives[texset.prim_index]
            texarray = numpy.copy(prim.texcoordset[texset.texcoordset_index])
            tile_x, tile_y = (float(i) for i in image_scales[path])
            
            #this shrinks the texcoords to 0,1 range for a tiled image
            if tile_x > 1.0:
                texarray[:,0] = texarray[:,0] / tile_x
            if tile_y > 1.0:
                texarray[:,1] = texarray[:,1] / tile_y
            
            #this computes the coordinates of the lowest and highest texel
            # if the texcoords go outside that range, rescale so they are inside
            # suggestion by nvidia texture atlasing white paper
            minx, maxx = numpy.min(texarray[:,0]), numpy.max(texarray[:,0])
            miny, maxy = numpy.min(texarray[:,1]), numpy.max(texarray[:,1])
            lowest_x = 0.5 / w
            lowest_y = 0.5 / h
            highest_x = 1.0 - lowest_x
            highest_y = 1.0 - lowest_y
            if minx < lowest_x or maxx > highest_x:
                texarray[:,0] = texarray[:,0] * (highest_x - lowest_x) + lowest_x
            if miny < lowest_y or maxy > highest_y:
                texarray[:,1] = texarray[:,1] * (highest_y - lowest_y) + lowest_y

            #this rescales the texcoords to map to the new atlas location
            texarray[:,0] = texarray[:,0] * (w / width) + (x / (width-1))
            texarray[:,1] = texarray[:,1] * (h / height) + (1.0 - (y+h)/height)
            
            oldsources = prim.getInputList().getList()
            newsources = collada.source.InputList()
            for (offset, semantic, source, setid) in oldsources:
                if semantic == 'TEXCOORD' and (setid is None or int(setid) == texset.texcoordset_index):
                    orig_source = source
                    i=0
                    while source[1:] in geom.sourceById:
                        source = orig_source + '-atlas-' + str(i)
                        i += 1
                    new_tex_src = collada.source.FloatSource(source[1:], texarray, ('S', 'T'))
                    geom.sourceById[source[1:]] = new_tex_src
                newsources.addInput(offset, semantic, source, setid)
            
            if geom not in to_del:
                to_del[geom] = []
            to_del[geom].append(texset.prim_index)
            
            if type(prim) is collada.triangleset.TriangleSet:
                prim.index.shape = -1
                newprim = geom.createTriangleSet(prim.index, newsources, prim.material)
            elif type(prim) is collada.polylist.Polylist:
                prim.index.shape = -1
                prim.vcounts.shape = -1
                newprim = geom.createPolylist(prim.index, prim.vcounts, newsources, prim.material)
            elif type(prim) is collada.polygons.Polygons:
                prim.index.shape = -1
                newprim = geom.createPolygons(prim.index, newsources, prim.material)
            elif type(prim) is collada.lineset.LineSet:
                prim.index.shape = -1
                newprim = geom.createLineSet(prim.index, newsources, prim.material)
            else:
                raise Exception("Unknown primitive type")
            
            geom.primitives.append(newprim)
        
    imgs_deleted = [cimg for cimg in mesh.images if cimg.path in unique_images]
    mesh.images = [cimg for cimg in mesh.images if cimg.path not in unique_images]
    
    baseimgid = imgs_deleted[0].id + '-atlas'
    baseimgpath = './atlas'
    newimgid = baseimgid
    newimgpath = baseimgpath
    ct = 0
    while newimgid in mesh.images or newimgpath + '.png' in [cimg.path for cimg in mesh.images]:
        newimgid = baseimgid + '-' + str(ct)
        newimgpath = baseimgpath + '-' + str(ct)
        ct += 1

    newimgpath = newimgpath + '.png'
    newcimage = collada.material.CImage(newimgid, newimgpath, mesh)
    
    strbuf = StringIO()
    atlasimg.save(strbuf, 'PNG', optimize=True)
    newcimage._data = strbuf.getvalue()
    mesh.images.append(newcimage)
    
    for effect in mesh.effects:
        for param in effect.params:
            if type(param) is collada.material.Surface:
                if param.image in imgs_deleted:
                    param.image = newcimage
                    
    return to_del

Example 13

Project: meshtool
Source File: load_obj.py
View license
def loadOBJ(data, aux_file_loader=None, validate_output=False):
    """Loads an OBJ file
    
    :param data: A binary data string containing the OBJ file
    :param aux_file_loader: Should be a callable function that takes one parameter.
                            The parameter will be a string containing an auxiliary
                            file that needs to be found, in this case usually a .mtl
                            file or a texture file.
    
    :returns: An instance of :class:`collada.Collada` or None if could not be loaded
    """
    
    mesh = collada.Collada(validate_output=validate_output)
    namer = NameUniqifier()
    material_map = {}
    cimages = []
    materialNamer = NameUniqifier()
    
    vertices = []
    normals = []
    texcoords = []
    
    groups = []
    group = ObjGroup(namer.name("default"))
    geometry_name = namer.name("convertedobjgeometry")
    
    file_like = StringIO(to_unicode(data))
    for line in file_like:
        line = line.strip()
        
        # ignore blank lines and comments
        if len(line) == 0 or line.startswith('#'):
            continue
        
        # split off the first non-whitespace token and ignore the line if there isn't > 1 token
        splitup = line.split(None, 1)
        if len(splitup) != 2:
            continue
        command, line = splitup
        
        if command == 'v':
            line_tokens = line.split()
            vertices.extend(line_tokens[:3])
            
        elif command == 'vn':
            line_tokens = line.split()
            normals.extend(line_tokens[:3])
           
        elif command == 'vt':
            line_tokens = line.split()
            texcoords.extend(line_tokens[:2])
            
        # TODO: other vertex data statements
        # vp
        # cstype
        # deg
        # bmat
        # step
            
        elif command == 'f':
            faces = line.split()
            
            if group.face_mode == FACEMODE.UNKNOWN:
                group.face_mode = detectFaceStyle(faces[0])
                if group.face_mode is None:
                    sys.stderr.write("Error: could not detect face type for line '%s'" % line)
                    return
            
            group.face_lengths.append(len(faces))
            
            # Don't decode the faces here because the / separators have to be parsed out
            # and this is very slow to do one at a time. Instead, just append to a list
            # which is much faster than appending to a string, and it will get joined and
            # parsed later
            group.face_indices.append(line)
        
        elif command == 'l':
            faces = line.split()
            
            if group.face_mode == FACEMODE.UNKNOWN:
                group.face_mode = detectFaceStyle(faces[0])
                if group.face_mode is None:
                    sys.stderr.write("Error: could not detect face type for line '%s'" % line)
                    return
            
            # COLLADA defines lines as a pair of points, so the index values "1 2 3 4" would
            # refer to *two* lines, one between 1 and 2 and one between 3 and 4. OBJ defines
            # lines as continous, so it would be three lines: 1-2, 2-3, 3-4. This duplicates
            # the points to get pairs for COLLADA. This is not very efficient, but not sure
            # of a faster way to do this and I've never seen any files with a huge number of
            # lines in it anyway.
            line = faces[0] + " " + faces[1]
            prev = faces[1]
            for cur in faces[2:]:
                line += " " + prev + " " + cur
                prev = cur
            group.line_indices.append(line)
        
        elif command == 'p':
            faces = line.split()
            
            if group.face_mode == FACEMODE.UNKNOWN:
                group.face_mode = detectFaceStyle(faces[0])
                if group.face_mode is None:
                    sys.stderr.write("Error: could not detect face type for line '%s'" % line)
                    return
                
            # COLLADA does not have points, so this converts a point to a line with two
            # identical endpoints
            line = " ".join(f + " " + f for f in faces)
            group.line_indices.append(line)
        
        # TODO: other elements
        # curv
        # curv2
        # surf
        
        elif command == 'g':
            if group.empty():
                # first group without any previous data, so just set name
                group.name = namer.name(line)
                continue
            
            # end of previous group and start of new group
            groups.append(group)
            group = ObjGroup(namer.name(line))
        
        elif command == 's':
            # there is no way to map shading groups into collada
            continue
        
        elif command == 'o':
            geometry_name = namer.name(line)
        
        # TODO: grouping info
        # mg
        
        # TODO: Free-form curve/surface body statements
        # parm
        # trim
        # hole
        # scrv
        # sp
        # end
        # con
        
        elif command == 'mtllib':
            mtl_file = None
            if aux_file_loader is not None:
                mtl_file = aux_file_loader(line)
            if mtl_file is not None:
                material_data = loadMaterialLib(mtl_file, namer=materialNamer, aux_file_loader=aux_file_loader)
                material_map.update(material_data['material_map'])
                cimages.extend(material_data['images'])
            
        elif command == 'usemtl':
            group.material = slugify(line)
        
        # TODO: display and render attributes
        # bevel
        # c_interp
        # d_interp
        # lod
        # shadow_obj
        # trace_obj
        # ctech
        # stech
        
        else:
            print '  MISSING LINE: %s %s' % (command, line)
    
    # done, append last group
    if not group.empty():
        groups.append(group)
    
    for material in material_map.values():
        mesh.effects.append(material.effect)
        mesh.materials.append(material)
    for cimg in cimages:
        mesh.images.append(cimg)
    
    vertices = numpy.array(vertices, dtype=numpy.float32).reshape(-1, 3)
    normals = numpy.array(normals, dtype=numpy.float32).reshape(-1, 3)
    texcoords = numpy.array(texcoords, dtype=numpy.float32).reshape(-1, 2)
    
    sources = []
    # all modes have vertex source
    sources.append(collada.source.FloatSource("obj-vertex-source", vertices, ('X', 'Y', 'Z')))
    if len(normals) > 0:
        sources.append(collada.source.FloatSource("obj-normal-source", normals, ('X', 'Y', 'Z')))
    if len(texcoords) > 0:
        sources.append(collada.source.FloatSource("obj-uv-source", texcoords, ('S', 'T')))
    
    geom = collada.geometry.Geometry(mesh, geometry_name, geometry_name, sources)
    
    materials_mapped = set()
    for group in groups:
        input_list = collada.source.InputList()
        input_list.addInput(0, 'VERTEX', "#obj-vertex-source")
        if group.face_mode == FACEMODE.VN:
            input_list.addInput(1, 'NORMAL', '#obj-normal-source')
        elif group.face_mode == FACEMODE.VT:
            input_list.addInput(1, 'TEXCOORD', '#obj-uv-source')
        elif group.face_mode == FACEMODE.VTN:
            input_list.addInput(1, 'TEXCOORD', '#obj-uv-source')
            input_list.addInput(2, 'NORMAL', '#obj-normal-source')
        
        if len(group.face_lengths) > 0:
            face_lengths = numpy.array(group.face_lengths, dtype=numpy.int32)
    
            # First, join the individual face lines together, separated by spaces. Then,        
            # just replace 1/2/3 and 1//3 with "1 2 3" and "1  3", as numpy.fromstring can
            # handle any whitespace it's given, similar to python's split(). Concatenating
            # together this way is much faster than parsing the numbers in python - let
            # numpy do it. Note that sep=" " is actually misleading - it handles tabs and
            # other whitespace also
            group.face_indices = (" ".join(group.face_indices)).replace("/", " ")
            face_indices = numpy.fromstring(group.face_indices, dtype=numpy.int32, sep=" ")
            
            # obj indices start at 1, while collada start at 0
            face_indices -= 1
            
            polylist = geom.createPolylist(face_indices, face_lengths, input_list, group.material or namer.name("nullmaterial"))
            geom.primitives.append(polylist)
            
        if len(group.line_indices) > 0:
            group.line_indices = (" ".join(group.line_indices)).replace("/", " ")
            line_indices = numpy.fromstring(group.line_indices, dtype=numpy.int32, sep=" ")
            line_indices -= 1
            lineset = geom.createLineSet(line_indices, input_list, group.material or namer.name("nullmaterial"))
            geom.primitives.append(lineset)
        
        if group.material in material_map:
            materials_mapped.add(group.material)
    
    mesh.geometries.append(geom)
    
    matnodes = []
    for matref in materials_mapped:
        matnode = collada.scene.MaterialNode(matref, material_map[matref], inputs=[('TEX0', 'TEXCOORD', '0')])
        matnodes.append(matnode)
    geomnode = collada.scene.GeometryNode(geom, matnodes)
    node = collada.scene.Node(namer.name("node"), children=[geomnode])
    myscene = collada.scene.Scene(namer.name("scene"), [node])
    mesh.scenes.append(myscene)
    mesh.scene = myscene
    
    return mesh

Example 14

Project: pyjs
Source File: repl.py
View license
    def __init__(self, compiler,
                 module_name, module_file_name, src, mod, output,
                 dynamic=0, findFile=None, **kw):

        self.compiler = compiler
        self.ast = compiler.ast
        self.js_module_name = self.jsname("variable", module_name)
        if module_name:
            self.module_prefix = "$module."
        else:
            self.module_prefix = ""
        self.module_name = module_name
        src = src.replace("\r\n", "\n")
        src = src.replace("\n\r", "\n")
        src = src.replace("\r",   "\n")
        self.src = src.split("\n")

        self.output = output
        self.dynamic = dynamic
        self.findFile = findFile

        self.set_compile_options(kw)

        self.future_division = False

        self.imported_modules = []
        self.imported_js = []
        self.is_class_definition = False
        self.local_prefix = None
        self.track_lines = {}
        self.stacksize_depth = 0
        self.option_stack = []
        self.lookup_stack = [{}]
        self.indent_level = 0
        self.__unique_ids__ = {}
        self.try_depth = -1
        self.is_generator = False
        self.generator_states = []
        self.state_max_depth = len(self.generator_states)
        self.constant_int = {}
        self.constant_long = {}
        self.top_level = True
        translator.PYJSLIB_BUILTIN_MAPPING['__file__'] = "'%s'" % module_file_name
        self.import_context = "null"

        self.w(self.indent() + "runinteractive(function (ctxt) {")
        self.w(self.spacing() + "var $module = ctxt;")
        self.w(self.spacing() + "var __name__ = ctxt.__name__;")
        self.w(self.spacing() + "var %s = ctxt;" % self.js_module_name)

        if self.attribute_checking and not module_name in ['sys', 'pyjslib']:
            attribute_checking = True
            self.w( self.indent() + 'try {')
        else:
            attribute_checking = False

        save_output = self.output
        self.output = StringIO()

        mod.lineno = 1
        self.track_lineno(mod, True)
        for child in mod.node:
            self.has_js_return = False
            self.has_yield = False
            self.is_generator = False
            self.track_lineno(child)
            assert self.top_level
            if isinstance(child, self.ast.Function):
                self._function(child, None)
            elif isinstance(child, self.ast.Class):
                self._class(child)
            elif isinstance(child, self.ast.Import):
                self._import(child, None, True)
            elif isinstance(child, self.ast.From):
                self._from(child, None, True)
            elif isinstance(child, self.ast.Discard):
                self._discard(child, None)
            elif isinstance(child, self.ast.Assign):
                self._assign(child, None)
            elif isinstance(child, self.ast.AugAssign):
                self._augassign(child, None)
            elif isinstance(child, self.ast.If):
                self._if(child, None)
            elif isinstance(child, self.ast.For):
                self._for(child, None)
            elif isinstance(child, self.ast.While):
                self._while(child, None)
            elif isinstance(child, self.ast.Subscript):
                self._subscript_stmt(child, None)
            elif isinstance(child, self.ast.Global):
                self._global(child, None)
            elif isinstance(child, self.ast.Printnl):
               self._print(child, None)
            elif isinstance(child, self.ast.Print):
               self._print(child, None)
            elif isinstance(child, self.ast.TryExcept):
                self._tryExcept(child, None)
            elif isinstance(child, self.ast.TryFinally):
                self._tryFinally(child, None)
            elif isinstance(child, self.ast.Raise):
                self._raise(child, None)
            elif isinstance(child, self.ast.Stmt):
                self._stmt(child, None, True)
            elif isinstance(child, self.ast.With):
                self._with(child, None)
            elif isinstance(child, self.ast.AssAttr):
                self._assattr(child, None)
            elif isinstance(child, self.ast.AssName):
                self._assname(child, None)
            elif isinstance(child, self.ast.AssTuple):
                for node in child.nodes:
                    self._stmt(node, None)
            elif isinstance(child, self.ast.Slice):
                self.w( self.spacing() + self._slice(child, None))
            else:
                raise translator.TranslationError(
                    "unsupported type (in __init__)",
                    child, self.module_name)

        captured_output = self.output.getvalue()
        self.output = save_output
        if self.source_tracking and self.store_source:
            for l in self.track_lines.keys():
                self.w( self.spacing() + '''%s.__track_lines__[%d] = "%s";''' % (self.js_module_name, l, self.track_lines[l].replace('"', '\"')), translate=False)
        self.w( self.local_js_vars_decl([]))
        if captured_output.find("@[email protected]") >= 0:
            captured_output = captured_output.replace("@[email protected]", self.constant_decl())
        else:
            self.w( self.constant_decl())
        if captured_output.find("@[email protected]") >= 0:
            captured_output = captured_output.replace("@[email protected]", self.attrib_remap_decl())
        self.w( captured_output, False)

        if attribute_checking:
            self.w( self.dedent() + "} catch ($pyjs_attr_err) {throw @{{_errorMapping}}($pyjs_attr_err);};")
        self.w( self.dedent() + "}, __main__);")

Example 15

Project: rdflib
Source File: __init__.py
View license
def processURI(uri, outputFormat, form={}) :
	"""The standard processing of an RDFa uri options in a form; used as an entry point from a CGI call.

	The call accepts extra form options (i.e., HTTP GET options) as follows:

	 - C{graph=[output|processor|output,processor|processor,output]} specifying which graphs are returned. Default: C{output}
	 - C{space_preserve=[true|false]} means that plain literals are normalized in terms of white spaces. Default: C{false}
	 - C{rfa_version} provides the RDFa version that should be used for distilling. The string should be of the form "1.0" or "1.1". Default is the highest version the current package implements, currently "1.1"
	 - C{host_language=[xhtml,html,xml]} : the host language. Used when files are uploaded or text is added verbatim, otherwise the HTTP return header should be used. Default C{xml}
	 - C{embedded_rdf=[true|false]} : whether embedded turtle or RDF/XML content should be added to the output graph. Default: C{false}
	 - C{vocab_expansion=[true|false]} : whether the vocabularies should be expanded through the restricted RDFS entailment. Default: C{false}
	 - C{vocab_cache=[true|false]} : whether vocab caching should be performed or whether it should be ignored and vocabulary files should be picked up every time. Default: C{false}
	 - C{vocab_cache_report=[true|false]} : whether vocab caching details should be reported. Default: C{false}
	 - C{vocab_cache_bypass=[true|false]} : whether vocab caches have to be regenerated every time. Default: C{false}
	 - C{rdfa_lite=[true|false]} : whether warnings should be generated for non RDFa Lite attribute usage. Default: C{false}

	@param uri: URI to access. Note that the C{text:} and C{uploaded:} fake URI values are treated separately; the former is for textual intput (in which case a StringIO is used to get the data) and the latter is for uploaded file, where the form gives access to the file directly.
	@param outputFormat: serialization format, as defined by the package. Currently "xml", "turtle", "nt", or "json". Default is "turtle", also used if any other string is given.
	@param form: extra call options (from the CGI call) to set up the local options
	@type form: cgi FieldStorage instance
	@return: serialized graph
	@rtype: string
	"""
	def _get_option(param, compare_value, default) :
		param_old = param.replace('_','-')
		if param in list(form.keys()) :
			val = form.getfirst(param).lower()
			return val == compare_value
		elif param_old in list(form.keys()) :
			# this is to ensure the old style parameters are still valid...
			# in the old days I used '-' in the parameters, the standard favours '_'
			val = form.getfirst(param_old).lower()
			return val == compare_value
		else :
			return default

	if uri == "uploaded:" :
		input	= form["uploaded"].file
		base	= ""
	elif uri == "text:" :
		input	= StringIO(form.getfirst("text"))
		base	= ""
	else :
		input	= uri
		base	= uri

	if "rdfa_version" in list(form.keys()) :
		rdfa_version = form.getfirst("rdfa_version")
	else :
		rdfa_version = None

	# working through the possible options
	# Host language: HTML, XHTML, or XML
	# Note that these options should be used for the upload and inline version only in case of a form
	# for real uris the returned content type should be used
	if "host_language" in list(form.keys()) :
		if form.getfirst("host_language").lower() == "xhtml" :
			media_type = MediaTypes.xhtml
		elif form.getfirst("host_language").lower() == "html" :
			media_type = MediaTypes.html
		elif form.getfirst("host_language").lower() == "svg" :
			media_type = MediaTypes.svg
		elif form.getfirst("host_language").lower() == "atom" :
			media_type = MediaTypes.atom
		else :
			media_type = MediaTypes.xml
	else :
		media_type = ""

	transformers = []

	check_lite = "rdfa_lite" in list(form.keys()) and form.getfirst("rdfa_lite").lower() == "true"

	# The code below is left for backward compatibility only. In fact, these options are not exposed any more,
	# they are not really in use
	if "extras" in list(form.keys()) and form.getfirst("extras").lower() == "true" :
		from .transform.metaname              	import meta_transform
		from .transform.OpenID                	import OpenID_transform
		from .transform.DublinCore            	import DC_transform
		for t in [OpenID_transform, DC_transform, meta_transform] :
			transformers.append(t)
	else :
		if "extra-meta" in list(form.keys()) and form.getfirst("extra-meta").lower() == "true" :
			from .transform.metaname import meta_transform
			transformers.append(meta_transform)
		if "extra-openid" in list(form.keys()) and form.getfirst("extra-openid").lower() == "true" :
			from .transform.OpenID import OpenID_transform
			transformers.append(OpenID_transform)
		if "extra-dc" in list(form.keys()) and form.getfirst("extra-dc").lower() == "true" :
			from .transform.DublinCore import DC_transform
			transformers.append(DC_transform)

	output_default_graph 	= True
	output_processor_graph 	= False
	# Note that I use the 'graph' and the 'rdfagraph' form keys here. Reason is that
	# I used 'graph' in the previous versions, including the RDFa 1.0 processor,
	# so if I removed that altogether that would create backward incompatibilities
	# On the other hand, the RDFa 1.1 doc clearly refers to 'rdfagraph' as the standard
	# key.
	a = None
	if "graph" in list(form.keys()) :
		a = form.getfirst("graph").lower()
	elif "rdfagraph" in list(form.keys()) :
		a = form.getfirst("rdfagraph").lower()
	if a != None :
		if a == "processor" :
			output_default_graph 	= False
			output_processor_graph 	= True
		elif a == "processor,output" or a == "output,processor" :
			output_processor_graph 	= True

	embedded_rdf        = _get_option( "embedded_rdf", "true", False)
	space_preserve      = _get_option( "space_preserve", "true", True)
	vocab_cache         = _get_option( "vocab_cache", "true", True)
	vocab_cache_report  = _get_option( "vocab_cache_report", "true", False)
	refresh_vocab_cache = _get_option( "vocab_cache_refresh", "true", False)
	vocab_expansion     = _get_option( "vocab_expansion", "true", False)
	if vocab_cache_report : output_processor_graph = True

	options = Options(output_default_graph   = output_default_graph,
					  output_processor_graph = output_processor_graph,
					  space_preserve         = space_preserve,
					  transformers           = transformers,
					  vocab_cache            = vocab_cache,
					  vocab_cache_report     = vocab_cache_report,
					  refresh_vocab_cache    = refresh_vocab_cache,
					  vocab_expansion        = vocab_expansion,
					  embedded_rdf           = embedded_rdf,
					  check_lite             = check_lite
					  )
	processor = pyRdfa(options = options, base = base, media_type = media_type, rdfa_version = rdfa_version)

	# Decide the output format; the issue is what should happen in case of a top level error like an inaccessibility of
	# the html source: should a graph be returned or an HTML page with an error message?

	# decide whether HTML or RDF should be sent.
	htmlOutput = False
	#if 'HTTP_ACCEPT' in os.environ :
	#	acc = os.environ['HTTP_ACCEPT']
	#	possibilities = ['text/html',
	#					 'application/rdf+xml',
	#					 'text/turtle; charset=utf-8',
	#					 'application/json',
	#					 'application/ld+json',
	#					 'text/rdf+n3']
	#
	#	# this nice module does content negotiation and returns the preferred format
	#	sg = acceptable_content_type(acc, possibilities)
	#	htmlOutput = (sg != None and sg[0] == content_type('text/html'))
	#	os.environ['rdfaerror'] = 'true'

	# This is really for testing purposes only, it is an unpublished flag to force RDF output no
	# matter what
	try :
		graph = processor.rdf_from_source(input, outputFormat, rdfOutput = ("forceRDFOutput" in list(form.keys())) or not htmlOutput)
		if outputFormat == "n3" :
			retval = 'Content-Type: text/rdf+n3; charset=utf-8\n'
		elif outputFormat == "nt" or outputFormat == "turtle" :
			retval = 'Content-Type: text/turtle; charset=utf-8\n'
		elif outputFormat == "json-ld" or outputFormat == "json" :
			retval = 'Content-Type: application/ld+json; charset=utf-8\n'
		else :
			retval = 'Content-Type: application/rdf+xml; charset=utf-8\n'
		retval += '\n'
		retval += graph
		return retval
	except HTTPError :
		(type,h,traceback) = sys.exc_info()
		import cgi

		retval = 'Content-type: text/html; charset=utf-8\nStatus: %s \n\n' % h.http_code
		retval += "<html>\n"
		retval += "<head>\n"
		retval += "<title>HTTP Error in distilling RDFa content</title>\n"
		retval += "</head><body>\n"
		retval += "<h1>HTTP Error in distilling RDFa content</h1>\n"
		retval += "<p>HTTP Error: %s (%s)</p>\n" % (h.http_code,h.msg)
		retval += "<p>On URI: <code>'%s'</code></p>\n" % cgi.escape(uri)
		retval +="</body>\n"
		retval +="</html>\n"
		return retval
	except :
		# This branch should occur only if an exception is really raised, ie, if it is not turned
		# into a graph value.
		(type,value,traceback) = sys.exc_info()

		import traceback, cgi

		retval = 'Content-type: text/html; charset=utf-8\nStatus: %s\n\n' % processor.http_status
		retval += "<html>\n"
		retval += "<head>\n"
		retval += "<title>Exception in RDFa processing</title>\n"
		retval += "</head><body>\n"
		retval += "<h1>Exception in distilling RDFa</h1>\n"
		retval += "<pre>\n"
		strio  = StringIO()
		traceback.print_exc(file=strio)
		retval += strio.getvalue()
		retval +="</pre>\n"
		retval +="<pre>%s</pre>\n" % value
		retval +="<h1>Distiller request details</h1>\n"
		retval +="<dl>\n"
		if uri == "text:" and "text" in form and form["text"].value != None and len(form["text"].value.strip()) != 0 :
			retval +="<dt>Text input:</dt><dd>%s</dd>\n" % cgi.escape(form["text"].value).replace('\n','<br/>')
		elif uri == "uploaded:" :
			retval +="<dt>Uploaded file</dt>\n"
		else :
			retval +="<dt>URI received:</dt><dd><code>'%s'</code></dd>\n" % cgi.escape(uri)
		if "host_language" in list(form.keys()) :
			retval +="<dt>Media Type:</dt><dd>%s</dd>\n" % media_type
		if "graph" in list(form.keys()) :
			retval +="<dt>Requested graphs:</dt><dd>%s</dd>\n" % form.getfirst("graph").lower()
		else :
			retval +="<dt>Requested graphs:</dt><dd>default</dd>\n"
		retval +="<dt>Output serialization format:</dt><dd> %s</dd>\n" % outputFormat
		if "space_preserve" in form : retval +="<dt>Space preserve:</dt><dd> %s</dd>\n" % form["space_preserve"].value
		retval +="</dl>\n"
		retval +="</body>\n"
		retval +="</html>\n"
		return retval

Example 16

Project: rootpy
Source File: gen_rst.py
View license
def generate_file_rst(fname, target_dir, src_dir, plot_gallery):
    """ Generate the rst file for a given example.
    """
    base_image_name = os.path.splitext(fname)[0]
    image_fname = '%s_%%s.png' % base_image_name
    root_image_fname = 'root_%s_%%s.png' % base_image_name
    root_fig_num = 1

    this_template = rst_template
    last_dir = os.path.split(src_dir)[-1]
    # to avoid leading . in file names, and wrong names in links
    if last_dir == '.' or last_dir == 'examples':
        last_dir = ''
    else:
        last_dir += '_'
    short_fname = last_dir + fname
    src_file = os.path.join(src_dir, fname)
    example_file = os.path.join(target_dir, fname)
    shutil.copyfile(src_file, example_file)

    # The following is a list containing all the figure names
    figure_list = []

    image_dir = os.path.join(target_dir, 'images')
    thumb_dir = os.path.join(image_dir, 'thumb')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)
    image_path = os.path.join(image_dir, image_fname)
    root_image_path = os.path.join(image_dir, root_image_fname)

    stdout_path = os.path.join(image_dir,
                               'stdout_%s.txt' % base_image_name)
    time_path = os.path.join(image_dir,
                               'time_%s.txt' % base_image_name)
    thumb_file = os.path.join(thumb_dir, fname[:-3] + '.png')
    time_elapsed = 0
    if plot_gallery and fname.startswith('plot'):
        # generate the plot as png image if file name
        # starts with plot and if it is more recent than an
        # existing image.
        first_image_file = image_path % 1
        first_root_image_file = root_image_path % 1
        if os.path.exists(stdout_path):
            stdout = open(stdout_path).read()
        else:
            stdout = ''
        if os.path.exists(time_path):
            time_elapsed = float(open(time_path).read())

        if (not os.path.exists(first_image_file) or
            not os.path.exists(first_root_image_file) or
                os.stat(first_image_file).st_mtime <=
                                    os.stat(src_file).st_mtime):
            # We need to execute the code
            print 'plotting %s' % fname
            t0 = time()
            import matplotlib.pyplot as plt
            plt.close('all')
            cwd = os.getcwd()
            try:
                # First CD in the original example dir, so that any file
                # created by the example get created in this directory
                orig_stdout = sys.stdout
                os.chdir(os.path.dirname(src_file))
                my_buffer = StringIO()
                my_stdout = Tee(sys.stdout, my_buffer)
                sys.stdout = my_stdout
                my_globals = {'pl': plt}
                execfile(os.path.basename(src_file), my_globals)
                time_elapsed = time() - t0
                sys.stdout = orig_stdout
                my_stdout = my_buffer.getvalue()
                if '__doc__' in my_globals:
                    # The __doc__ is often printed in the example, we
                    # don't with to echo it
                    my_stdout = my_stdout.replace(
                                            my_globals['__doc__'],
                                            '')
                my_stdout = my_stdout.strip()
                if my_stdout:
                    stdout = '**Script output**::\n\n  %s\n\n' % (
                        '\n  '.join(my_stdout.split('\n')))
                open(stdout_path, 'w').write(stdout)
                open(time_path, 'w').write('%f' % time_elapsed)
                os.chdir(cwd)

                # In order to save every figure we have two solutions :
                # * iterate from 1 to infinity and call plt.fignum_exists(n)
                #   (this requires the figures to be numbered
                #    incrementally: 1, 2, 3 and not 1, 2, 5)
                # * iterate over [fig_mngr.num for fig_mngr in
                #   matplotlib._pylab_helpers.Gcf.get_all_fig_managers()]
                for fig_num in (fig_mngr.num for fig_mngr in
                        matplotlib._pylab_helpers.Gcf.get_all_fig_managers()):
                    # Set the fig_num figure as the current figure as we can't
                    # save a figure that's not the current figure.
                    plt.figure(fig_num)
                    plt.savefig(image_path % fig_num)
                    figure_list.append(image_fname % fig_num)
                for canvas in ROOT.gROOT.GetListOfCanvases():
                    canvas.SaveAs(root_image_path % root_fig_num)
                    canvas.Close()
                    figure_list.append(root_image_fname % root_fig_num)
                    root_fig_num += 1
            except:
                print 80 * '_'
                print '%s is not compiling:' % fname
                traceback.print_exc()
                print 80 * '_'
            finally:
                os.chdir(cwd)
                sys.stdout = orig_stdout

            print " - time elapsed : %.2g sec" % time_elapsed
        else:
            figure_list = [f[len(image_dir):]
                            for f in glob.glob(image_path % '[1-9]')]
                            #for f in glob.glob(image_path % '*')]

        # generate thumb file
        this_template = plot_rst_template
        from matplotlib import image
        if os.path.exists(first_image_file):
            image.thumbnail(first_image_file, thumb_file, 0.2)
        elif os.path.exists(first_root_image_file):
            image.thumbnail(first_root_image_file, thumb_file, 0.2)

    if not os.path.exists(thumb_file):
        # create something not to replace the thumbnail
        shutil.copy('images/blank_image.png', thumb_file)

    docstring, short_desc, end_row = extract_docstring(example_file)

    # Depending on whether we have one or more figures, we're using a
    # horizontal list or a single rst call to 'image'.
    if len(figure_list) == 1:
        figure_name = figure_list[0]
        image_list = SINGLE_IMAGE % figure_name.lstrip('/')
    else:
        image_list = HLIST_HEADER
        for figure_name in figure_list:
            image_list += HLIST_IMAGE_TEMPLATE % figure_name.lstrip('/')

    f = open(os.path.join(target_dir, fname[:-2] + 'rst'), 'w')
    f.write(this_template % locals())
    f.flush()

Example 17

Project: root_numpy
Source File: gen_rst.py
View license
def generate_file_rst(fname, target_dir, src_dir, plot_gallery):
    """ Generate the rst file for a given example.
    """
    base_image_name = os.path.splitext(fname)[0]
    image_fname = '%s_%%s.png' % base_image_name
    root_image_fname = 'root_%s_%%s.png' % base_image_name
    root_fig_num = 1

    this_template = rst_template
    last_dir = os.path.split(src_dir)[-1]
    # to avoid leading . in file names, and wrong names in links
    if last_dir == '.' or last_dir == 'examples':
        last_dir = ''
    else:
        last_dir += '_'
    short_fname = last_dir + fname
    src_file = os.path.join(src_dir, fname)
    example_file = os.path.join(target_dir, fname)
    shutil.copyfile(src_file, example_file)

    # The following is a list containing all the figure names
    figure_list = []

    image_dir = os.path.join(target_dir, 'images')
    thumb_dir = os.path.join(image_dir, 'thumb')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)
    image_path = os.path.join(image_dir, image_fname)
    root_image_path = os.path.join(image_dir, root_image_fname)

    stdout_path = os.path.join(image_dir,
                               'stdout_%s.txt' % base_image_name)
    time_path = os.path.join(image_dir,
                               'time_%s.txt' % base_image_name)
    thumb_file = os.path.join(thumb_dir, fname[:-3] + '.png')
    time_elapsed = 0
    if plot_gallery and fname.startswith('plot'):
        # generate the plot as png image if file name
        # starts with plot and if it is more recent than an
        # existing image.
        first_image_file = image_path % 1
        first_root_image_file = root_image_path % 1
        if os.path.exists(stdout_path):
            stdout = open(stdout_path).read()
        else:
            stdout = ''
        if os.path.exists(time_path):
            time_elapsed = float(open(time_path).read())

        if (not os.path.exists(first_image_file) or
            not os.path.exists(first_root_image_file) or
                os.stat(first_image_file).st_mtime <=
                                    os.stat(src_file).st_mtime):
            # We need to execute the code
            print 'plotting %s' % fname
            t0 = time()
            import matplotlib.pyplot as plt
            plt.close('all')
            cwd = os.getcwd()
            try:
                # First CD in the original example dir, so that any file
                # created by the example get created in this directory
                orig_stdout = sys.stdout
                os.chdir(os.path.dirname(src_file))
                my_buffer = StringIO()
                my_stdout = Tee(sys.stdout, my_buffer)
                sys.stdout = my_stdout
                my_globals = {'pl': plt}
                execfile(os.path.basename(src_file), my_globals)
                time_elapsed = time() - t0
                sys.stdout = orig_stdout
                my_stdout = my_buffer.getvalue()
                if '__doc__' in my_globals:
                    # The __doc__ is often printed in the example, we
                    # don't with to echo it
                    my_stdout = my_stdout.replace(
                                            my_globals['__doc__'],
                                            '')
                my_stdout = my_stdout.strip()
                if my_stdout:
                    stdout = '**Script output**::\n\n  %s\n\n' % (
                        '\n  '.join(my_stdout.split('\n')))
                open(stdout_path, 'w').write(stdout)
                open(time_path, 'w').write('%f' % time_elapsed)
                os.chdir(cwd)

                # In order to save every figure we have two solutions :
                # * iterate from 1 to infinity and call plt.fignum_exists(n)
                #   (this requires the figures to be numbered
                #    incrementally: 1, 2, 3 and not 1, 2, 5)
                # * iterate over [fig_mngr.num for fig_mngr in
                #   matplotlib._pylab_helpers.Gcf.get_all_fig_managers()]
                for fig_num in (fig_mngr.num for fig_mngr in
                        matplotlib._pylab_helpers.Gcf.get_all_fig_managers()):
                    # Set the fig_num figure as the current figure as we can't
                    # save a figure that's not the current figure.
                    plt.figure(fig_num)
                    plt.savefig(image_path % fig_num)
                    figure_list.append(image_fname % fig_num)
                for canvas in ROOT.gROOT.GetListOfCanvases():
                    canvas.SaveAs(root_image_path % root_fig_num)
                    canvas.Close()
                    figure_list.append(root_image_fname % root_fig_num)
                    root_fig_num += 1
            except:
                print 80 * '_'
                print '%s is not compiling:' % fname
                traceback.print_exc()
                print 80 * '_'
            finally:
                os.chdir(cwd)
                sys.stdout = orig_stdout

            print " - time elapsed : %.2g sec" % time_elapsed
        else:
            figure_list = [f[len(image_dir):]
                            for f in glob.glob(image_path % '[1-9]')]
                            #for f in glob.glob(image_path % '*')]

        # generate thumb file
        this_template = plot_rst_template
        from matplotlib import image
        if os.path.exists(first_image_file):
            image.thumbnail(first_image_file, thumb_file, 0.2)
        elif os.path.exists(first_root_image_file):
            image.thumbnail(first_root_image_file, thumb_file, 0.2)

    if not os.path.exists(thumb_file):
        # create something not to replace the thumbnail
        shutil.copy('images/blank_image.png', thumb_file)

    docstring, short_desc, end_row = extract_docstring(example_file)

    # Depending on whether we have one or more figures, we're using a
    # horizontal list or a single rst call to 'image'.
    if len(figure_list) == 1:
        figure_name = figure_list[0]
        image_list = SINGLE_IMAGE % figure_name.lstrip('/')
    else:
        image_list = HLIST_HEADER
        for figure_name in figure_list:
            image_list += HLIST_IMAGE_TEMPLATE % figure_name.lstrip('/')

    f = open(os.path.join(target_dir, fname[:-2] + 'rst'), 'w')
    f.write(this_template % locals())
    f.flush()

Example 18

Project: babble
Source File: test_cgi.py
View license
def main():
    for orig, expect in parse_qsl_test_cases:
        result = cgi.parse_qsl(orig, keep_blank_values=True)
        print repr(orig), '=>', result
        verify(result == expect, "Error parsing %s" % repr(orig))

    for orig, expect in parse_strict_test_cases:
        # Test basic parsing
        print repr(orig)
        d = do_test(orig, "GET")
        verify(d == expect, "Error parsing %s" % repr(orig))
        d = do_test(orig, "POST")
        verify(d == expect, "Error parsing %s" % repr(orig))

        env = {'QUERY_STRING': orig}
        fcd = cgi.FormContentDict(env)
        sd = cgi.SvFormContentDict(env)
        fs = cgi.FieldStorage(environ=env)
        if type(expect) == type({}):
            # test dict interface
            verify(len(expect) == len(fcd))
            verify(norm(expect.keys()) == norm(fcd.keys()))
            verify(norm(expect.values()) == norm(fcd.values()))
            verify(norm(expect.items()) == norm(fcd.items()))
            verify(fcd.get("nonexistent field", "default") == "default")
            verify(len(sd) == len(fs))
            verify(norm(sd.keys()) == norm(fs.keys()))
            verify(fs.getvalue("nonexistent field", "default") == "default")
            # test individual fields
            for key in expect.keys():
                expect_val = expect[key]
                verify(fcd.has_key(key))
                verify(norm(fcd[key]) == norm(expect[key]))
                verify(fcd.get(key, "default") == fcd[key])
                verify(fs.has_key(key))
                if len(expect_val) > 1:
                    single_value = 0
                else:
                    single_value = 1
                try:
                    val = sd[key]
                except IndexError:
                    verify(not single_value)
                    verify(fs.getvalue(key) == expect_val)
                else:
                    verify(single_value)
                    verify(val == expect_val[0])
                    verify(fs.getvalue(key) == expect_val[0])
                verify(norm(sd.getlist(key)) == norm(expect_val))
                if single_value:
                    verify(norm(sd.values()) == \
                           first_elts(norm(expect.values())))
                    verify(norm(sd.items()) == \
                           first_second_elts(norm(expect.items())))

    # Test the weird FormContentDict classes
    env = {'QUERY_STRING': "x=1&y=2.0&z=2-3.%2b0&1=1abc"}
    expect = {'x': 1, 'y': 2.0, 'z': '2-3.+0', '1': '1abc'}
    d = cgi.InterpFormContentDict(env)
    for k, v in expect.items():
        verify(d[k] == v)
    for k, v in d.items():
        verify(expect[k] == v)
    verify(norm(expect.values()) == norm(d.values()))

    print "Testing log"
    cgi.log("Testing")
    cgi.logfp = sys.stdout
    cgi.initlog("%s", "Testing initlog 1")
    cgi.log("%s", "Testing log 2")
    if os.path.exists("/dev/null"):
        cgi.logfp = None
        cgi.logfile = "/dev/null"
        cgi.initlog("%s", "Testing log 3")
        cgi.log("Testing log 4")

    print "Test FieldStorage methods that use readline"
    # FieldStorage uses readline, which has the capacity to read all
    # contents of the input file into memory; we use readline's size argument
    # to prevent that for files that do not contain any newlines in
    # non-GET/HEAD requests
    class TestReadlineFile:
        def __init__(self, file):
            self.file = file
            self.numcalls = 0

        def readline(self, size=None):
            self.numcalls += 1
            if size:
                return self.file.readline(size)
            else:
                return self.file.readline()

        def __getattr__(self, name):
            file = self.__dict__['file']
            a = getattr(file, name)
            if not isinstance(a, int):
                setattr(self, name, a)
            return a

    f = TestReadlineFile(tempfile.TemporaryFile())
    f.write('x' * 256 * 1024)
    f.seek(0)
    env = {'REQUEST_METHOD':'PUT'}
    fs = cgi.FieldStorage(fp=f, environ=env)
    # if we're not chunking properly, readline is only called twice
    # (by read_binary); if we are chunking properly, it will be called 5 times
    # as long as the chunksize is 1 << 16.
    verify(f.numcalls > 2)

    print "Test basic FieldStorage multipart parsing"
    env = {'REQUEST_METHOD':'POST', 'CONTENT_TYPE':'multipart/form-data; boundary=---------------------------721837373350705526688164684', 'CONTENT_LENGTH':'558'}
    postdata = """-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="id"

1234
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="title"


-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="file"; filename="test.txt"
Content-Type: text/plain

Testing 123.

-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="submit"

 Add\x20
-----------------------------721837373350705526688164684--
"""
    fs = cgi.FieldStorage(fp=StringIO(postdata), environ=env)
    verify(len(fs.list) == 4)
    expect = [{'name':'id', 'filename':None, 'value':'1234'},
              {'name':'title', 'filename':None, 'value':''},
              {'name':'file', 'filename':'test.txt','value':'Testing 123.\n'},
              {'name':'submit', 'filename':None, 'value':' Add '}]
    for x in range(len(fs.list)):
        for k, exp in expect[x].items():
            got = getattr(fs.list[x], k)
            verify(got == exp)

Example 19

Project: feedvalidator
Source File: __init__.py
View license
def validateURL(url, firstOccurrenceOnly=1, wantRawData=0):
  """validate RSS from URL, returns events list, or (events, rawdata) tuple"""
  loggedEvents = []
  request = urllib2.Request(url)
  request.add_header("Accept-encoding", "gzip, deflate")
  request.add_header("User-Agent", "FeedValidator/1.3")
  usock = None
  try:
    try:
      usock = urllib2.urlopen(request)
      rawdata = usock.read(MAXDATALENGTH)
      if usock.read(1):
        raise ValidationFailure(logging.ValidatorLimit({'limit': 'feed length > ' + str(MAXDATALENGTH) + ' bytes'}))

      # check for temporary redirects
      if usock.geturl() != request.get_full_url():
        from urlparse import urlsplit
        (scheme, netloc, path, query, fragment) = urlsplit(url)
        if scheme == 'http':
          from httplib import HTTPConnection
          requestUri = (path or '/') + (query and '?' + query)

          conn=HTTPConnection(netloc)
          conn.request("GET", requestUri)
          resp=conn.getresponse()
          if resp.status != 301:
            loggedEvents.append(TempRedirect({}))

    except BadStatusLine as status:
      raise ValidationFailure(logging.HttpError({'status': status.__class__}))

    except urllib2.HTTPError as status:
      rawdata = status.read()
      if len(rawdata) < 512 or 'content-encoding' in status.headers:
        loggedEvents.append(logging.HttpError({'status': status}))
        usock = status
      else:
        rawdata=re.sub('<!--.*?-->','',rawdata)
        lastline = rawdata.strip().split('\n')[-1].strip()
        if sniffPossibleFeed(rawdata):
          loggedEvents.append(logging.HttpError({'status': status}))
          loggedEvents.append(logging.HttpErrorWithPossibleFeed({}))
          usock = status
        else:
          raise ValidationFailure(logging.HttpError({'status': status}))
    except urllib2.URLError as x:
      raise ValidationFailure(logging.HttpError({'status': x.reason}))
    except Timeout as x:
      raise ValidationFailure(logging.IOError({"message": 'Server timed out', "exception":x}))
    except Exception as x:
      raise ValidationFailure(logging.IOError({"message": x.__class__.__name__,
        "exception":x}))

    if usock.headers.get('content-encoding', None) == None:
      loggedEvents.append(Uncompressed({}))

    if usock.headers.get('content-encoding', None) == 'gzip':
      import gzip, StringIO
      try:
        rawdata = gzip.GzipFile(fileobj=StringIO.StringIO(rawdata)).read()
      except:
        import sys
        exctype, value = sys.exc_info()[:2]
        event=logging.IOError({"message": 'Server response declares Content-Encoding: gzip', "exception":value})
        raise ValidationFailure(event)

    if usock.headers.get('content-encoding', None) == 'deflate':
      import zlib
      try:
        rawdata = zlib.decompress(rawdata, -zlib.MAX_WBITS)
      except:
        import sys
        exctype, value = sys.exc_info()[:2]
        event=logging.IOError({"message": 'Server response declares Content-Encoding: deflate', "exception":value})
        raise ValidationFailure(event)

    if usock.headers.get('content-type', None) == 'application/vnd.google-earth.kmz':
      import tempfile, zipfile, os
      try:
        (fd, tempname) = tempfile.mkstemp()
        os.write(fd, rawdata)
        os.close(fd)
        zfd = zipfile.ZipFile(tempname)
        namelist = zfd.namelist()
        for name in namelist:
          if name.endswith('.kml'):
            rawdata = zfd.read(name)
        zfd.close()
        os.unlink(tempname)
      except:
        import sys
        value = sys.exc_info()[:1]
        event=logging.IOError({"message": 'Problem decoding KMZ', "exception":value})
        raise ValidationFailure(event)

    mediaType = None
    charset = None

    # Is the Content-Type correct?
    contentType = usock.headers.get('content-type', None)
    if contentType:
      (mediaType, charset) = mediaTypes.checkValid(contentType, loggedEvents)

    # Check for malformed HTTP headers
    for (h, v) in usock.headers.items():
      if (h.find(' ') >= 0):
        loggedEvents.append(HttpProtocolError({'header': h}))

    selfURIs = [request.get_full_url()]
    baseURI = usock.geturl()
    if not baseURI in selfURIs: selfURIs.append(baseURI)

    # Get baseURI from content-location and/or redirect information
    if usock.headers.get('content-location', None):
      from urlparse import urljoin
      baseURI=urljoin(baseURI,usock.headers.get('content-location', ""))
    elif usock.headers.get('location', None):
      from urlparse import urljoin
      baseURI=urljoin(baseURI,usock.headers.get('location', ""))

    if not baseURI in selfURIs: selfURIs.append(baseURI)
    usock.close()
    usock = None

    mediaTypes.contentSniffing(mediaType, rawdata, loggedEvents)

    encoding, rawdata = xmlEncoding.decode(mediaType, charset, rawdata, loggedEvents, fallback='utf-8')

    if rawdata is None:
      return {'loggedEvents': loggedEvents}

    rawdata = rawdata.replace('\r\n', '\n').replace('\r', '\n') # normalize EOL
    validator = _validate(rawdata, firstOccurrenceOnly, loggedEvents, baseURI, encoding, selfURIs, mediaType=mediaType)

    # Warn about mismatches between media type and feed version
    if mediaType and validator.feedType:
      mediaTypes.checkAgainstFeedType(mediaType, validator.feedType, validator.loggedEvents)

    params = {"feedType":validator.feedType, "loggedEvents":validator.loggedEvents}
    if wantRawData:
      params['rawdata'] = rawdata
    return params

  finally:
    try:
      if usock: usock.close()
    except:
      pass

Example 20

Project: eden
Source File: survey.py
View license
def series_export_spreadsheet(matrix, matrix_answers, logo):
    """
        Now take the matrix data type and generate a spreadsheet from it
    """

    try:
        import xlwt
    except ImportError:
        response.error = T("xlwt not installed, so cannot export as a Spreadsheet")
        output = s3_rest_controller(module, "survey_series",
                                    rheader=s3db.survey_series_rheader)
        return output

    import math

    # -------------------------------------------------------------------------
    def wrap_text(sheet, cell, style):
        row = cell.row
        col = cell.col
        try:
            text = s3_unicode(cell.text)
        except:
            text = cell.text
        width = 16
        # Wrap text and calculate the row width and height
        characters_in_cell = float(width-2)
        twips_per_row = 255 #default row height for 10 point font
        if cell.merged():
            try:
                sheet.write_merge(cell.row,
                                  cell.row + cell.mergeV,
                                  cell.col,
                                  cell.col + cell.mergeH,
                                  text,
                                  style,
                                  )
            except Exception as msg:
                log = current.log
                log.error(msg)
                log.debug("row: %s + vert: %s, col: %s + horiz %s" % \
                          (cell.row, cell.mergeV, cell.col, cell.mergeH))
                posn = "%s,%s" % (cell.row, cell.col)
                if matrix.matrix[posn]:
                    log.debug(matrix.matrix[posn])
            rows = math.ceil((len(text) / characters_in_cell) / (1 + cell.mergeH))
        else:
            sheet.write(cell.row,
                        cell.col,
                        text,
                        style,
                        )
            rows = math.ceil(len(text) / characters_in_cell)
        new_row_height = int(rows * twips_per_row)
        new_col_width = width * COL_WIDTH_MULTIPLIER
        if sheet.row(row).height < new_row_height:
            sheet.row(row).height = new_row_height
        if sheet.col(col).width < new_col_width:
            sheet.col(col).width = new_col_width

    # -------------------------------------------------------------------------
    def merge_styles(list_template, style_list):
        """
            Take a list of styles and return a single style object with
            all the differences from a newly created object added to the
            resultant style.
        """
        if len(style_list) == 0:
            final_style = xlwt.XFStyle()
        elif len(style_list) == 1:
            final_style = list_template[style_list[0]]
        else:
            zero_style = xlwt.XFStyle()
            final_style = xlwt.XFStyle()
            for i in range(0, len(style_list)):
                final_style = merge_object_diff(final_style,
                                                list_template[style_list[i]],
                                                zero_style)
        return final_style

    # -------------------------------------------------------------------------
    def merge_object_diff(base_obj, new_obj, zero_obj):
        """
            Function to copy all the elements in new_obj that are different from
            the zero_obj and place them in the base_obj
        """

        element_list = new_obj.__dict__
        for (element, value) in element_list.items():
            try:
                base_obj.__dict__[element] = merge_object_diff(base_obj.__dict__[element],
                                                               value,
                                                               zero_obj.__dict__[element])
            except:
                if zero_obj.__dict__[element] != value:
                    base_obj.__dict__[element] = value
        return base_obj

    COL_WIDTH_MULTIPLIER = 240
    book = xlwt.Workbook(encoding="utf-8")
    output = StringIO()

    protection = xlwt.Protection()
    protection.cell_locked = 1
    no_protection = xlwt.Protection()
    no_protection.cell_locked = 0

    borders = xlwt.Borders()
    borders.left = xlwt.Borders.DOTTED
    borders.right = xlwt.Borders.DOTTED
    borders.top = xlwt.Borders.DOTTED
    borders.bottom = xlwt.Borders.DOTTED

    border_t1 = xlwt.Borders()
    border_t1.top = xlwt.Borders.THIN
    border_t2 = xlwt.Borders()
    border_t2.top = xlwt.Borders.MEDIUM

    border_l1 = xlwt.Borders()
    border_l1.left = xlwt.Borders.THIN
    border_l2 = xlwt.Borders()
    border_l2.left = xlwt.Borders.MEDIUM

    border_r1 = xlwt.Borders()
    border_r1.right = xlwt.Borders.THIN
    border_r2 = xlwt.Borders()
    border_r2.right = xlwt.Borders.MEDIUM

    border_b1 = xlwt.Borders()
    border_b1.bottom = xlwt.Borders.THIN
    border_b2 = xlwt.Borders()
    border_b2.bottom = xlwt.Borders.MEDIUM

    align_base = xlwt.Alignment()
    align_base.horz = xlwt.Alignment.HORZ_LEFT
    align_base.vert = xlwt.Alignment.VERT_TOP

    align_wrap = xlwt.Alignment()
    align_wrap.horz = xlwt.Alignment.HORZ_LEFT
    align_wrap.vert = xlwt.Alignment.VERT_TOP
    align_wrap.wrap = xlwt.Alignment.WRAP_AT_RIGHT

    shaded_fill = xlwt.Pattern()
    shaded_fill.pattern = xlwt.Pattern.SOLID_PATTERN
    shaded_fill.pattern_fore_colour = 0x16 # 25% Grey
    shaded_fill.pattern_back_colour = 0x08 # Black

    heading_fill = xlwt.Pattern()
    heading_fill.pattern = xlwt.Pattern.SOLID_PATTERN
    heading_fill.pattern_fore_colour = 0x1F # ice_blue
    heading_fill.pattern_back_colour = 0x08 # Black

    style_title =  xlwt.XFStyle()
    style_title.font.height = 0x0140 # 320 twips, 16 points
    style_title.font.bold = True
    style_title.alignment = align_base
    style_header = xlwt.XFStyle()
    style_header.font.height = 0x00F0 # 240 twips, 12 points
    style_header.font.bold = True
    style_header.alignment = align_base
    style_sub_header = xlwt.XFStyle()
    style_sub_header.font.bold = True
    style_sub_header.alignment = align_wrap
    style_section_heading = xlwt.XFStyle()
    style_section_heading.font.bold = True
    style_section_heading.alignment = align_wrap
    style_section_heading.pattern = heading_fill
    style_hint = xlwt.XFStyle()
    style_hint.protection = protection
    style_hint.font.height = 160 # 160 twips, 8 points
    style_hint.font.italic = True
    style_hint.alignment = align_wrap
    style_text = xlwt.XFStyle()
    style_text.protection = protection
    style_text.alignment = align_wrap
    style_instructions = xlwt.XFStyle()
    style_instructions.font.height = 0x00B4 # 180 twips, 9 points
    style_instructions.font.italic = True
    style_instructions.protection = protection
    style_instructions.alignment = align_wrap
    style_box = xlwt.XFStyle()
    style_box.borders = borders
    style_box.protection = no_protection
    style_input = xlwt.XFStyle()
    style_input.borders = borders
    style_input.protection = no_protection
    style_input.pattern = shaded_fill
    box_l1 = xlwt.XFStyle()
    box_l1.borders = border_l1
    box_l2 = xlwt.XFStyle()
    box_l2.borders = border_l2
    box_t1 = xlwt.XFStyle()
    box_t1.borders = border_t1
    box_t2 = xlwt.XFStyle()
    box_t2.borders = border_t2
    box_r1 = xlwt.XFStyle()
    box_r1.borders = border_r1
    box_r2 = xlwt.XFStyle()
    box_r2.borders = border_r2
    box_b1 = xlwt.XFStyle()
    box_b1.borders = border_b1
    box_b2 = xlwt.XFStyle()
    box_b2.borders = border_b2
    style_list = {}
    style_list["styleTitle"] = style_title
    style_list["styleHeader"] = style_header
    style_list["styleSubHeader"] = style_sub_header
    style_list["styleSectionHeading"] = style_section_heading
    style_list["styleHint"] = style_hint
    style_list["styleText"] = style_text
    style_list["styleInstructions"] = style_instructions
    style_list["styleInput"] = style_input
    style_list["boxL1"] = box_l1
    style_list["boxL2"] = box_l2
    style_list["boxT1"] = box_t1
    style_list["boxT2"] = box_t2
    style_list["boxR1"] = box_r1
    style_list["boxR2"] = box_r2
    style_list["boxB1"] = box_b1
    style_list["boxB2"] = box_b2

    sheet1 = book.add_sheet(T("Assessment"))
    sheet2 = book.add_sheet(T("Metadata"))
    max_col = 0
    for cell in matrix.matrix.values():
        if cell.col + cell.mergeH > 255:
            current.log.warning("Cell (%s,%s) - (%s,%s) ignored" % \
                (cell.col, cell.row, cell.col + cell.mergeH, cell.row + cell.mergeV))
            continue
        if cell.col + cell.mergeH > max_col:
            max_col = cell.col + cell.mergeH
        if cell.joined():
            continue
        style = merge_styles(style_list, cell.styleList)
        if (style.alignment.wrap == style.alignment.WRAP_AT_RIGHT):
            # get all the styles from the joined cells
            # and merge these styles in.
            joined_styles = matrix.joinedElementStyles(cell)
            joined_style =  merge_styles(style_list, joined_styles)
            try:
                wrap_text(sheet1, cell, joined_style)
            except:
                pass
        else:
            if cell.merged():
                # get all the styles from the joined cells
                # and merge these styles in.
                joined_styles = matrix.joinedElementStyles(cell)
                joined_style =  merge_styles(style_list, joined_styles)
                try:
                    sheet1.write_merge(cell.row,
                                       cell.row + cell.mergeV,
                                       cell.col,
                                       cell.col + cell.mergeH,
                                       s3_unicode(cell.text),
                                       joined_style,
                                       )
                except Exception as msg:
                    log = current.log
                    log.error(msg)
                    log.debug("row: %s + vert: %s, col: %s + horiz %s" % \
                              (cell.row, cell.mergeV, cell.col, cell.mergeH))
                    posn = "%s,%s" % (cell.row, cell.col)
                    if matrix.matrix[posn]:
                        log.debug(matrix.matrix[posn])
            else:
                sheet1.write(cell.row,
                             cell.col,
                             s3_unicode(cell.text),
                             style,
                             )
    CELL_WIDTH = 480 # approximately 2 characters
    if max_col > 255:
        max_col = 255
    for col in xrange(max_col + 1):
        sheet1.col(col).width = CELL_WIDTH

    sheet2.write(0, 0, "Question Code")
    sheet2.write(0, 1, "Response Count")
    sheet2.write(0, 2, "Values")
    sheet2.write(0, 3, "Cell Address")
    for cell in matrix_answers.matrix.values():
        style = merge_styles(style_list, cell.styleList)
        sheet2.write(cell.row,
                     cell.col,
                     s3_unicode(cell.text),
                     style,
                     )

    if logo != None:
        sheet1.insert_bitmap(logo, 0, 0)

    sheet1.protect = True
    sheet2.protect = True
    for i in range(26):
        sheet2.col(i).width = 0
    sheet2.write(0,
                 26,
                 s3_unicode(T("Please do not remove this sheet")),
                 style_header,
                 )
    sheet2.col(26).width = 12000
    book.save(output)
    return output

Example 21

Project: eden
Source File: xls.py
View license
    def encode(self, data_source, title=None, as_stream=False, **attr):
        """
            Export data as a Microsoft Excel spreadsheet

            @param data_source: the source of the data that is to be encoded
                                as a spreadsheet, can be either of:
                                1) an S3Resource
                                2) an array of value dicts (dict of
                                   column labels as first item, list of
                                   field types as second item)
                                3) a dict like:
                                   {columns: [key, ...],
                                    headers: {key: label},
                                    types: {key: type},
                                    rows: [{key:value}],
                                    }
            @param title: the title for the output document
            @param as_stream: return the buffer (StringIO) rather than
                              its contents (str), useful when the output
                              is supposed to be stored locally
            @param attr: keyword parameters

            @keyword title: the main title of the report
            @keyword list_fields: fields to include in list views
            @keyword report_groupby: used to create a grouping of the result:
                                     either a Field object of the resource
                                     or a string which matches a value in
                                     the heading
            @keyword use_colour: True to add colour to the cells, default False
            @keyword evenodd: render different background colours
                              for even/odd rows ("stripes")
        """

        # Do not redirect from here!
        # ...but raise proper status code, which can be caught by caller
        try:
            import xlwt
        except ImportError:
            error = self.ERROR.XLWT_ERROR
            current.log.error(error)
            raise HTTP(503, body=error)
        try:
            from xlrd.xldate import xldate_from_date_tuple, \
                                    xldate_from_time_tuple, \
                                    xldate_from_datetime_tuple
        except ImportError:
            error = self.ERROR.XLRD_ERROR
            current.log.error(error)
            raise HTTP(503, body=error)

        import datetime

        MAX_CELL_SIZE = self.MAX_CELL_SIZE
        COL_WIDTH_MULTIPLIER = self.COL_WIDTH_MULTIPLIER

        # Get the attributes
        title = attr.get("title")
        if title is None:
            title = current.T("Report")
        list_fields = attr.get("list_fields")
        group = attr.get("dt_group")
        use_colour = attr.get("use_colour", False)
        evenodd = attr.get("evenodd", True)

        # Extract the data from the data_source
        if isinstance(data_source, dict):
            headers = data_source.get("headers", {})
            lfields = data_source.get("columns", list_fields)
            column_types = data_source.get("types")
            types = [column_types[col] for col in lfields]
            rows = data_source.get("rows")
        elif isinstance(data_source, (list, tuple)):
            headers = data_source[0]
            types = data_source[1]
            rows = data_source[2:]
            lfields = list_fields
        else:
            if not list_fields:
                list_fields = data_source.list_fields()
            (title, types, lfields, headers, rows) = self.extract(data_source,
                                                                  list_fields,
                                                                  )

        # Verify columns in items
        request = current.request
        if len(rows) > 0 and len(lfields) > len(rows[0]):
            msg = """modules/s3/codecs/xls: There is an error in the list items, a field doesn't exist
requesting url %s
Headers = %d, Data Items = %d
Headers     %s
List Fields %s""" % (request.url, len(lfields), len(rows[0]), headers, lfields)
            current.log.error(msg)

        # Grouping
        report_groupby = lfields[group] if group else None
        groupby_label = headers[report_groupby] if report_groupby else None

        # Date/Time formats from L10N deployment settings
        settings = current.deployment_settings
        date_format = settings.get_L10n_date_format()
        date_format_str = str(date_format)

        dt_format_translate = self.dt_format_translate
        date_format = dt_format_translate(date_format)
        time_format = dt_format_translate(settings.get_L10n_time_format())
        datetime_format = dt_format_translate(settings.get_L10n_datetime_format())

        title_row = settings.get_xls_title_row()

        # Get styles
        styles = self._styles(use_colour = use_colour,
                              evenodd = evenodd,
                              datetime_format = datetime_format,
                              )

        # Create the workbook
        book = xlwt.Workbook(encoding="utf-8")

        # Add sheets
        sheets = []
        # XLS exports are limited to 65536 rows per sheet, we bypass
        # this by creating multiple sheets
        row_limit = 65536
        sheetnum = len(rows) / row_limit
        # Can't have a / in the sheet_name, so replace any with a space
        sheet_name = str(title.replace("/", " "))
        if len(sheet_name) > 31:
            # Sheet name cannot be over 31 chars
            # (take sheet number suffix into account)
            sheet_name = sheet_name[:31] if sheetnum == 1 else sheet_name[:28]
        count = 1
        while len(sheets) <= sheetnum:
            sheets.append(book.add_sheet("%s-%s" % (sheet_name, count)))
            count += 1

        if callable(title_row):
            # Calling with sheet None to get the number of title rows
            title_row_length = title_row(None)
        else:
            title_row_length = 2

        # Add header row to all sheets, determine columns widths
        header_style = styles["header"]
        for sheet in sheets:
            # Move this down if a title row will be added
            if title_row:
                header_row = sheet.row(title_row_length)
            else:
                header_row = sheet.row(0)
            column_widths = []
            has_id = False
            col_index = 0
            for selector in lfields:
                if selector == report_groupby:
                    continue
                label = headers[selector]
                if label == "Id":
                    # Indicate to adjust col_index when writing out
                    has_id = True
                    column_widths.append(0)
                    col_index += 1
                    continue
                if label == "Sort":
                    continue
                if has_id:
                    # Adjust for the skipped column
                    write_col_index = col_index - 1
                else:
                    write_col_index = col_index
                header_row.write(write_col_index, str(label), header_style)
                width = max(len(label) * COL_WIDTH_MULTIPLIER, 2000)
                width = min(width, 65535) # USHRT_MAX
                column_widths.append(width)
                sheet.col(write_col_index).width = width
                col_index += 1

        title = s3_str(title)

        # Title row (optional, deployment setting)
        if title_row:
            T = current.T
            large_header_style = styles["large_header"]
            notes_style = styles["notes"]
            for sheet in sheets:
                if callable(title_row):
                    # Custom title rows
                    title_row(sheet)
                else:
                    # First row => Title (standard = "title_list" CRUD string)
                    current_row = sheet.row(0)
                    if col_index > 0:
                        sheet.write_merge(0, 0, 0, col_index,
                                          title,
                                          large_header_style,
                                          )
                    current_row.height = 500
                    # Second row => Export date/time
                    current_row = sheet.row(1)
                    current_row.write(0, "%s:" % T("Date Exported"), notes_style)
                    current_row.write(1, request.now, notes_style)
                    # Fix the size of the last column to display the date
                    if 16 * COL_WIDTH_MULTIPLIER > width:
                        sheet.col(col_index).width = 16 * COL_WIDTH_MULTIPLIER

        # Initialize counters
        totalCols = col_index
        # Move the rows down if a title row is included
        if title_row:
            row_index = title_row_length
        else:
            row_index = 0

        # Helper function to get the current row
        def get_current_row(row_count, row_limit):

            sheet_count = int(row_count / row_limit)
            row_number = row_count - (sheet_count * row_limit)
            if sheet_count > 0:
                row_number += 1
            return sheets[sheet_count], sheets[sheet_count].row(row_number)

        # Write the table contents
        subheading = None
        odd_style = styles["odd"]
        even_style = styles["even"]
        subheader_style = styles["subheader"]
        for row in rows:
            # Current row
            row_index += 1
            current_sheet, current_row = get_current_row(row_index, row_limit)
            style = even_style if row_index % 2 == 0 else odd_style

            # Group headers
            if report_groupby:
                represent = s3_strip_markup(s3_unicode(row[report_groupby]))
                if subheading != represent:
                    # Start of new group - write group header
                    subheading = represent
                    current_sheet.write_merge(row_index, row_index, 0, totalCols,
                                             subheading,
                                             subheader_style,
                                             )
                    # Move on to next row
                    row_index += 1
                    current_sheet, current_row = get_current_row(row_index, row_limit)
                    style = even_style if row_index % 2 == 0 else odd_style

            col_index = 0
            remaining_fields = lfields

            # Custom row style?
            row_style = None
            if "_style" in row:
                stylename = row["_style"]
                if stylename in styles:
                    row_style = styles[stylename]

            # Group header/footer row?
            if "_group" in row:
                group_info = row["_group"]
                label = group_info.get("label")
                totals = group_info.get("totals")
                if label:
                    label = s3_strip_markup(s3_unicode(label))
                    style = row_style or subheader_style
                    span = group_info.get("span")
                    if span == 0:
                        current_sheet.write_merge(row_index,
                                                  row_index,
                                                  0,
                                                  totalCols - 1,
                                                  label,
                                                  style,
                                                  )
                        if totals:
                            # Write totals into the next row
                            row_index += 1
                            current_sheet, current_row = \
                                get_current_row(row_index, row_limit)
                    else:
                        current_sheet.write_merge(row_index,
                                                  row_index,
                                                  0,
                                                  span - 1,
                                                  label,
                                                  style,
                                                  )
                        col_index = span
                        remaining_fields = lfields[span:]
                if not totals:
                    continue

            for field in remaining_fields:
                label = headers[field]
                if label == groupby_label:
                    continue
                if label == "Id":
                    # Skip the ID column from XLS exports
                    col_index += 1
                    continue

                if field not in row:
                    represent = ""
                else:
                    represent = s3_strip_markup(s3_unicode(row[field]))

                coltype = types[col_index]
                if coltype == "sort":
                    continue
                if len(represent) > MAX_CELL_SIZE:
                    represent = represent[:MAX_CELL_SIZE]
                value = represent
                if coltype == "date":
                    try:
                        cell_datetime = datetime.datetime.strptime(value,
                                                                   date_format_str)
                        date_tuple = (cell_datetime.year,
                                      cell_datetime.month,
                                      cell_datetime.day)
                        value = xldate_from_date_tuple(date_tuple, 0)
                        style.num_format_str = date_format
                    except:
                        pass
                elif coltype == "datetime":
                    try:
                        cell_datetime = datetime.datetime.strptime(value,
                                                                   date_format_str)
                        date_tuple = (cell_datetime.year,
                                      cell_datetime.month,
                                      cell_datetime.day,
                                      cell_datetime.hour,
                                      cell_datetime.minute,
                                      cell_datetime.second)
                        value = xldate_from_datetime_tuple(date_tuple, 0)
                        style.num_format_str = datetime_format
                    except:
                        pass
                elif coltype == "time":
                    try:
                        cell_datetime = datetime.datetime.strptime(value,
                                                                   date_format_str)
                        date_tuple = (cell_datetime.hour,
                                      cell_datetime.minute,
                                      cell_datetime.second)
                        value = xldate_from_time_tuple(date_tuple)
                        style.num_format_str = time_format
                    except:
                        pass
                elif coltype == "integer":
                    try:
                        value = int(value)
                        style.num_format_str = "0"
                    except:
                        pass
                elif coltype == "double":
                    try:
                        value = float(value)
                        style.num_format_str = "0.00"
                    except:
                        pass
                if has_id:
                    # Adjust for the skipped column
                    write_col_index = col_index - 1
                else:
                    write_col_index = col_index

                current_row.write(write_col_index, value, style)
                width = len(represent) * COL_WIDTH_MULTIPLIER
                if width > column_widths[col_index]:
                    column_widths[col_index] = width
                    current_sheet.col(write_col_index).width = width
                col_index += 1

        # Additional sheet settings
        for sheet in sheets:
            sheet.panes_frozen = True
            sheet.horz_split_pos = 1

        # Write output
        output = StringIO()
        book.save(output)
        output.seek(0)

        if as_stream:
            return output

        # Response headers
        filename = "%s_%s.xls" % (request.env.server_name, title)
        disposition = "attachment; filename=\"%s\"" % filename
        response = current.response
        response.headers["Content-Type"] = contenttype(".xls")
        response.headers["Content-disposition"] = disposition

        return output.read()

Example 22

Project: eden
Source File: s3notify.py
View license
    @classmethod
    def send(cls, r, resource):
        """
            Method to retrieve updates for a subscription, render the
            notification message and send it - responds to POST?format=msg
            requests to the respective resource.

            @param r: the S3Request
            @param resource: the S3Resource
        """

        _debug("S3Notifications.send()")

        json_message = current.xml.json_message

        # Read subscription data
        source = r.body
        source.seek(0)
        data = source.read()
        subscription = json.loads(data)

        #_debug("Notify PE #%s by %s on %s of %s since %s",
        #       subscription["pe_id"],
        #       str(subscription["method"]),
        #       str(subscription["notify_on"]),
        #       subscription["resource"],
        #       subscription["last_check_time"],
        #       )

        # Check notification settings
        notify_on = subscription["notify_on"]
        methods = subscription["method"]
        if not notify_on or not methods:
            return json_message(message="No notifications configured "
                                        "for this subscription")

        # Authorization (pe_id must not be None)
        pe_id = subscription["pe_id"]

        if not pe_id:
            r.unauthorised()

        # Fields to extract
        fields = resource.list_fields(key="notify_fields")
        if "created_on" not in fields:
            fields.append("created_on")

        # Extract the data
        data = resource.select(fields,
                               represent=True,
                               raw_data=True)
        rows = data["rows"]

        # How many records do we have?
        numrows = len(rows)
        if not numrows:
            return json_message(message="No records found")

        #_debug("%s rows:", numrows)

        # Prepare meta-data
        get_config = resource.get_config
        settings = current.deployment_settings

        page_url = subscription["page_url"]

        crud_strings = current.response.s3.crud_strings.get(resource.tablename)
        if crud_strings:
            resource_name = crud_strings.title_list
        else:
            resource_name = string.capwords(resource.name, "_")

        last_check_time = s3_decode_iso_datetime(subscription["last_check_time"])

        email_format = subscription["email_format"]
        if not email_format:
            email_format = settings.get_msg_notify_email_format()

        filter_query = subscription.get("filter_query")

        meta_data = {"systemname": settings.get_system_name(),
                     "systemname_short": settings.get_system_name_short(),
                     "resource": resource_name,
                     "page_url": page_url,
                     "notify_on": notify_on,
                     "last_check_time": last_check_time,
                     "filter_query": filter_query,
                     "total_rows": numrows,
                     }

        # Render contents for the message template(s)
        renderer = get_config("notify_renderer")
        if not renderer:
            renderer = settings.get_msg_notify_renderer()
        if not renderer:
            renderer = cls._render

        contents = {}
        if email_format == "html" and "EMAIL" in methods:
            contents["html"] = renderer(resource, data, meta_data, "html")
            contents["default"] = contents["html"]
        if email_format != "html" or "EMAIL" not in methods or len(methods) > 1:
            contents["text"] = renderer(resource, data, meta_data, "text")
            contents["default"] = contents["text"]

        # Subject line
        subject = get_config("notify_subject")
        if not subject:
            subject = settings.get_msg_notify_subject()
        if callable(subject):
            subject = subject(resource, data, meta_data)

        from string import Template
        subject = Template(subject).safe_substitute(S="%(systemname)s",
                                                    s="%(systemname_short)s",
                                                    r="%(resource)s")
        subject = subject % meta_data

        # Helper function to find templates from a priority list
        join = lambda *f: os.path.join(current.request.folder, *f)
        def get_template(path, filenames):
            for fn in filenames:
                filepath = join(path, fn)
                if os.path.exists(filepath):
                    try:
                        return open(filepath, "rb")
                    except:
                        pass
            return None

        # Render and send the message(s)
        themes = settings.get_template()
        prefix = resource.get_config("notify_template", "notify")

        send = current.msg.send_by_pe_id

        success = False
        errors = []

        for method in methods:

            error = None

            # Get the message template
            template = None
            filenames = ["%s_%s.html" % (prefix, method.lower())]
            if method == "EMAIL" and email_format:
                filenames.insert(0, "%s_email_%s.html" % (prefix, email_format))
            if themes != "default":
                location = settings.get_template_location()
                if not isinstance(themes, (tuple, list)):
                    themes = (themes,)
                for theme in themes[::-1]:
                    path = join(location, "templates", theme, "views", "msg")
                    template = get_template(path, filenames)
                    if template is not None:
                        break
            if template is None:
                path = join("views", "msg")
                template = get_template(path, filenames)
            if template is None:
                template = StringIO(s3_str(current.T("New updates are available.")))

            # Select contents format
            if method == "EMAIL" and email_format == "html":
                output = contents["html"]
            else:
                output = contents["text"]

            # Render the message
            try:
                message = current.response.render(template, output)
            except:
                exc_info = sys.exc_info()[:2]
                error = ("%s: %s" % (exc_info[0].__name__, exc_info[1]))
                errors.append(error)
                continue

            if not message:
                continue

            # Send the message
            #_debug("Sending message per %s", method)
            #_debug(message)
            try:
                sent = send(pe_id,
                            subject=s3_truncate(subject, 78),
                            message=message,
                            contact_method=method,
                            system_generated=True)
            except:
                exc_info = sys.exc_info()[:2]
                error = ("%s: %s" % (exc_info[0].__name__, exc_info[1]))
                sent = False

            if sent:
                # Successful if at least one notification went out
                success = True
            else:
                if not error:
                    error = current.session.error
                    if isinstance(error, list):
                        error = "/".join(error)
                if error:
                    errors.append(error)

        # Done
        if errors:
            message = ", ".join(errors)
        else:
            message = "Success"
        return json_message(success=success,
                            statuscode=200 if success else 403,
                            message=message)

Example 23

Project: eden
Source File: parser.py
View license
    @staticmethod
    def parse_rss_2_cap(message):
        """
            Parse RSS Feeds into the CAP Module
        """

        db = current.db
        s3db = current.s3db
        table = s3db.msg_rss
        message_id = message.message_id
        record = db(table.message_id == message_id).select(table.id,
                                                           table.channel_id,
                                                           table.title,
                                                           table.from_address,
                                                           table.body,
                                                           table.date,
                                                           table.location_id,
                                                           table.author,
                                                           limitby=(0, 1)
                                                           ).first()
        if not record:
            return

        pstable = s3db.msg_parsing_status
        # not adding (pstable.channel_id == record.channel_id) to query
        # because two channels (http://host.domain/eden/cap/public.rss and
        # (http://host.domain/eden/cap/alert.rss) may contain common url
        # eg. http://host.domain/eden/cap/public/xx.cap
        pquery = (pstable.message_id == message_id)
        prows = db(pquery).select(pstable.id, pstable.is_parsed)
        for prow in prows:
            if prow.is_parsed:
                return

        alert_table = s3db.cap_alert
        info_table = s3db.cap_info

        # Is this an Update or a Create?
        # @ToDo: Use guid?
        # Use Body
        body = record.body or record.title
        query = (info_table.description == body)
        exists = db(query).select(info_table.id,
                                  limitby=(0, 1)
                                  ).first()

        author = record.author
        if author:
            ptable = s3db.pr_person
            # https://code.google.com/p/python-nameparser/
            from nameparser import HumanName
            name = HumanName(author)
            first_name = name.first
            middle_name = name.middle
            last_name = name.last
            query = (ptable.first_name == first_name) & \
                    (ptable.middle_name == middle_name) & \
                    (ptable.last_name == last_name)
            pexists = db(query).select(ptable.id,
                                       limitby=(0, 1)
                                       ).first()
            if pexists:
                person_id = pexists.id
            else:
                person_id = ptable.insert(first_name = first_name,
                                          middle_name = middle_name,
                                          last_name = last_name)
                s3db.update_super(ptable, dict(id=person_id))
        else:
            person_id = None

        if exists:
            # @ToDo: Use XSLT
            info_id = exists.id
            db(info_table.id == info_id).update(headline = record.title,
                                                description = body,
                                                created_on = record.date,
                                                #location_id = record.location_id,
                                                #person_id = person_id,
                                                )

        else:
            # Embedded link
            url = record.from_address
            import_xml = s3db.resource("cap_alert").import_xml
            stylesheet = os.path.join(current.request.folder, "static", "formats", "cap", "import.xsl")
            try:
                file = fetch(url)
            except urllib2.HTTPError, e:
                import base64
                rss_table = s3db.msg_rss_channel
                query = (rss_table.channel_id == record.channel_id)
                channel = db(query).select(rss_table.date,
                                           rss_table.etag,
                                           rss_table.url,
                                           rss_table.username,
                                           rss_table.password,
                                           limitby=(0, 1)).first()
                username = channel.username
                password = channel.password
                if e.code == 401 and username and password:
                    request = urllib2.Request(url)
                    base64string = base64.encodestring("%s:%s" % (username, password))
                    request.add_header("Authorization", "Basic %s" % base64string)
                else:
                    request = None

                try:
                    file = urllib2.urlopen(request).read() if request else fetch(url)
                except urllib2.HTTPError, e:
                    # Check if there are links to look into
                    from urlparse import urlparse
                    ltable = s3db.msg_rss_link
                    query_ = (ltable.rss_id == record.id) & (ltable.deleted != True)
                    rows_ = db(query_).select(ltable.type,
                                              ltable.url)
                    url_format = "{uri.scheme}://{uri.netloc}/".format
                    url_domain = url_format(uri=urlparse(url))
                    for row_ in rows_:
                        url = row_.url
                        if url and row_.type == "application/cap+xml" and \
                           url_domain == url_format(uri=urlparse(url)):
                            # Same domain, so okey to use same username/pwd combination
                            if e.code == 401 and username and password:
                                request = urllib2.Request(url)
                                request.add_header("Authorization", "Basic %s" % base64string)
                            else:
                                request = None
                            try:
                                file = urllib2.urlopen(request).read() if request else fetch(url)
                            except urllib2.HTTPError, e:
                                current.log.error("Getting content from link failed: %s" % e)
                            else:
                                # Import via XSLT
                                import_xml(StringIO(file), stylesheet=stylesheet, ignore_errors=True)
                else:
                    # Import via XSLT
                    import_xml(StringIO(file), stylesheet=stylesheet, ignore_errors=True)
            else:
                # Public Alerts
                # eg. http://host.domain/eden/cap/public/xx.cap
                # Import via XSLT
                import_xml(StringIO(file), stylesheet=stylesheet, ignore_errors=True)

        # No Reply
        return

Example 24

View license
def bind_api(**config):

    class APIMethod(object):

        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        require_auth = config.get('require_auth', False)
        search_api = config.get('search_api', False)
        use_cache = config.get('use_cache', True)

        def __init__(self, api, args, kargs):
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

            self.api = api
            self.post_data = kargs.pop('post_data', None)
            self.retry_count = kargs.pop('retry_count', api.retry_count)
            self.retry_delay = kargs.pop('retry_delay', api.retry_delay)
            self.retry_errors = kargs.pop('retry_errors', api.retry_errors)
            self.headers = kargs.pop('headers', {})
            self.build_parameters(args, kargs)

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()

            if api.secure:
                self.scheme = 'https://'
            else:
                self.scheme = 'http://'

            if self.search_api:
                self.host = api.search_host
            else:
                self.host = api.host

            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
            # See Issue https://github.com/tweepy/tweepy/issues/12
            self.headers['Host'] = self.host

        def build_parameters(self, args, kargs):
            self.parameters = {}
            for idx, arg in enumerate(args):
                if arg is None:
                    continue

                try:
                    self.parameters[self.allowed_param[idx]] = convert_to_utf8_str(arg)
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

            for k, arg in kargs.items():
                if arg is None:
                    continue
                if k in self.parameters:
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

                self.parameters[k] = convert_to_utf8_str(arg)

        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

                if name == 'user' and 'user' not in self.parameters and self.api.auth:
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
                        value = urllib.quote(self.parameters[name])
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
                    del self.parameters[name]

                self.path = self.path.replace(variable, value)

        def execute(self):
            # Build the request URL
            url = self.api_root + self.path
            if len(self.parameters):
                url = '%s?%s' % (url, urllib.urlencode(self.parameters))

            # Query the cache if one is available
            # and this request uses a GET method.
            if self.use_cache and self.api.cache and self.method == 'GET':
                cache_result = self.api.cache.get(url)
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
                            if isinstance(result, Model):
                                result._api = self.api
                    else:
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
                # Open connection
                if self.api.secure:
                    conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout)
                else:
                    conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout)

                # Apply authentication
                if self.api.auth:
                    self.api.auth.apply_auth(
                            self.scheme + self.host + url,
                            self.method, self.headers, self.parameters
                    )

                # Request compression if configured
                if self.api.compression:
                    self.headers['Accept-encoding'] = 'gzip'

                # Execute request
                try:
                    conn.request(self.method, url, headers=self.headers, body=self.post_data)
                    resp = conn.getresponse()

                except Exception, e:
                    raise TweepError('Failed to send request: %s' % e)

                # Exit request loop if non-retry error code
                if self.retry_errors:
                    if resp.status not in self.retry_errors: break
                else:
                    if resp.status == 200: break

                # Sleep before retrying request again
                time.sleep(self.retry_delay)
                retries_performed += 1

            # If an error was returned, throw an exception
            self.api.last_response = resp
            if resp.status and not 200 <= resp.status < 300:
                try:
                    error_msg = self.api.parser.parse_error(resp.read())
                except Exception:
                    error_msg = "Twitter error response: status code = %s" % resp.status
                raise TweepError(error_msg, resp)

            # Parse the response payload
            body = resp.read()
            if resp.getheader('Content-Encoding', '') == 'gzip':
                try:
                    zipper = gzip.GzipFile(fileobj=StringIO(body))
                    body = zipper.read()
                except Exception, e:
                    raise TweepError('Failed to decompress data: %s' % e)
            result = self.api.parser.parse(self, body)

            conn.close()

            # Store result into cache if one is available.
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
                self.api.cache.store(url, result)

            return result


    def _call(api, *args, **kargs):

        method = APIMethod(api, args, kargs)
        return method.execute()


    # Set pagination mode
    if 'cursor' in APIMethod.allowed_param:
        _call.pagination_mode = 'cursor'
    elif 'max_id' in APIMethod.allowed_param and \
         'since_id' in APIMethod.allowed_param:
        _call.pagination_mode = 'id'
    elif 'page' in APIMethod.allowed_param:
        _call.pagination_mode = 'page'

    return _call

Example 25

Project: CrisisLex
Source File: binder.py
View license
def bind_api(**config):

    class APIMethod(object):

        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        require_auth = config.get('require_auth', False)
        search_api = config.get('search_api', False)
        use_cache = config.get('use_cache', True)

        def __init__(self, api, args, kargs):
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

            self.api = api
            self.post_data = kargs.pop('post_data', None)
            self.retry_count = kargs.pop('retry_count', api.retry_count)
            self.retry_delay = kargs.pop('retry_delay', api.retry_delay)
            self.retry_errors = kargs.pop('retry_errors', api.retry_errors)
            self.headers = kargs.pop('headers', {})
            self.build_parameters(args, kargs)

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()

            if api.secure:
                self.scheme = 'https://'
            else:
                self.scheme = 'http://'

            if self.search_api:
                self.host = api.search_host
            else:
                self.host = api.host

            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
            # See Issue https://github.com/tweepy/tweepy/issues/12
            self.headers['Host'] = self.host

        def build_parameters(self, args, kargs):
            self.parameters = {}
            for idx, arg in enumerate(args):
                if arg is None:
                    continue

                try:
                    self.parameters[self.allowed_param[idx]] = convert_to_utf8_str(arg)
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

            for k, arg in kargs.items():
                if arg is None:
                    continue
                if k in self.parameters:
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

                self.parameters[k] = convert_to_utf8_str(arg)

        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

                if name == 'user' and 'user' not in self.parameters and self.api.auth:
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
                        value = urllib.quote(self.parameters[name])
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
                    del self.parameters[name]

                self.path = self.path.replace(variable, value)

        def execute(self):
            # Build the request URL
            url = self.api_root + self.path
            if len(self.parameters):
                url = '%s?%s' % (url, urllib.urlencode(self.parameters))

            # Query the cache if one is available
            # and this request uses a GET method.
            if self.use_cache and self.api.cache and self.method == 'GET':
                cache_result = self.api.cache.get(url)
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
                            if isinstance(result, Model):
                                result._api = self.api
                    else:
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
                # Open connection
                if self.api.secure:
                    conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout)
                else:
                    conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout)

                # Apply authentication
                if self.api.auth:
                    self.api.auth.apply_auth(
                            self.scheme + self.host + url,
                            self.method, self.headers, self.parameters
                    )

                # Request compression if configured
                if self.api.compression:
                    self.headers['Accept-encoding'] = 'gzip'

                # Execute request
                try:
                    conn.request(self.method, url, headers=self.headers, body=self.post_data)
                    resp = conn.getresponse()
                except Exception, e:
                    raise TweepError('Failed to send request: %s' % e)

                # Exit request loop if non-retry error code
                if self.retry_errors:
                    if resp.status not in self.retry_errors: break
                else:
                    if resp.status == 200: break

                # Sleep before retrying request again
                time.sleep(self.retry_delay)
                retries_performed += 1

            # If an error was returned, throw an exception
            self.api.last_response = resp
            if resp.status != 200:
                try:
                    error_msg = self.api.parser.parse_error(resp.read())
                except Exception:
                    error_msg = "Twitter error response: status code = %s" % resp.status
                raise TweepError(error_msg, resp)

            # Parse the response payload
            body = resp.read()
            if resp.getheader('Content-Encoding', '') == 'gzip':
                try:
                    zipper = gzip.GzipFile(fileobj=StringIO(body))
                    body = zipper.read()
                except Exception, e:
                    raise TweepError('Failed to decompress data: %s' % e)
            result = self.api.parser.parse(self, body)

            conn.close()

            # Store result into cache if one is available.
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
                self.api.cache.store(url, result)

            return result


    def _call(api, *args, **kargs):

        method = APIMethod(api, args, kargs)
        return method.execute()


    # Set pagination mode
    if 'cursor' in APIMethod.allowed_param:
        _call.pagination_mode = 'cursor'
    elif 'max_id' in APIMethod.allowed_param and \
         'since_id' in APIMethod.allowed_param:
        _call.pagination_mode = 'id'
    elif 'page' in APIMethod.allowed_param:
        _call.pagination_mode = 'page'

    return _call

Example 26

Project: NOT_UPDATED_Sick-Beard-Dutch
Source File: mp4.py
View license
    def _readatom(self, file):
        s = file.read(8)
        if len(s) < 8:
            return 0

        atomsize, atomtype = struct.unpack('>I4s', s)
        if not str(atomtype).decode('latin1').isalnum():
            # stop at nonsense data
            return 0

        log.debug(u'%r [%X]' % (atomtype, atomsize))

        if atomtype == 'udta':
            # Userdata (Metadata)
            pos = 0
            tabl = {}
            i18ntabl = {}
            atomdata = file.read(atomsize - 8)
            while pos < atomsize - 12:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
                if ord(datatype[0]) == 169:
                    # i18n Metadata...
                    mypos = 8 + pos
                    while mypos + 4 < datasize + pos:
                        # first 4 Bytes are i18n header
                        (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos + 4])
                        i18ntabl[lang] = i18ntabl.get(lang, {})
                        l = atomdata[mypos + 4:mypos + tlen + 4]
                        i18ntabl[lang][datatype[1:]] = l
                        mypos += tlen + 4
                elif datatype == 'WLOC':
                    # Drop Window Location
                    pass
                else:
                    if ord(atomdata[pos + 8:pos + datasize][0]) > 1:
                        tabl[datatype] = atomdata[pos + 8:pos + datasize]
                pos += datasize
            if len(i18ntabl.keys()) > 0:
                for k in i18ntabl.keys():
                    if QTLANGUAGES.has_key(k) and QTLANGUAGES[k] == 'en':
                        self._appendtable('QTUDTA', i18ntabl[k])
                        self._appendtable('QTUDTA', tabl)
            else:
                log.debug(u'NO i18')
                self._appendtable('QTUDTA', tabl)

        elif atomtype == 'trak':
            atomdata = file.read(atomsize - 8)
            pos = 0
            trackinfo = {}
            tracktype = None
            while pos < atomsize - 8:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])

                if datatype == 'tkhd':
                    tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos + 8:pos + datasize])
                    trackinfo['width'] = tkhd[10] >> 16
                    trackinfo['height'] = tkhd[11] >> 16
                    trackinfo['id'] = tkhd[3]

                    try:
                        # XXX Timestamp of Seconds is since January 1st 1904!
                        # XXX 2082844800 is the difference between Unix and
                        # XXX Apple time. FIXME to work on Apple, too
                        self.timestamp = int(tkhd[1]) - 2082844800
                    except Exception, e:
                        log.exception(u'There was trouble extracting timestamp')

                elif datatype == 'mdia':
                    pos += 8
                    datasize -= 8
                    log.debug(u'--> mdia information')

                    while datasize:
                        mdia = struct.unpack('>I4s', atomdata[pos:pos + 8])
                        if mdia[1] == 'mdhd':
                            # Parse based on version of mdhd header.  See
                            # http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd
                            ver = ord(atomdata[pos + 8])
                            if ver == 0:
                                mdhd = struct.unpack('>IIIIIhh', atomdata[pos + 8:pos + 8 + 24])
                            elif ver == 1:
                                mdhd = struct.unpack('>IQQIQhh', atomdata[pos + 8:pos + 8 + 36])
                            else:
                                mdhd = None

                            if mdhd:
                                # duration / time scale
                                trackinfo['length'] = mdhd[4] / mdhd[3]
                                if mdhd[5] in QTLANGUAGES:
                                    trackinfo['language'] = QTLANGUAGES[mdhd[5]]
                                # mdhd[6] == quality
                                self.length = max(self.length, mdhd[4] / mdhd[3])
                        elif mdia[1] == 'minf':
                            # minf has only atoms inside
                            pos -= (mdia[0] - 8)
                            datasize += (mdia[0] - 8)
                        elif mdia[1] == 'stbl':
                            # stbl has only atoms inside
                            pos -= (mdia[0] - 8)
                            datasize += (mdia[0] - 8)
                        elif mdia[1] == 'hdlr':
                            hdlr = struct.unpack('>I4s4s', atomdata[pos + 8:pos + 8 + 12])
                            if hdlr[1] == 'mhlr':
                                if hdlr[2] == 'vide':
                                    tracktype = 'video'
                                if hdlr[2] == 'soun':
                                    tracktype = 'audio'
                        elif mdia[1] == 'stsd':
                            stsd = struct.unpack('>2I', atomdata[pos + 8:pos + 8 + 8])
                            if stsd[1] > 0:
                                codec = atomdata[pos + 16:pos + 16 + 8]
                                codec = struct.unpack('>I4s', codec)
                                trackinfo['codec'] = codec[1]
                                if codec[1] == 'jpeg':
                                    tracktype = 'image'
                        elif mdia[1] == 'dinf':
                            dref = struct.unpack('>I4s', atomdata[pos + 8:pos + 8 + 8])
                            log.debug(u'  --> %r, %r (useless)' % mdia)
                            if dref[1] == 'dref':
                                num = struct.unpack('>I', atomdata[pos + 20:pos + 20 + 4])[0]
                                rpos = pos + 20 + 4
                                for ref in range(num):
                                    # FIXME: do somthing if this references
                                    ref = struct.unpack('>I3s', atomdata[rpos:rpos + 7])
                                    data = atomdata[rpos + 7:rpos + ref[0]]
                                    rpos += ref[0]
                        else:
                            if mdia[1].startswith('st'):
                                log.debug(u'  --> %r, %r (sample)' % mdia)
                            elif mdia[1] == 'vmhd' and not tracktype:
                                # indicates that this track is video
                                tracktype = 'video'
                            elif mdia[1] in ['vmhd', 'smhd'] and not tracktype:
                                # indicates that this track is audio
                                tracktype = 'audio'
                            else:
                                log.debug(u'  --> %r, %r (unknown)' % mdia)

                        pos += mdia[0]
                        datasize -= mdia[0]

                elif datatype == 'udta':
                    log.debug(u'udta: %r' % struct.unpack('>I4s', atomdata[:8]))
                else:
                    if datatype == 'edts':
                        log.debug(u'--> %r [%d] (edit list)' % \
                                  (datatype, datasize))
                    else:
                        log.debug(u'--> %r [%d] (unknown)' % \
                                  (datatype, datasize))
                pos += datasize

            info = None
            if tracktype == 'video':
                info = core.VideoStream()
                self.video.append(info)
            if tracktype == 'audio':
                info = core.AudioStream()
                self.audio.append(info)
            if info:
                for key, value in trackinfo.items():
                    setattr(info, key, value)

        elif atomtype == 'mvhd':
            # movie header
            mvhd = struct.unpack('>6I2h', file.read(28))
            self.length = max(self.length, mvhd[4] / mvhd[3])
            self.volume = mvhd[6]
            file.seek(atomsize - 8 - 28, 1)


        elif atomtype == 'cmov':
            # compressed movie
            datasize, atomtype = struct.unpack('>I4s', file.read(8))
            if not atomtype == 'dcom':
                return atomsize

            method = struct.unpack('>4s', file.read(datasize - 8))[0]

            datasize, atomtype = struct.unpack('>I4s', file.read(8))
            if not atomtype == 'cmvd':
                return atomsize

            if method == 'zlib':
                data = file.read(datasize - 8)
                try:
                    decompressed = zlib.decompress(data)
                except Exception, e:
                    try:
                        decompressed = zlib.decompress(data[4:])
                    except Exception, e:
                        log.exception(u'There was a proble decompressiong atom')
                        return atomsize

                decompressedIO = StringIO.StringIO(decompressed)
                while self._readatom(decompressedIO):
                    pass

            else:
                log.info(u'unknown compression %r' % method)
                # unknown compression method
                file.seek(datasize - 8, 1)

        elif atomtype == 'moov':
            # decompressed movie info
            while self._readatom(file):
                pass

        elif atomtype == 'mdat':
            pos = file.tell() + atomsize - 8
            # maybe there is data inside the mdat
            log.info(u'parsing mdat')
            while self._readatom(file):
                pass
            log.info(u'end of mdat')
            file.seek(pos, 0)


        elif atomtype == 'rmra':
            # reference list
            while self._readatom(file):
                pass

        elif atomtype == 'rmda':
            # reference
            atomdata = file.read(atomsize - 8)
            pos = 0
            url = ''
            quality = 0
            datarate = 0
            while pos < atomsize - 8:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
                if datatype == 'rdrf':
                    rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos + 8:pos + 20])
                    if rtype == 'url ':
                        url = atomdata[pos + 20:pos + 20 + rlen]
                        if url.find('\0') > 0:
                            url = url[:url.find('\0')]
                elif datatype == 'rmqu':
                    quality = struct.unpack('>I', atomdata[pos + 8:pos + 12])[0]

                elif datatype == 'rmdr':
                    datarate = struct.unpack('>I', atomdata[pos + 12:pos + 16])[0]

                pos += datasize
            if url:
                self._references.append((url, quality, datarate))

        else:
            if not atomtype in ['wide', 'free']:
                log.info(u'unhandled base atom %r' % atomtype)

            # Skip unknown atoms
            try:
                file.seek(atomsize - 8, 1)
            except IOError:
                return 0

        return atomsize

Example 27

Project: exporters
Source File: test_readers_s3.py
View license
    def setUp(self):
        self.mock_s3 = moto.mock_s3()
        self.mock_s3.start()
        self.s3_conn = boto.connect_s3()
        self.s3_conn.create_bucket('no_keys_bucket')
        bucket = self.s3_conn.get_bucket('no_keys_bucket')
        for key_name in NO_KEYS:
            key = bucket.new_key(key_name)
            key.set_contents_from_string('')
            key.close()

        self.s3_conn.create_bucket('valid_keys_bucket')
        bucket = self.s3_conn.get_bucket('valid_keys_bucket')

        for key_name in VALID_KEYS:
            key = bucket.new_key(key_name)
            out = StringIO.StringIO()
            with gzip.GzipFile(fileobj=out, mode='w') as f:
                f.write(json.dumps({'name': key_name}))
            key.set_contents_from_string(out.getvalue())
            key.close()

        self.options_no_keys = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'no_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_list/',
                'pattern': 'dump_p(.*)_US_(.*)'
            }
        }

        self.options_valid = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_list/',
                'pattern': 'dump_p(.*)_US_(.*)'
            }
        }

        self.options_no_pattern = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_list/',
                'batch_size': 1
            }
        }

        self.options_no_prefix = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'pattern': '(.*)dump_p(.*)_US_(.*)'
            }
        }

        self.options_prefix_and_prefix_pointer = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_list/',
                'prefix_pointer': 'test_list/LAST'
            }
        }

        self.options_date_prefix = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_prefix/%Y-%m-%d'
            }
        }

        self.options_dateparser = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_prefix/%Y-%m-%d',
                'prefix_format_using_date': 'yesterday'
            }
        }

        self.options_dateparser_range_3_days = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_prefix/%Y-%m-%d',
                'prefix_format_using_date': ['2 days ago', 'today']
            }
        }

        self.options_date_prefix_list = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': ['a_prefix/daily/%Y-%m-%d',
                           'b_prefix/daily/%Y-%m-%d',
                           'c_prefix/daily/%Y-%m-%d']
            }
        }

        self.options_prefix_list_using_date = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': ['a_prefix/daily/%Y-%m-%d',
                           'b_prefix/daily/%Y-%m-%d',
                           'c_prefix/daily/%Y-%m-%d'],
                'prefix_format_using_date': 'yesterday'
            }
        }

        self.options_with_invalid_date_range = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 'valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_prefix/%Y-%m-%d',
                'prefix_format_using_date': ['today', '2 days ago']
            }
        }

        self.options_valid_prefix = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 's3://valid_keys_bucket',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_list/',
                'pattern': 'dump_p(.*)_US_(.*)'
            }
        }

        self.options_valid_prefix_and_suffix = {
            'name': 'exporters.readers.s3_reader.S3Reader',
            'options': {
                'bucket': 's3://valid_keys_bucket/',
                'aws_access_key_id': 'KEY',
                'aws_secret_access_key': 'SECRET',
                'prefix': 'test_list/',
                'pattern': 'dump_p(.*)_US_(.*)'
            }
        }

Example 28

Project: sdaps
Source File: generic.py
View license
def create_stamp_pdf(survey, output_filename, questionnaire_ids):
    sheets = 1 if questionnaire_ids is None else len(questionnaire_ids)

    questionnaire_length = survey.questionnaire.page_count

    have_pdftk = False
    # Test if pdftk is present, if it is we can use it to be faster
    try:
        result = subprocess.Popen(['pdftk', '--version'], stdout=subprocess.PIPE)
        # Just assume pdftk is there, if it was executed sucessfully
        if result is not None:
            have_pdftk = True
    except OSError:
        pass

    if not have_pdftk:
        try:
            import pyPdf
        except:
            log.error(_(u'You need to have either pdftk or pyPdf installed. pdftk is the faster method.'))
            sys.exit(1)

    # Write the "stamp" out to tmp.pdf if are using pdftk.
    if have_pdftk:
        stampsfile = file(survey.path('tmp.pdf'), 'wb')
    else:
        stampsfile = StringIO.StringIO()

    canvas = \
        reportlab.pdfgen.canvas.Canvas(stampsfile,
                                       bottomup=False,
                                       pagesize=(survey.defs.paper_width * mm,
                                                 survey.defs.paper_height * mm))
    # bottomup = False =>(0, 0) is the upper left corner

    print ungettext(u'Creating stamp PDF for %i sheet', u'Creating stamp PDF for %i sheets', sheets) % sheets
    log.progressbar.start(sheets)
    for i in range(sheets):
        if questionnaire_ids is not None:
            id = questionnaire_ids.pop(0)

        for j in range(questionnaire_length):
            if survey.defs.style == "classic":
                draw_corner_marks(survey, canvas)
                draw_corner_boxes(survey, canvas, j)
                if not survey.defs.duplex or j % 2:
                    if questionnaire_ids is not None:
                        draw_questionnaire_id(canvas, survey, id)

                    if survey.defs.print_survey_id:
                        draw_survey_id(canvas, survey)

            elif survey.defs.style == "code128":
                draw_corner_marks(survey, canvas)

                if not survey.defs.duplex or j % 2:
                    if questionnaire_ids is not None:
                        draw_code128_questionnaire_id(canvas, survey, id)

                    # Survey ID has to be printed in CODE128 mode, because it
                    # contains the page number and rotation.
                    draw_code128_sdaps_info(canvas, survey, j + 1)

                    if survey.global_id is not None:
                        draw_code128_global_id(canvas, survey)

            elif survey.defs.style == "qr":
                draw_corner_marks(survey, canvas)

                if not survey.defs.duplex or j % 2:
                    if questionnaire_ids is not None:
                        draw_qr_questionnaire_id(canvas, survey, id)

                    # Survey ID has to be printed in QR mode, because it
                    # contains the page number and rotation.
                    draw_qr_sdaps_info(canvas, survey, j + 1)

                    if survey.global_id is not None:
                        draw_qr_global_id(canvas, survey)

            elif survey.defs.style == "custom":
                # Only draw corner marker
                draw_corner_marks(survey, canvas)

                pass
            else:
                raise AssertionError()

            canvas.showPage()
        log.progressbar.update(i + 1)

    canvas.save()

    print ungettext(u'%i sheet; %f seconds per sheet', u'%i sheet; %f seconds per sheet', log.progressbar.max_value) % (
        log.progressbar.max_value,
        float(log.progressbar.elapsed_time) /
        float(log.progressbar.max_value)
    )

    if have_pdftk:
        stampsfile.close()
        # Merge using pdftk
        print _("Stamping using pdftk")
        tmp_dir = tempfile.mkdtemp()

        if sheets == 1:
            # Shortcut if we only have one sheet.
            # In this case form data in the PDF will *not* break, in
            # the other code path it *will* break.
            print _(u"pdftk: Overlaying the original PDF with the markings.")
            subprocess.call(['pdftk',
                             survey.path('questionnaire.pdf'),
                             'multistamp',
                             survey.path('tmp.pdf'),
                             'output',
                             output_filename])
        else:
            for page in xrange(1, questionnaire_length + 1):
                print ungettext(u"pdftk: Splitting out page %d of each sheet.", u"pdftk: Splitting out page %d of each sheet.", page) % page
                args = []
                args.append('pdftk')
                args.append(survey.path('tmp.pdf'))
                args.append('cat')
                cur = page
                for i in range(sheets):
                    args.append('%d' % cur)
                    cur += questionnaire_length
                args.append('output')
                args.append(os.path.join(tmp_dir, 'stamp-%d.pdf' % page))

                subprocess.call(args)

            print _(u"pdftk: Splitting the questionnaire for watermarking.")
            subprocess.call(['pdftk', survey.path('questionnaire.pdf'),
                             'dump_data', 'output',
                             os.path.join(tmp_dir, 'doc_data.txt')])
            for page in xrange(1, questionnaire_length + 1):
                subprocess.call(['pdftk', survey.path('questionnaire.pdf'), 'cat',
                                 '%d' % page, 'output',
                                 os.path.join(tmp_dir, 'watermark-%d.pdf' % page)])

            if sheets == 1:
                for page in xrange(1, questionnaire_length + 1):
                    print ungettext(u"pdftk: Watermarking page %d of all sheets.", u"pdftk: Watermarking page %d of all sheets.", page) % page
                    subprocess.call(['pdftk',
                                     os.path.join(tmp_dir, 'stamp-%d.pdf' % page),
                                     'background',
                                     os.path.join(tmp_dir, 'watermark-%d.pdf' % page),
                                     'output',
                                     os.path.join(tmp_dir, 'watermarked-%d.pdf' % page)])
            else:
                for page in xrange(1, questionnaire_length + 1):
                    print ungettext(u"pdftk: Watermarking page %d of all sheets.", u"pdftk: Watermarking page %d of all sheets.", page) % page
                    subprocess.call(['pdftk',
                                     os.path.join(tmp_dir, 'stamp-%d.pdf' % page),
                                     'background',
                                     os.path.join(tmp_dir, 'watermark-%d.pdf' % page),
                                     'output',
                                     os.path.join(tmp_dir, 'watermarked-%d.pdf' % page)])

            args = []
            args.append('pdftk')
            for page in xrange(1, questionnaire_length + 1):
                char = chr(ord('A') + page - 1)
                args.append('%s=' % char + os.path.join(tmp_dir, 'watermarked-%d.pdf' % page))

            args.append('cat')

            for i in range(sheets):
                for page in xrange(1, questionnaire_length + 1):
                    char = chr(ord('A') + page - 1)
                    args.append('%s%d' % (char, i + 1))

            args.append('output')
            args.append(os.path.join(tmp_dir, 'final.pdf'))
            print _(u"pdftk: Assembling everything into the final PDF.")
            subprocess.call(args)

            subprocess.call(['pdftk', os.path.join(tmp_dir, 'final.pdf'),
                             'update_info', os.path.join(tmp_dir, 'doc_data.txt'),
                             'output', output_filename])

        # Remove tmp.pdf
        os.unlink(survey.path('tmp.pdf'))
        # Remove all the temporary files
        shutil.rmtree(tmp_dir)

    else:
        # Merge using pyPdf
        stamped = pyPdf.PdfFileWriter()
        stamped._info.getObject().update({
            pyPdf.generic.NameObject('/Producer'): pyPdf.generic.createStringObject(u'sdaps'),
            pyPdf.generic.NameObject('/Title'): pyPdf.generic.createStringObject(survey.title),
        })

        subject = []
        for key, value in survey.info.iteritems():
            subject.append(u'%(key)s: %(value)s' % {'key': key, 'value': value})
        subject = u'\n'.join(subject)

        stamped._info.getObject().update({
            pyPdf.generic.NameObject('/Subject'): pyPdf.generic.createStringObject(subject),
        })

        stamps = pyPdf.PdfFileReader(stampsfile)

        del stampsfile

        questionnaire = pyPdf.PdfFileReader(
            file(survey.path('questionnaire.pdf'), 'rb')
        )

        print _(u'Stamping using pyPdf. For faster stamping, install pdftk.')
        log.progressbar.start(sheets)

        for i in range(sheets):
            for j in range(questionnaire_length):
                s = stamps.getPage(i * questionnaire_length + j)
                if not have_pdftk:
                    q = questionnaire.getPage(j)
                    s.mergePage(q)
                stamped.addPage(s)
            log.progressbar.update(i + 1)

        stamped.write(open(output_filename, 'wb'))

        print ungettext(u'%i sheet; %f seconds per sheet', u'%i sheet; %f seconds per sheet',
                        log.progressbar.max_value) % (
                            log.progressbar.max_value,
                            float(log.progressbar.elapsed_time) /
                            float(log.progressbar.max_value))

Example 29

Project: baruwa
Source File: sendpdfreports.py
View license
    def handle(self, *args, **options):
        if len(args) != 0:
            raise CommandError(_("Command doesn't accept any arguments"))

        by_domain = options.get('by_domain')
        domain_name = options.get('domain_name')
        copy_admin = options.get('copy_admin')
        period = options.get('period')
        include_daily = options.get('include_daily')
        startdate =  options.get('startdate')
        end_date =  options.get('enddate')
        enddate = None

        if startdate and end_date:
            if not checkdate(startdate) or not checkdate(end_date):
                raise CommandError(_("The startdate, enddate specified is invalid"))
            daterange = (startdate, end_date)
        else:
            daterange = None

        period_re = re.compile(r"(?P<num>(\d+))\s+(?P<period>(day|week|month))(?:s)?")
        if period:
            match = period_re.match(period)
            if not match:
                raise CommandError(_("The period you specified is invalid"))
            num = match.group('num')
            ptype = match.group('period')
            if not ptype.endswith('s'):
                ptype = ptype + 's'
            delta = datetime.timedelta(**{ptype: int(num)})
            enddate = datetime.date.today() - delta

        table_style = TableStyle([
            ('FONT', (0, 0), (-1, -1), 'Helvetica'),
            ('FONT', (0, 0), (-1, 0), 'Helvetica-Bold'),
            ('FONTSIZE', (0, 0), (-1, -1), 8),
            ('GRID', (0, 0), (-1, -1), 0.15, colors.black),
            ('ALIGN', (0, 0), (-1, 0), 'CENTER'),
            ('ALIGN', (4, 1), (-1, -1), 'CENTER'),
            ('ALIGN', (0, 0), (0, -1), 'CENTER'),
            ('VALIGN', (4, 1), (-1, -1), 'MIDDLE'),
            ('SPAN', (4, 1), (-1, -1)),
        ])

        styles = getSampleStyleSheet()

        reports = [
            [
                'from_address', {'from_address__exact': ""}, 'num_count',
                'Top senders by quantity'],
            [
                'from_address', {'from_address__exact': ""}, 'total_size',
                'Top senders by volume'],
            [
                'from_domain', {'from_domain__exact': ""}, 'num_count',
                'Top sender domains by quantity'],
            [
                'from_domain', {'from_domain__exact': ""}, 'total_size',
                'Top sender domains by volume'],
            [
                'to_address', {'to_address__exact': ""}, 'num_count',
                'Top recipients by quantity'],
            [
                'to_address', {'to_address__exact': ""}, 'total_size',
                'Top recipients by volume'],
            [
                'to_domain', {'to_domain__exact': "",
                'to_domain__isnull': False}, 'num_count',
                'Top recipient domains by quantity'],
            [
                'to_domain', {'to_domain__exact': "",
                'to_domain__isnull': False}, 'total_size',
                'Top recipient domains by volume'],
        ]

        emails = []
        admin_addrs = []
        if copy_admin:
            mails = User.objects.values('email').filter(is_superuser=True)
            admin_addrs = [mail['email'] for mail in mails]

        from_email = getattr(settings, 'DEFAULT_FROM_EMAIL',
            '[email protected]')
        url = getattr(settings, 'QUARANTINE_REPORT_HOSTURL', '')
        logo_dir = getattr(settings, 'MEDIA_ROOT', '')
        img = Image(logo_dir + '/imgs/css/logo.jpg')

        def build_chart(data, column, order, title):
            "build chart"
            headings = [('', _('Address'), _('Count'), _('Volume'), '')]
            rows = [[draw_square(PIE_CHART_COLORS[index]),
            tds_trunc(row[column], 45), row['num_count'],
            filesizeformat(row['total_size']), '']
            for index, row in enumerate(data)]

            if len(rows) != 10:
                missing = 10 - len(rows)
                add_rows = [
                    ('', '', '', '', '') for ind in range(missing)
                    ]
                rows.extend(add_rows)

            headings.extend(rows)
            dat = [row[order] for row in data]
            total = sum(dat)
            labels = [
                    ("%.1f%%" % ((1.0 * row[order] / total) * 100))
                    for row in data
                ]

            pie = PieChart()
            pie.chart.labels = labels
            pie.chart.data = dat
            headings[1][4] = pie

            table_with_style = Table(headings, [0.2 * inch,
                2.8 * inch, 0.5 * inch, 0.7 * inch, 3.2 * inch])
            table_with_style.setStyle(table_style)

            paragraph = Paragraph(title, styles['Heading1'])

            return [paragraph, table_with_style]

        def build_parts(account, enddate, isdom=None, daterange=None):
            "build parts"
            parts = []
            sentry = 0
            for report in reports:
                column = report[0]
                exclude_kwargs = report[1]
                order_by = "-%s" % report[2]
                order = report[2]
                title = report[3]

                if isdom:
                    #dom
                    data = Message.objects.values(column).\
                    filter(Q(from_domain=account.address) | \
                    Q(to_domain=account.address)).\
                    exclude(**exclude_kwargs).annotate(
                        num_count=Count(column), total_size=Sum('size')
                    ).order_by(order_by)
                    if daterange:
                        data.filter(date__range=(daterange[0], daterange[1]))
                    elif enddate:
                        data.filter(date__gt=enddate)
                    data = data[:10]
                else:
                    #all users
                    data = Message.report.all(user, enddate, daterange).values(
                            column).exclude(**exclude_kwargs).annotate(
                            num_count=Count(column), total_size=Sum('size')
                            ).order_by(order_by)
                    data = data[:10]

                if data:
                    sentry += 1
                    pgraphs = build_chart(data, column, order, title)
                    parts.extend(pgraphs)
                    parts.append(Spacer(1, 70))
                    if (sentry % 2) == 0:
                        parts.append(PageBreak())
            parts.append(Paragraph(_('Message Totals'), styles['Heading1']))
            if isdom:
                #doms
                msg_totals = MessageTotals.objects.doms(account.address, enddate)
            else:
                #norm
                filters = []
                addrs = [
                    addr.address for addr in UserAddresses.objects.filter(
                        user=account
                    ).exclude(enabled__exact=0)]
                if enddate:
                    efilter = {
                                'filter': 3,
                                'field': 'date',
                                'value': str(enddate)
                               }
                    filters.append(efilter)
                msg_totals = MessageTotals.objects.all(
                                account, filters, addrs,
                                profile.account_type,
                                daterange)

            mail_total = []
            spam_total = []
            virus_total = []
            dates = []
            if include_daily:
                rows = [(
                Table([[draw_square(colors.white),
                Paragraph('Date', styles["Heading6"])]],
                [0.35 * inch, 1.50 * inch, ]),
                Table([[draw_square(colors.green),
                Paragraph('Mail totals', styles["Heading6"])]],
                [0.35 * inch, 1.50 * inch, ]),
                Table([[draw_square(colors.pink),
                Paragraph('Spam totals', styles["Heading6"])]],
                [0.35 * inch, 1.50 * inch, ]),
                Table([[draw_square(colors.red),
                Paragraph('Virus totals', styles["Heading6"])]],
                [0.35 * inch, 1.50 * inch, ]),
                )]
            for ind, msgt in enumerate(msg_totals):
                if ind % 10:
                    dates.append('')
                else:
                    dates.append(str(msgt.date))

                mail_total.append(int(msgt.mail_total))
                spam_total.append(int(msgt.spam_total))
                virus_total.append(int(msgt.virus_total))
                if include_daily:
                    rows.append((str(msgt.date), msgt.mail_total,
                    msgt.spam_total, msgt.virus_total))

            graph = BarChart()
            graph.chart.data = [
                    tuple(mail_total), tuple(spam_total),
                    tuple(virus_total)
                ]
            graph.chart.categoryAxis.categoryNames = dates
            graph_table = Table([[graph]], [7.4 * inch])
            parts.append(graph_table)
            if include_daily:
                rows.append(('Totals', sum(mail_total), sum(spam_total),
                sum(virus_total)))
                parts.append(Spacer(1, 20))
                graph_table = Table(rows, [1.85 * inch, 1.85 * inch,
                1.85 * inch, 1.85 * inch, ])
                graph_table.setStyle(TableStyle([
                ('FONTSIZE', (0, 0), (-1, -1), 8),
                ('FONT', (0, 0), (-1, -1), 'Helvetica'),
                ('FONT', (0, 0), (-1, 0), 'Helvetica-Bold'),
                ('GRID', (0, 0), (-1, -1), 0.15, colors.black),
                ('FONT', (0, -1), (-1, -1), 'Helvetica-Bold'),
                #('BACKGROUND', (0, -1), (-1, -1), colors.green),
                ]))
                parts.append(graph_table)
            return parts

        def build_pdf(charts):
            "Build a PDF"
            pdf = StringIO()
            doc = SimpleDocTemplate(pdf, topMargin=50, bottomMargin=18)
            logo = [(img, _('Baruwa mail report'))]
            logo_table = Table(logo, [2.0 * inch, 5.4 * inch])
            logo_table.setStyle(TableStyle([
            ('FONT', (0, 0), (-1, 0), 'Helvetica-Bold'),
            ('ALIGN', (0, 0), (-1, 0), 'LEFT'),
            ('ALIGN', (1, 0), (-1, 0), 'RIGHT'),
            ('FONTSIZE', (1, 0), (-1, 0), 10),
            ('LINEBELOW', (0, 0), (-1, -1), 0.15, colors.black),
            ]))
            parts = [logo_table]
            parts.append(Spacer(1, 20))
            parts.extend(charts)
            try:
                doc.build(parts)
            except IndexError:
                pass
            return pdf

        def gen_email(pdf, user, owner):
            "generate and return email"
            text_content = render_to_string('reports/pdf_report.txt',
                {'user': user, 'url': url})
            subject = _('Baruwa usage report for: %(user)s') % {
                        'user': owner}
            if email_re.match(user.username):
                toaddr = user.username
            if email_re.match(user.email):
                toaddr = user.email

            if admin_addrs:
                msg = EmailMessage(subject, text_content, from_email, [toaddr], admin_addrs)
            else:
                msg = EmailMessage(subject, text_content, from_email, [toaddr])
            msg.attach('baruwa.pdf', pdf.getvalue(), "application/pdf")
            print _("* Queue %(user)s's report to: %(addr)s") % {
                'user': owner, 'addr': toaddr}
            pdf.close()
            return msg

        print _("=================== Processing reports ======================")
        if by_domain:
            #do domain query
            #print "camacamlilone"
            domains = UserAddresses.objects.filter(Q(enabled=1), Q(address_type=1))
            if domain_name != 'all':
                domains = domains.filter(address=domain_name)
                if not domains:
                    print _("========== domain name %(dom)s does not exist ==========") % {
                    'dom': domain_name
                    }
            for domain in domains:
                if email_re.match(domain.user.email):
                    parts = build_parts(domain, enddate, True, daterange)
                    if parts:
                        pdf = build_pdf(parts)
                        email = gen_email(pdf, domain.user, domain.address)
                        emails.append(email)
        else:
            #do normal query
            profiles = UserProfile.objects.filter(send_report=1)
            for profile in profiles:
                try:
                    user = profile.user
                    if email_re.match(user.email) or email_re.match(user.username):
                        parts = build_parts(user, enddate, False, daterange)
                        if parts:
                            pdf = build_pdf(parts)
                            email = gen_email(pdf, user, user.username)
                            emails.append(email)
                except User.DoesNotExist:
                    pass

        if emails:
            try:
                conn = SMTPConnection()
                conn.send_messages(emails)
                print _("====== sending %(num)s messages =======") % {
                        'num': str(len(emails))}
            except Exception, exception:
                print _("Sending failed ERROR: %(error)s") % {'error': str(exception)}

Example 30

Project: SickGear
Source File: mp4.py
View license
    def _readatom(self, file):
        s = file.read(8)
        if len(s) < 8:
            return 0

        atomsize, atomtype = struct.unpack('>I4s', s)
        if not str(atomtype).decode('latin1').isalnum():
            # stop at nonsense data
            return 0

        log.debug(u'%r [%X]' % (atomtype, atomsize))

        if atomtype == 'udta':
            # Userdata (Metadata)
            pos = 0
            tabl = {}
            i18ntabl = {}
            atomdata = file.read(atomsize - 8)
            while pos < atomsize - 12:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
                if ord(datatype[0]) == 169:
                    # i18n Metadata...
                    mypos = 8 + pos
                    while mypos + 4 < datasize + pos:
                        # first 4 Bytes are i18n header
                        (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos + 4])
                        i18ntabl[lang] = i18ntabl.get(lang, {})
                        l = atomdata[mypos + 4:mypos + tlen + 4]
                        i18ntabl[lang][datatype[1:]] = l
                        mypos += tlen + 4
                elif datatype == 'WLOC':
                    # Drop Window Location
                    pass
                else:
                    if ord(atomdata[pos + 8:pos + datasize][0]) > 1:
                        tabl[datatype] = atomdata[pos + 8:pos + datasize]
                pos += datasize
            if len(i18ntabl.keys()) > 0:
                for k in i18ntabl.keys():
                    if QTLANGUAGES.has_key(k) and QTLANGUAGES[k] == 'en':
                        self._appendtable('QTUDTA', i18ntabl[k])
                        self._appendtable('QTUDTA', tabl)
            else:
                log.debug(u'NO i18')
                self._appendtable('QTUDTA', tabl)

        elif atomtype == 'trak':
            atomdata = file.read(atomsize - 8)
            pos = 0
            trackinfo = {}
            tracktype = None
            while pos < atomsize - 8:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])

                if datatype == 'tkhd':
                    tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos + 8:pos + datasize])
                    trackinfo['width'] = tkhd[10] >> 16
                    trackinfo['height'] = tkhd[11] >> 16
                    trackinfo['id'] = tkhd[3]

                    try:
                        # XXX Timestamp of Seconds is since January 1st 1904!
                        # XXX 2082844800 is the difference between Unix and
                        # XXX Apple time. FIXME to work on Apple, too
                        self.timestamp = int(tkhd[1]) - 2082844800
                    except Exception, e:
                        log.exception(u'There was trouble extracting timestamp')

                elif datatype == 'mdia':
                    pos += 8
                    datasize -= 8
                    log.debug(u'--> mdia information')

                    while datasize:
                        mdia = struct.unpack('>I4s', atomdata[pos:pos + 8])
                        if mdia[1] == 'mdhd':
                            # Parse based on version of mdhd header.  See
                            # http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd
                            ver = ord(atomdata[pos + 8])
                            if ver == 0:
                                mdhd = struct.unpack('>IIIIIhh', atomdata[pos + 8:pos + 8 + 24])
                            elif ver == 1:
                                mdhd = struct.unpack('>IQQIQhh', atomdata[pos + 8:pos + 8 + 36])
                            else:
                                mdhd = None

                            if mdhd:
                                # duration / time scale
                                trackinfo['length'] = mdhd[4] / mdhd[3]
                                if mdhd[5] in QTLANGUAGES:
                                    trackinfo['language'] = QTLANGUAGES[mdhd[5]]
                                elif mdhd[5] == 0x7FF:
                                    trackinfo['language'] = 'und'
                                elif mdhd[5] >= 0x400:
                                    # language code detected as explained in:
                                    # https://developer.apple.com/library/mac/documentation/QuickTime/qtff/QTFFChap4/qtff4.html#//apple_ref/doc/uid/TP40000939-CH206-35103
                                    language = bytearray([ ((mdhd[5] & 0x7C00) >> 10) + 0x60, ((mdhd[5] & 0x3E0) >> 5) + 0x60, (mdhd[5] & 0x1F) + 0x60])
                                    trackinfo['language'] = str(language)
                                # mdhd[6] == quality
                                self.length = max(self.length, mdhd[4] / mdhd[3])
                        elif mdia[1] == 'minf':
                            # minf has only atoms inside
                            pos -= (mdia[0] - 8)
                            datasize += (mdia[0] - 8)
                        elif mdia[1] == 'stbl':
                            # stbl has only atoms inside
                            pos -= (mdia[0] - 8)
                            datasize += (mdia[0] - 8)
                        elif mdia[1] == 'hdlr':
                            hdlr = struct.unpack('>I4s4s', atomdata[pos + 8:pos + 8 + 12])
                            if hdlr[1] == 'mhlr' or hdlr[1] == '\0\0\0\0':
                                if hdlr[2] == 'vide':
                                    tracktype = 'video'
                                if hdlr[2] == 'soun':
                                    tracktype = 'audio'
                                if hdlr[2] == 'subt' or hdlr[2] == 'sbtl' or hdlr[2] == 'subp' or hdlr[2] == 'text':
                                    tracktype = 'subtitle'
                        elif mdia[1] == 'stsd':
                            stsd = struct.unpack('>2I', atomdata[pos + 8:pos + 8 + 8])
                            if stsd[1] > 0:
                                codec = atomdata[pos + 16:pos + 16 + 8]
                                codec = struct.unpack('>I4s', codec)
                                trackinfo['codec'] = codec[1]
                                if codec[1] == 'jpeg':
                                    tracktype = 'image'
                        elif mdia[1] == 'dinf':
                            dref = struct.unpack('>I4s', atomdata[pos + 8:pos + 8 + 8])
                            log.debug(u'  --> %r, %r (useless)' % mdia)
                            if dref[1] == 'dref':
                                num = struct.unpack('>I', atomdata[pos + 20:pos + 20 + 4])[0]
                                rpos = pos + 20 + 4
                                for ref in range(num):
                                    # FIXME: do somthing if this references
                                    ref = struct.unpack('>I3s', atomdata[rpos:rpos + 7])
                                    data = atomdata[rpos + 7:rpos + ref[0]]
                                    rpos += ref[0]
                        else:
                            if mdia[1].startswith('st'):
                                log.debug(u'  --> %r, %r (sample)' % mdia)
                            elif mdia[1] == 'vmhd' and not tracktype:
                                # indicates that this track is video
                                tracktype = 'video'
                            elif mdia[1] in ['vmhd', 'smhd'] and not tracktype:
                                # indicates that this track is audio
                                tracktype = 'audio'
                            else:
                                log.debug(u'  --> %r, %r (unknown)' % mdia)

                        pos += mdia[0]
                        datasize -= mdia[0]

                elif datatype == 'udta':
                    log.debug(u'udta: %r' % struct.unpack('>I4s', atomdata[:8]))
                else:
                    if datatype == 'edts':
                        log.debug(u'--> %r [%d] (edit list)' % \
                                  (datatype, datasize))
                    else:
                        log.debug(u'--> %r [%d] (unknown)' % \
                                  (datatype, datasize))
                pos += datasize

            info = None
            if tracktype == 'video':
                info = core.VideoStream()
                self.video.append(info)
            if tracktype == 'audio':
                info = core.AudioStream()
                self.audio.append(info)
            if tracktype == 'subtitle':
                info = core.Subtitle()
                self.subtitles.append(info)
            if info:
                for key, value in trackinfo.items():
                    setattr(info, key, value)

        elif atomtype == 'mvhd':
            # movie header
            mvhd = struct.unpack('>6I2h', file.read(28))
            self.length = max(self.length, mvhd[4] / mvhd[3])
            self.volume = mvhd[6]
            file.seek(atomsize - 8 - 28, 1)


        elif atomtype == 'cmov':
            # compressed movie
            datasize, atomtype = struct.unpack('>I4s', file.read(8))
            if not atomtype == 'dcom':
                return atomsize

            method = struct.unpack('>4s', file.read(datasize - 8))[0]

            datasize, atomtype = struct.unpack('>I4s', file.read(8))
            if not atomtype == 'cmvd':
                return atomsize

            if method == 'zlib':
                data = file.read(datasize - 8)
                try:
                    decompressed = zlib.decompress(data)
                except Exception, e:
                    try:
                        decompressed = zlib.decompress(data[4:])
                    except Exception, e:
                        log.exception(u'There was a proble decompressiong atom')
                        return atomsize

                decompressedIO = StringIO.StringIO(decompressed)
                while self._readatom(decompressedIO):
                    pass

            else:
                log.info(u'unknown compression %r' % method)
                # unknown compression method
                file.seek(datasize - 8, 1)

        elif atomtype == 'moov':
            # decompressed movie info
            while self._readatom(file):
                pass

        elif atomtype == 'mdat':
            pos = file.tell() + atomsize - 8
            # maybe there is data inside the mdat
            log.info(u'parsing mdat')
            while self._readatom(file):
                pass
            log.info(u'end of mdat')
            file.seek(pos, 0)


        elif atomtype == 'rmra':
            # reference list
            while self._readatom(file):
                pass

        elif atomtype == 'rmda':
            # reference
            atomdata = file.read(atomsize - 8)
            pos = 0
            url = ''
            quality = 0
            datarate = 0
            while pos < atomsize - 8:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
                if datatype == 'rdrf':
                    rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos + 8:pos + 20])
                    if rtype == 'url ':
                        url = atomdata[pos + 20:pos + 20 + rlen]
                        if url.find('\0') > 0:
                            url = url[:url.find('\0')]
                elif datatype == 'rmqu':
                    quality = struct.unpack('>I', atomdata[pos + 8:pos + 12])[0]

                elif datatype == 'rmdr':
                    datarate = struct.unpack('>I', atomdata[pos + 12:pos + 16])[0]

                pos += datasize
            if url:
                self._references.append((url, quality, datarate))

        else:
            if not atomtype in ['wide', 'free']:
                log.info(u'unhandled base atom %r' % atomtype)

            # Skip unknown atoms
            try:
                file.seek(atomsize - 8, 1)
            except IOError:
                return 0

        return atomsize

Example 31

Project: SiCKRAGE
Source File: mp4.py
View license
    def _readatom(self, file):
        s = file.read(8)
        if len(s) < 8:
            return 0

        atomsize, atomtype = struct.unpack('>I4s', s)
        if not str(atomtype).decode('latin1').isalnum():
            # stop at nonsense data
            return 0

        log.debug('%r [%X]' % (atomtype, atomsize))

        if atomtype == 'udta':
            # Userdata (Metadata)
            pos = 0
            tabl = {}
            i18ntabl = {}
            atomdata = file.read(atomsize - 8)
            while pos < atomsize - 12:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
                if ord(datatype[0]) == 169:
                    # i18n Metadata...
                    mypos = 8 + pos
                    while mypos + 4 < datasize + pos:
                        # first 4 Bytes are i18n header
                        (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos + 4])
                        i18ntabl[lang] = i18ntabl.get(lang, {})
                        l = atomdata[mypos + 4:mypos + tlen + 4]
                        i18ntabl[lang][datatype[1:]] = l
                        mypos += tlen + 4
                elif datatype == 'WLOC':
                    # Drop Window Location
                    pass
                else:
                    if ord(atomdata[pos + 8:pos + datasize][0]) > 1:
                        tabl[datatype] = atomdata[pos + 8:pos + datasize]
                pos += datasize
            if len(i18ntabl.keys()) > 0:
                for k in i18ntabl.keys():
                    if QTLANGUAGES.has_key(k) and QTLANGUAGES[k] == 'en':
                        self._appendtable('QTUDTA', i18ntabl[k])
                        self._appendtable('QTUDTA', tabl)
            else:
                log.debug('NO i18')
                self._appendtable('QTUDTA', tabl)

        elif atomtype == 'trak':
            atomdata = file.read(atomsize - 8)
            pos = 0
            trackinfo = {}
            tracktype = None
            while pos < atomsize - 8:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])

                if datatype == 'tkhd':
                    tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos + 8:pos + datasize])
                    trackinfo['width'] = tkhd[10] >> 16
                    trackinfo['height'] = tkhd[11] >> 16
                    trackinfo['id'] = tkhd[3]

                    try:
                        # XXX Timestamp of Seconds is since January 1st 1904!
                        # XXX 2082844800 is the difference between Unix and
                        # XXX Apple time. FIXME to work on Apple, too
                        self.timestamp = int(tkhd[1]) - 2082844800
                    except Exception, e:
                        log.exception('There was trouble extracting timestamp')

                elif datatype == 'mdia':
                    pos += 8
                    datasize -= 8
                    log.debug('--> mdia information')

                    while datasize:
                        mdia = struct.unpack('>I4s', atomdata[pos:pos + 8])

                        if mdia[0] == 0:
                            break

                        if mdia[1] == 'mdhd':
                            # Parse based on version of mdhd header.  See
                            # http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd
                            ver = ord(atomdata[pos + 8])
                            if ver == 0:
                                mdhd = struct.unpack('>IIIIIhh', atomdata[pos + 8:pos + 8 + 24])
                            elif ver == 1:
                                mdhd = struct.unpack('>IQQIQhh', atomdata[pos + 8:pos + 8 + 36])
                            else:
                                mdhd = None

                            if mdhd:
                                # duration / time scale
                                trackinfo['length'] = mdhd[4] / mdhd[3]
                                if mdhd[5] in QTLANGUAGES:
                                    trackinfo['language'] = QTLANGUAGES[mdhd[5]]
                                # mdhd[6] == quality
                                self.length = max(self.length, mdhd[4] / mdhd[3])
                        elif mdia[1] == 'minf':
                            # minf has only atoms inside
                            pos -= (mdia[0] - 8)
                            datasize += (mdia[0] - 8)
                        elif mdia[1] == 'stbl':
                            # stbl has only atoms inside
                            pos -= (mdia[0] - 8)
                            datasize += (mdia[0] - 8)
                        elif mdia[1] == 'hdlr':
                            hdlr = struct.unpack('>I4s4s', atomdata[pos + 8:pos + 8 + 12])
                            if hdlr[1] == 'mhlr':
                                if hdlr[2] == 'vide':
                                    tracktype = 'video'
                                if hdlr[2] == 'soun':
                                    tracktype = 'audio'
                        elif mdia[1] == 'stsd':
                            stsd = struct.unpack('>2I', atomdata[pos + 8:pos + 8 + 8])
                            if stsd[1] > 0:
                                codec = atomdata[pos + 16:pos + 16 + 8]
                                codec = struct.unpack('>I4s', codec)
                                trackinfo['codec'] = codec[1]
                                if codec[1] == 'jpeg':
                                    tracktype = 'image'
                        elif mdia[1] == 'dinf':
                            dref = struct.unpack('>I4s', atomdata[pos + 8:pos + 8 + 8])
                            log.debug('  --> %r, %r (useless)' % mdia)
                            if dref[1] == 'dref':
                                num = struct.unpack('>I', atomdata[pos + 20:pos + 20 + 4])[0]
                                rpos = pos + 20 + 4
                                for ref in range(num):
                                    # FIXME: do somthing if this references
                                    ref = struct.unpack('>I3s', atomdata[rpos:rpos + 7])
                                    data = atomdata[rpos + 7:rpos + ref[0]]
                                    rpos += ref[0]
                        else:
                            if mdia[1].startswith('st'):
                                log.debug('  --> %r, %r (sample)' % mdia)
                            elif mdia[1] == 'vmhd' and not tracktype:
                                # indicates that this track is video
                                tracktype = 'video'
                            elif mdia[1] in ['vmhd', 'smhd'] and not tracktype:
                                # indicates that this track is audio
                                tracktype = 'audio'
                            else:
                                log.debug('  --> %r, %r (unknown)' % mdia)

                        pos += mdia[0]
                        datasize -= mdia[0]

                elif datatype == 'udta':
                    log.debug('udta: %r' % struct.unpack('>I4s', atomdata[:8]))
                else:
                    if datatype == 'edts':
                        log.debug('--> %r [%d] (edit list)' % \
                                  (datatype, datasize))
                    else:
                        log.debug('--> %r [%d] (unknown)' % \
                                  (datatype, datasize))
                pos += datasize

            info = None
            if tracktype == 'video':
                info = core.VideoStream()
                self.video.append(info)
            if tracktype == 'audio':
                info = core.AudioStream()
                self.audio.append(info)
            if info:
                for key, value in trackinfo.items():
                    setattr(info, key, value)

        elif atomtype == 'mvhd':
            # movie header
            mvhd = struct.unpack('>6I2h', file.read(28))
            self.length = max(self.length, mvhd[4] / mvhd[3])
            self.volume = mvhd[6]
            file.seek(atomsize - 8 - 28, 1)


        elif atomtype == 'cmov':
            # compressed movie
            datasize, atomtype = struct.unpack('>I4s', file.read(8))
            if not atomtype == 'dcom':
                return atomsize

            method = struct.unpack('>4s', file.read(datasize - 8))[0]

            datasize, atomtype = struct.unpack('>I4s', file.read(8))
            if not atomtype == 'cmvd':
                return atomsize

            if method == 'zlib':
                data = file.read(datasize - 8)
                try:
                    decompressed = zlib.decompress(data)
                except Exception, e:
                    try:
                        decompressed = zlib.decompress(data[4:])
                    except Exception, e:
                        log.exception('There was a proble decompressiong atom')
                        return atomsize

                decompressedIO = StringIO.StringIO(decompressed)
                while self._readatom(decompressedIO):
                    pass

            else:
                log.info('unknown compression %r' % method)
                # unknown compression method
                file.seek(datasize - 8, 1)

        elif atomtype == 'moov':
            # decompressed movie info
            while self._readatom(file):
                pass

        elif atomtype == 'mdat':
            pos = file.tell() + atomsize - 8
            # maybe there is data inside the mdat
            log.info('parsing mdat')
            while self._readatom(file):
                pass
            log.info('end of mdat')
            file.seek(pos, 0)


        elif atomtype == 'rmra':
            # reference list
            while self._readatom(file):
                pass

        elif atomtype == 'rmda':
            # reference
            atomdata = file.read(atomsize - 8)
            pos = 0
            url = ''
            quality = 0
            datarate = 0
            while pos < atomsize - 8:
                (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
                if datatype == 'rdrf':
                    rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos + 8:pos + 20])
                    if rtype == 'url ':
                        url = atomdata[pos + 20:pos + 20 + rlen]
                        if url.find('\0') > 0:
                            url = url[:url.find('\0')]
                elif datatype == 'rmqu':
                    quality = struct.unpack('>I', atomdata[pos + 8:pos + 12])[0]

                elif datatype == 'rmdr':
                    datarate = struct.unpack('>I', atomdata[pos + 12:pos + 16])[0]

                pos += datasize
            if url:
                self._references.append((url, quality, datarate))

        else:
            if not atomtype in ['wide', 'free']:
                log.info('unhandled base atom %r' % atomtype)

            # Skip unknown atoms
            try:
                file.seek(atomsize - 8, 1)
            except IOError:
                return 0

        return atomsize

Example 32

View license
def bind_api(**config):

    class APIMethod(object):

        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        require_auth = config.get('require_auth', False)
        search_api = config.get('search_api', False)
        use_cache = config.get('use_cache', True)

        def __init__(self, api, args, kargs):
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

            self.api = api
            self.post_data = kargs.pop('post_data', None)
            self.retry_count = kargs.pop('retry_count', api.retry_count)
            self.retry_delay = kargs.pop('retry_delay', api.retry_delay)
            self.retry_errors = kargs.pop('retry_errors', api.retry_errors)
            self.headers = kargs.pop('headers', {})
            self.build_parameters(args, kargs)

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()

            if api.secure:
                self.scheme = 'https://'
            else:
                self.scheme = 'http://'

            if self.search_api:
                self.host = api.search_host
            else:
                self.host = api.host

            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
            # See Issue https://github.com/tweepy/tweepy/issues/12
            self.headers['Host'] = self.host

        def build_parameters(self, args, kargs):
            self.parameters = {}
            for idx, arg in enumerate(args):
                if arg is None:
                    continue

                try:
                    self.parameters[self.allowed_param[idx]] = convert_to_utf8_str(arg)
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

            for k, arg in kargs.items():
                if arg is None:
                    continue
                if k in self.parameters:
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

                self.parameters[k] = convert_to_utf8_str(arg)

        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

                if name == 'user' and 'user' not in self.parameters and self.api.auth:
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
                        value = urllib.quote(self.parameters[name])
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
                    del self.parameters[name]

                self.path = self.path.replace(variable, value)

        def execute(self):
            self.api.cached_result = False

            # Build the request URL
            url = self.api_root + self.path
            if len(self.parameters):
                url = '%s?%s' % (url, urllib.urlencode(self.parameters))

            # Query the cache if one is available
            # and this request uses a GET method.
            if self.use_cache and self.api.cache and self.method == 'GET':
                cache_result = self.api.cache.get(url)
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
                            if isinstance(result, Model):
                                result._api = self.api
                    else:
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
                    self.api.cached_result = True
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
                # Open connection
                if self.api.secure:
                    conn = httplib.HTTPSConnection(self.host, timeout=self.api.timeout)
                else:
                    conn = httplib.HTTPConnection(self.host, timeout=self.api.timeout)

                # Apply authentication
                if self.api.auth:
                    self.api.auth.apply_auth(
                            self.scheme + self.host + url,
                            self.method, self.headers, self.parameters
                    )

                # Request compression if configured
                if self.api.compression:
                    self.headers['Accept-encoding'] = 'gzip'

                # Execute request
                try:
                    conn.request(self.method, url, headers=self.headers, body=self.post_data)
                    resp = conn.getresponse()
                except Exception, e:
                    raise TweepError('Failed to send request: %s' % e)

                # Exit request loop if non-retry error code
                if self.retry_errors:
                    if resp.status not in self.retry_errors: break
                else:
                    if resp.status == 200: break

                # Sleep before retrying request again
                time.sleep(self.retry_delay)
                retries_performed += 1

            # If an error was returned, throw an exception
            self.api.last_response = resp
            if resp.status and not 200 <= resp.status < 300:
                try:
                    error_msg = self.api.parser.parse_error(resp.read())
                except Exception:
                    error_msg = "Twitter error response: status code = %s" % resp.status
                raise TweepError(error_msg, resp)

            # Parse the response payload
            body = resp.read()
            if resp.getheader('Content-Encoding', '') == 'gzip':
                try:
                    zipper = gzip.GzipFile(fileobj=StringIO(body))
                    body = zipper.read()
                except Exception, e:
                    raise TweepError('Failed to decompress data: %s' % e)
            result = self.api.parser.parse(self, body)

            conn.close()

            # Store result into cache if one is available.
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
                self.api.cache.store(url, result)

            return result


    def _call(api, *args, **kargs):

        method = APIMethod(api, args, kargs)
        return method.execute()


    # Set pagination mode
    if 'cursor' in APIMethod.allowed_param:
        _call.pagination_mode = 'cursor'
    elif 'max_id' in APIMethod.allowed_param and \
         'since_id' in APIMethod.allowed_param:
        _call.pagination_mode = 'id'
    elif 'page' in APIMethod.allowed_param:
        _call.pagination_mode = 'page'

    return _call

Example 33

Project: TACTIC
Source File: test_tools.py
View license
def setup_server():
    
    # Put check_access in a custom toolbox with its own namespace
    myauthtools = cherrypy._cptools.Toolbox("myauth")
    
    def check_access(default=False):
        if not getattr(cherrypy.request, "userid", default):
            raise cherrypy.HTTPError(401)
    myauthtools.check_access = cherrypy.Tool('before_request_body', check_access)
    
    def numerify():
        def number_it(body):
            for chunk in body:
                for k, v in cherrypy.request.numerify_map:
                    chunk = chunk.replace(k, v)
                yield chunk
        cherrypy.response.body = number_it(cherrypy.response.body)
    
    class NumTool(cherrypy.Tool):
        def _setup(self):
            def makemap():
                m = self._merged_args().get("map", {})
                cherrypy.request.numerify_map = m.items()
            cherrypy.request.hooks.attach('on_start_resource', makemap)
            
            def critical():
                cherrypy.request.error_response = cherrypy.HTTPError(502).set_response
            critical.failsafe = True
            
            cherrypy.request.hooks.attach('on_start_resource', critical)
            cherrypy.request.hooks.attach(self._point, self.callable)
    
    tools.numerify = NumTool('before_finalize', numerify)
    
    # It's not mandatory to inherit from cherrypy.Tool.
    class NadsatTool:
        
        def __init__(self):
            self.ended = {}
            self._name = "nadsat"
        
        def nadsat(self):
            def nadsat_it_up(body):
                for chunk in body:
                    chunk = chunk.replace("good", "horrorshow")
                    chunk = chunk.replace("piece", "lomtick")
                    yield chunk
            cherrypy.response.body = nadsat_it_up(cherrypy.response.body)
        nadsat.priority = 0
        
        def cleanup(self):
            # This runs after the request has been completely written out.
            cherrypy.response.body = "razdrez"
            id = cherrypy.request.params.get("id")
            if id:
                self.ended[id] = True
        cleanup.failsafe = True
        
        def _setup(self):
            cherrypy.request.hooks.attach('before_finalize', self.nadsat)
            cherrypy.request.hooks.attach('on_end_request', self.cleanup)
    tools.nadsat = NadsatTool()
    
    def pipe_body():
        cherrypy.request.process_request_body = False
        clen = int(cherrypy.request.headers['Content-Length'])
        cherrypy.request.body = cherrypy.request.rfile.read(clen)
    
    # Assert that we can use a callable object instead of a function.
    class Rotator(object):
        def __call__(self, scale):
            r = cherrypy.response
            r.collapse_body()
            r.body = [chr((ord(x) + scale) % 256) for x in r.body[0]]
    cherrypy.tools.rotator = cherrypy.Tool('before_finalize', Rotator())
    
    def stream_handler(next_handler, *args, **kwargs):
        cherrypy.response.output = o = StringIO()
        try:
            response = next_handler(*args, **kwargs)
            # Ignore the response and return our accumulated output instead.
            return o.getvalue()
        finally:
            o.close()
    cherrypy.tools.streamer = cherrypy._cptools.HandlerWrapperTool(stream_handler)
    
    class Root:
        def index(self):
            return "Howdy earth!"
        index.exposed = True
        
        def tarfile(self):
            cherrypy.response.output.write('I am ')
            cherrypy.response.output.write('a tarfile')
        tarfile.exposed = True
        tarfile._cp_config = {'tools.streamer.on': True}
        
        def euro(self):
            hooks = list(cherrypy.request.hooks['before_finalize'])
            hooks.sort()
            cbnames = [x.callback.__name__ for x in hooks]
            assert cbnames == ['gzip'], cbnames
            priorities = [x.priority for x in hooks]
            assert priorities == [80], priorities
            yield u"Hello,"
            yield u"world"
            yield europoundUnicode
        euro.exposed = True
        
        # Bare hooks
        def pipe(self):
            return cherrypy.request.body
        pipe.exposed = True
        pipe._cp_config = {'hooks.before_request_body': pipe_body}
        
        # Multiple decorators; include kwargs just for fun.
        # Note that rotator must run before gzip.
        def decorated_euro(self, *vpath):
            yield u"Hello,"
            yield u"world"
            yield europoundUnicode
        decorated_euro.exposed = True
        decorated_euro = tools.gzip(compress_level=6)(decorated_euro)
        decorated_euro = tools.rotator(scale=3)(decorated_euro)
    
    root = Root()
    
    
    class TestType(type):
        """Metaclass which automatically exposes all functions in each subclass,
        and adds an instance of the subclass as an attribute of root.
        """
        def __init__(cls, name, bases, dct):
            type.__init__(cls, name, bases, dct)
            for value in dct.itervalues():
                if isinstance(value, types.FunctionType):
                    value.exposed = True
            setattr(root, name.lower(), cls())
    class Test(object):
        __metaclass__ = TestType
    
    
    # METHOD ONE:
    # Declare Tools in _cp_config
    class Demo(Test):
        
        _cp_config = {"tools.nadsat.on": True}
        
        def index(self, id=None):
            return "A good piece of cherry pie"
        
        def ended(self, id):
            return repr(tools.nadsat.ended[id])
        
        def err(self, id=None):
            raise ValueError()
        
        def errinstream(self, id=None):
            yield "nonconfidential"
            raise ValueError()
            yield "confidential"
        
        # METHOD TWO: decorator using Tool()
        # We support Python 2.3, but the @-deco syntax would look like this:
        # @tools.check_access()
        def restricted(self):
            return "Welcome!"
        restricted = myauthtools.check_access()(restricted)
        userid = restricted
        
        def err_in_onstart(self):
            return "success!"
        
        def stream(self, id=None):
            for x in xrange(100000000):
                yield str(x)
        stream._cp_config = {'response.stream': True}
    
    
    conf = {
        # METHOD THREE:
        # Declare Tools in detached config
        '/demo': {
            'tools.numerify.on': True,
            'tools.numerify.map': {"pie": "3.14159"},
        },
        '/demo/restricted': {
            'request.show_tracebacks': False,
        },
        '/demo/userid': {
            'request.show_tracebacks': False,
            'myauth.check_access.default': True,
        },
        '/demo/errinstream': {
            'response.stream': True,
        },
        '/demo/err_in_onstart': {
            # Because this isn't a dict, on_start_resource will error.
            'tools.numerify.map': "pie->3.14159"
        },
        # Combined tools
        '/euro': {
            'tools.gzip.on': True,
            'tools.encode.on': True,
        },
        # Priority specified in config
        '/decorated_euro/subpath': {
            'tools.gzip.priority': 10,
        },
        # Handler wrappers
        '/tarfile': {'tools.streamer.on': True}
    }
    app = cherrypy.tree.mount(root, config=conf)
    app.request_class.namespaces['myauth'] = myauthtools
    
    if sys.version_info >= (2, 5):
        from cherrypy.test import py25
        root.tooldecs = py25.ToolExamples()

Example 34

Project: pyNastran
Source File: test_shell.py
View license
    def test_pcomp_01(self):
        """
        asymmetrical, nsm=0.0 and nsm=1.0
        """
        #self.pid = data[0]
        #self.z0 = data[1]
        #self.nsm = data[2]
        #self.sb = data[3]
        #self.ft = data[4]
        #self.TRef = data[5]
        #self.ge = data[6]
        #self.lam = data[7]
        #Mid = data[8]
        #T = data[9]
        #Theta = data[10]
        #Sout = data[11]

        pid = 1
        z0 = 0.
        nsm = 0.
        sb = 0.
        ft = 'HILL'
        TRef = 0.
        ge = 0.
        #lam = 'NO'  # isSymmetrical YES/NO
        lam = 'BEND'  # isSymmetrical YES/NO
        Mid = [1, 2, 3]
        Theta = [0., 10., 20.]
        T = [.1, .2, .3]
        Sout = ['YES', 'YES', 'NO']  # 0-NO, 1-YES
        card_lines = [
            'PCOMP', pid, z0, nsm, sb, ft, TRef, ge, lam,
            Mid[0], T[0], Theta[0], Sout[0],
            Mid[1], T[1], Theta[1], Sout[1],
            Mid[2], T[2], Theta[2], Sout[2],
        ]
        model = BDF(debug=False)
        card_count = {
            'PCOMP' : 1,
            'MAT1' : 3,
        }
        model.allocate(card_count)

        model.add_card(card_lines, 'PCOMP', comment='', is_list=True)


        # material...
        mid = 1
        E = 1e7
        G = None
        nu = None
        rho = 1.0
        a = None
        St = None
        Sc = None
        Ss = None
        Mcsid = None
        mat1_a = ['MAT1', mid, E, G, nu, rho, a, TRef, ge, St, Sc, Ss, Mcsid]
        mat1_b = ['MAT1', mid + 1, E, G, nu, rho, a, TRef, ge, St, Sc, Ss, Mcsid]
        mat1_c = ['MAT1', mid + 2, E, G, nu, rho, a, TRef, ge, St, Sc, Ss, Mcsid]
        model.add_card(mat1_a, 'MAT1', comment='', is_list=True)
        model.add_card(mat1_b, 'MAT1', comment='', is_list=True)
        model.add_card(mat1_c, 'MAT1', comment='', is_list=True)
        #card = BDFCard(mat1)
        #m = MAT1(model)
        #m.allocate(1)
        #m.add(card)
        #m.build()

        model.build()
        #card = BDFCard(data)
        #p = PCOMP(model)
        #p = model.properties.pcomp
        #p.add(card)
        #p.build()
        p = model.properties_shell.pcomp
        m = model.materials.mat1
        #self.assertFalse(p.is_symmetrical())
        self.assertEqual(p.get_nplies_by_property_id(), 3)

        self.assertAlmostEqual(p.get_thickness_by_property_id(pid), 0.6)
        self.assertAlmostEqual(p.get_thickness_by_property_id_ply(pid, 0), 0.1)
        self.assertAlmostEqual(p.get_thickness_by_property_id_ply(pid, 1), 0.2)
        self.assertAlmostEqual(p.get_thickness_by_property_id_ply(pid, 2), 0.3)
        with self.assertRaises(IndexError):
            p.get_thickness_by_property_id_ply(pid, 3)

        self.assertAlmostEqual(p.get_theta_by_property_id_ply(pid, 0), 0.)
        self.assertAlmostEqual(p.get_theta_by_property_id_ply(pid, 1), 10.)
        self.assertAlmostEqual(p.get_theta_by_property_id_ply(pid, 2), 20.)
        with self.assertRaises(IndexError):
            p.get_theta_by_property_id_ply(pid, 3)

        self.assertEqual(p.get_material_id_by_property_id_ply(pid, 0), 1)
        self.assertEqual(p.get_material_id_by_property_id_ply(pid, 1), 2)
        self.assertEqual(p.get_material_id_by_property_id_ply(pid, 2), 3)
        with self.assertRaises(IndexError):
            p.get_material_id_by_property_id_ply(pid, 3)

        #print('get_material_id_by_property_id = ', p.get_material_id_by_property_id(pid))
        self.assertEqual(p.get_material_ids_by_property_id(pid)[0], 1)
        self.assertEqual(p.get_material_ids_by_property_id(pid)[0], 1)
        self.assertEqual(p.get_material_ids_by_property_id(pid)[1], 2)
        self.assertEqual(p.get_material_ids_by_property_id(pid)[2], 3)

        self.assertEqual(p.get_sout_by_property_id_ply(pid, 0), 'YES')
        self.assertEqual(p.get_sout_by_property_id_ply(pid, 1), 'YES')
        self.assertEqual(p.get_sout_by_property_id_ply(pid, 2), 'NO')
        with self.assertRaises(IndexError):
            p.get_sout_by_property_id_ply(pid, 3)

        #for iply in range(len(p.plies)):
            #mid = p.plies[iply][0]
            #p.plies[iply][0] = m # MAT1
            ##p.mids = [m, m, m]

        from StringIO import StringIO
        f = StringIO()

        #print(m.write_card(f, size=8, material_id=None))
        p.write_card(f)
        m.write_card(f)
        print(f.getvalue())

        #Mid
        self.assertAlmostEqual(p.get_material_id_by_property_id_ply(pid, 0), 1)
        self.assertAlmostEqual(p.get_material_id_by_property_id_ply(pid, 1), 2)
        self.assertAlmostEqual(p.get_material_id_by_property_id_ply(pid, 2), 3)
        with self.assertRaises(IndexError):
            p.get_density_by_property_id_ply(pid, 3)

        #Rho
        self.assertAlmostEqual(p.get_density_by_property_id_ply(pid, 0), 1.0)
        self.assertAlmostEqual(p.get_density_by_property_id_ply(pid, 1), 1.0)
        self.assertAlmostEqual(p.get_density_by_property_id_ply(pid, 2), 1.0)
        with self.assertRaises(IndexError):
            p.get_density_by_property_id_ply(pid, 3)

        # MassPerArea
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id(), 0.6)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id(pid), 0.6)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id([pid]), 0.6)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 0), 0.1)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 1), 0.2)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 2), 0.3)
        #with self.assertRaises(IndexError):
            #p.MassPerArea(3)

        #----------------------
        # change the nsm to 1.0
        p.set_nonstructural_mass_by_property_id(pid, 1.0)

        self.assertEqual(p.get_nonstructural_mass_by_property_id(), 1.0)
        # MassPerArea
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id(), 1.6)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 0, method='nplies'), 0.1+1/3.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 1, method='nplies'), 0.2+1/3.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 2, method='nplies'), 0.3+1/3.)

        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 0, method='t'), 0.1+1/6.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 1, method='t'), 0.2+2/6.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 2, method='t'), 0.3+3/6.)

        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 0, method='rho*t'), 0.1+1/6.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 1, method='rho*t'), 0.2+2/6.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 2, method='rho*t'), 0.3+3/6.)

        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 0, method='t'), 0.1+1/6.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 1, method='t'), 0.2+2/6.)
        self.assertAlmostEqual(p.get_mass_per_area_by_property_id_ply(pid, 2, method='t'), 0.3+3/6.)
        with self.assertRaises(IndexError):
            p.get_mass_per_area_by_property_id_ply(pid, 3, method='nplies')

        z0 = p.get_z0_by_property_id(pid)

        z = p.get_z_locations_by_property_id(pid)
        z_expected = array([0., T[0], T[0]+T[1], T[0]+T[1]+T[2]])
        for za, ze in zip(z, z_expected):
            self.assertAlmostEqual(za, ze)

        #z0  =
        p.z0[0] = 1.0
        z_expected = 1.0 + z_expected
        z = p.get_z_locations_by_property_id(pid)
        for za, ze in zip(z, z_expected):
            self.assertAlmostEqual(za, ze)

Example 35

View license
@ModuleInfo.plugin('wb.util.exportSQLite',
                   caption='Export SQLite CREATE script',
                   input=[wbinputs.currentCatalog()],
                   groups=['Catalog/Utilities', 'Menu/Catalog'])
@ModuleInfo.export(grt.INT, grt.classes.db_Catalog)
def exportSQLite(cat):
    """Function to go through all schemata in catalog and rename all FKs
    of table-objects
    """

    def validate_for_sqlite_export(cat):
        """Check uniqueness of schema, table and index names. Return 0 on
        success otherwise return 1 (the export process should abort)
        """

        have_errors = False
        idt = {}
        for i, schema in enumerate(cat.schemata):
            if schema.name in idt:
                have_errors = True
                if Workbench.confirm('Name conflict',
                        'Schemas %d and %d have the same name "%s".'
                        ' Please rename one of them.\n'
                        'Search for more such errors?' % (
                            idt[schema.name], i, schema.name)) == 0:
                    return False
            else:
                idt[schema.name] = i

        # Do not continue looking for errors on schema name error
        if have_errors:
            return False

        for schema in cat.schemata:
            idt = {}
            for i, tbl in enumerate(schema.tables):
                if tbl.name == '':
                    have_errors = True
                    if Workbench.confirm('Name conflict',
                            'Table %d in schema "%s". has no name.'
                            ' Please rename.\n'
                            'Search for more such errors?' % (
                                i, schema.name)) == 0:
                        return False
                if tbl.name in idt:
                    have_errors = True
                    if Workbench.confirm('Name conflict',
                            'Tables %d and %d in schema "%s"'
                            ' have the same name "%s".'
                            ' Please rename one of them.\n'
                            'Search for more such errors?' % (
                                idt[tbl.name], i, schema.name, tbl.name)) == 0:
                        return False
                else:
                    idt[tbl.name] = i

        if have_errors:
            return False

        for schema in cat.schemata:
            for tbl in schema.tables:
                idt = {}
                for i, column in enumerate(tbl.columns):
                    if column.name == '':
                        have_errors = True
                        if Workbench.confirm('Name conflict',
                                'Column %d in table "%s"."%s". has no name.'
                                ' Please rename.\n'
                                'Search for more such errors?' % (
                                    i, schema.name, tbl.name)) == 0:
                            return False
                    if column.name in idt:
                        have_errors = True
                        if Workbench.confirm('Name conflict',
                                'Columns %d and %d in table "%s"."%s"'
                                ' have the same name "%s".'
                                ' Please rename one of them.\n'
                                'Search for more such errors?' % (
                                    idt[column.name],
                                    i,
                                    schema.name,
                                    tbl.name,
                                    column.name)) == 0:
                            return False
                    else:
                        idt[column.name] = i

                # Now check indices (except primary/unique)
                idt = {}
                for i, index in enumerate(tbl.indices):
                    if index.indexType == 'INDEX':
                        if index.name == '':
                            have_errors = True
                            if Workbench.confirm('Name conflict',
                                    'Index %d in table "%s"."%s". has no name.'
                                    ' Please rename.\n'
                                    'Search for more such errors?' % (
                                        i, schema.name, tbl.name)) == 0:
                                return False
                        if index.name in idt:
                            have_errors = True
                            if Workbench.confirm('Name conflict',
                                    'Indices %d and %d in table "%s"."%s"'
                                    ' have the same name "%s".'
                                    ' Please rename one of them.\n'
                                    'Search for more such errors?' % (
                                        idt[index.name],
                                        i,
                                        schema.name,
                                        tbl.name,
                                        column.name)) == 0:
                                return False
                        else:
                            idt[index.name] = i

        if have_errors:
            return False

        return True

    def is_deferred(fkey):
        # Hack: if comment starts with "Defer..." we make it a deferred FK could
        # use member 'deferability' (WB has it), but there is no GUI for it
        return fkey.comment.lstrip().lower()[0:5] == 'defer'

    def export_table(out, db_name, schema, tbl):
        if len(tbl.columns) == 0:
            return

        out.write('CREATE TABLE %s%s(\n%s' % (
                  db_name, dq(tbl.name), schema_comment_format(tbl.comment)))

        primary_key = [i for i in tbl.indices if i.isPrimary == 1]
        primary_key = primary_key[0] if len(primary_key) > 0 else None

        pk_column = None
        if primary_key and len(primary_key.columns) == 1:
            pk_column = primary_key.columns[0].referencedColumn

        col_comment = ''
        for i, column in enumerate(tbl.columns):
            check, sqlite_type, flags = '', None, None
            if column.simpleType:
                sqlite_type = column.simpleType.name
                flags = column.simpleType.flags
            else:
                sqlite_type = column.userType.name
                flags = column.flags
            length = column.length
            # For INTEGER PRIMARY KEY column to become an alias for the rowid
            # the type needs to be "INTEGER" not "INT"
            # we fix it for other columns as well
            if 'INT' in sqlite_type or sqlite_type == 'LONG':
                sqlite_type = 'INTEGER'
                length = -1
                # Check flags for "unsigned"
                if 'UNSIGNED' in column.flags:
                    check = dq(column.name) + '>=0'
            # We even implement ENUM (because we can)
            if sqlite_type == 'ENUM':
                sqlite_type = 'TEXT'
                if column.datatypeExplicitParams:
                    check = (dq(column.name) + ' IN' +
                             column.datatypeExplicitParams)
            if i > 0:
                out.write(',' + comment_format(col_comment) + '\n')
            out.write('  ' + dq(column.name))
            # Type is optional in SQLite
            if sqlite_type != '':
                out.write(' ' + sqlite_type)
            # For [VAR]CHAR and such types specify length even though this is
            # not used in SQLite
            if length > 0:
                out.write('(%d)' % length)

            # Must specify single-column PKs as column-constraints for AI/rowid
            # behaviour
            if column == pk_column:
                out.write(' PRIMARY KEY')
                if primary_key.columns[0].descend == 1:
                    out.write(' DESC')
                # Only PK columns can be AI in SQLite
                if column.autoIncrement == 1:
                    out.write(' AUTOINCREMENT')
            # Check for NotNull
            if column.isNotNull == 1:
                out.write(' NOT NULL')

            if check != '':
                out.write(' CHECK(' + check + ')')

            if column.defaultValue != '':
                out.write(' DEFAULT ' + column.defaultValue)

            col_comment = column.comment

        # For multicolumn PKs
        if primary_key and not pk_column:
            out.write(',%s\n  PRIMARY KEY(%s)' % (
                      comment_format(col_comment),
                      print_index_columns(primary_key)))
            col_comment = ''

        # Put non-primary, UNIQUE Keys in CREATE TABLE as well (because we can)
        for index in tbl.indices:
            if index != primary_key and index.indexType == 'UNIQUE':
                out.write(',%s\n' % comment_format(col_comment))
                col_comment = ''
                if index.name != '':
                    out.write('  CONSTRAINT %s\n  ' % dq(index.name))
                out.write('  UNIQUE(%s)' % print_index_columns(index))

        for fkey in tbl.foreignKeys:
            have_fkeys = 1
            out.write(',%s\n' % comment_format(col_comment))
            col_comment = ''
            if fkey.name != '':
                out.write('  CONSTRAINT %s\n  ' % dq(fkey.name))
            out.write('  FOREIGN KEY(%s)\n' % print_fk_columns(fkey.columns))
            out.write('    REFERENCES %s(%s)' % (
                      dq(fkey.referencedTable.name),
                      print_fk_columns(fkey.referencedColumns)))
            if fkey.deleteRule in ['RESTRICT', 'CASCADE', 'SET NULL']:
                out.write('\n    ON DELETE ' + fkey.deleteRule)
            if fkey.updateRule in ['RESTRICT', 'CASCADE', 'SET NULL']:
                out.write('\n    ON UPDATE ' + fkey.updateRule)
            if is_deferred(fkey):
                out.write(' DEFERRABLE INITIALLY DEFERRED')

        out.write(comment_format(col_comment) + '\n);\n')

        # CREATE INDEX statements for all non-primary, non-unique, non-foreign
        # indexes
        for i, index in enumerate(tbl.indices):
            if index.indexType == 'INDEX':
                index_name = tbl.name + '.' + index.name
                if index.name == '':
                    index_name = tbl.name + '.index' + i
                out.write('CREATE INDEX %s%s ON %s (%s);\n' % (
                          db_name,
                          dq(index_name),
                          dq(tbl.name),
                          print_index_columns(index)))

        # Write the INSERTS (currently always)
        for insert in tbl.inserts().splitlines():
            columns_values = ''
            insert_start = 'insert into `%s`.`%s` (' % (schema.name, tbl.name)
            if insert[0:len(insert_start)].lower() == insert_start.lower():
                columns_values = insert[len(insert_start):]
            else:
                raise ExportSQLiteError(
                        'Error', 'Unrecognized command in insert')
            last_column = 0
            for i, column in enumerate(tbl.columns):
                column_name = '`' + column.name + '`'
                if columns_values[0:len(column_name)] == column_name:
                    columns_values = columns_values[len(column_name):]
                    if columns_values[0:1] == ')':
                        columns_values = columns_values[1:]
                        last_column = i
                        break
                    else:
                        if columns_values[0:2] == ', ':
                            columns_values = columns_values[2:]
                        else:
                            raise ExportSQLiteError(
                                    'Error',
                                    'Unrecognized character in column list')
                else:
                    raise ExportSQLiteError(
                            'Error', 'Unrecognized column in inserts')

            out.write('INSERT INTO %s(' % dq(tbl.name))
            for i in range(last_column + 1):
                if i > 0:
                    out.write(',')
                out.write(dq(tbl.columns[i].name))

            if columns_values[0:9].lower() != ' values (':
                raise ExportSQLiteError(
                        'Error', 'Unrecognized SQL in insert')
            columns_values = columns_values[9:]

            out.write(') VALUES(')
            out.write(columns_values.replace("\\'", "''"))
            out.write('\n')

    def order_tables(out, db_name, schema, unordered, respect_deferredness):
        have_ordered = False
        while not have_ordered:
            if len(unordered) == 0:
                have_ordered = True
            for tbl in unordered.values():
                has_forward_reference = False
                for fkey in tbl.foreignKeys:
                    if (fkey.referencedTable.name in unordered and
                            fkey.referencedTable.name != tbl.name and not (
                                respect_deferredness and is_deferred(fkey))):
                        has_forward_reference = True
                        break
                if not has_forward_reference:
                    export_table(out, db_name, schema, tbl)
                    del unordered[tbl.name]
                    have_ordered = True

    def export_schema(out, schema, is_main_schema):
        if len(schema.tables) == 0:
            return

        out.write('\n-- Schema: %s\n' % schema.name)
        out.write(schema_comment_format(schema.comment))

        db_name = ''
        if not is_main_schema:
            db_name = dq(schema.name) + '.'
            out.write('ATTACH "%s" AS %s;\n' % (
                    safe_file_name(schema.name + '.sdb'),
                    dq(schema.name)))
        out.write('BEGIN;\n')

        # Find a valid table order for inserts from FK constraints
        unordered = {t.name: t for t in schema.tables}

        # Try treating deferred keys like non-deferred keys first for ordering
        order_tables(out, db_name, schema, unordered, False)
        # Now try harder (leave out deferred keys from determining an order)
        order_tables(out, db_name, schema, unordered, True)

        # Loop through all remaining tables, if any. Have circular FK refs.
        # How to handle?
        for tbl in unordered.values():
            export_table(out, db_name, schema, tbl)

        out.write('COMMIT;\n')

    def print_index_columns(index):
        s = ''
        for i, column in enumerate(index.columns):
            if i > 0:
                s += ','
            s += dq(column.referencedColumn.name)
            if column.descend == 1:
                s += ' DESC'
        return s

    def print_fk_columns(columns):
        s = ''
        for i, column in enumerate(columns):
            if i > 0:
                s += ','
            s += dq(column.name)
        return s

    def dq(ident):
        """Double quote identifer, replacing " by "" """
        return '"' + re.sub(r'"', '""', ident) + '"'

    def safe_file_name(ident):
        """Create safe filename from identifer"""

        def repl(c):
            return ["%%%02x" % c for c in bytearray(c, 'ascii')]

        return re.sub(r'[/\:*?"<>|%]', repl, ident)

    def info_format(header, body):
        """Format a info field as SQL comment"""
        body = body.strip()
        if body == '':
            return ''
        elif '\n' in body:
            # Multiline comment
            return '-- %s:\n--   %s\n' % (
                        header, re.sub(r'\n', '\n--   ', body))
        else:
            # Single line
            return '-- %-14s %s\n' % (header + ':', body)

    def schema_comment_format(body):
        """Format a schema or table comment as SQL comment
        table comments to be stored in SQLite schema
        """
        body = body.strip()
        if body == '':
            return ''
        else:
            # Multiline comment
            return '--   %s\n' % re.sub(r'\n', '\n--   ', body)

    def comment_format(body):
        body = body.strip()
        if body == '':
            return ''
        elif '\n' in body:
            # Multiline comment
            return '\n--   %s' % re.sub(r'\n', '\n--   ', body)
        else:
            # Single line
            return '-- %s' % body

    if not validate_for_sqlite_export(cat):
        return 1

    out = StringIO.StringIO()
    out.write(info_format(
                'Creator',
                'MySQL Workbench %d.%d.%d/ExportSQLite Plugin %s\n' % (
                    grt.root.wb.info.version.majorNumber,
                    grt.root.wb.info.version.minorNumber,
                    grt.root.wb.info.version.releaseNumber,
                    ModuleInfo.version)))
    out.write(info_format('Author', grt.root.wb.doc.info.author))
    out.write(info_format('Caption', grt.root.wb.doc.info.caption))
    out.write(info_format('Project', grt.root.wb.doc.info.project))
    out.write(info_format('Changed', grt.root.wb.doc.info.dateChanged))
    out.write(info_format('Created', grt.root.wb.doc.info.dateCreated))
    out.write(info_format('Description', grt.root.wb.doc.info.description))

    out.write('PRAGMA foreign_keys = OFF;\n')

    # Loop over all catalogs in schema, find main schema main schema is first
    # nonempty schema or nonempty schema named "main"
    try:
        for schema in [(s, s.name == 'main') for s in cat.schemata]:
            export_schema(out, schema[0], schema[1])
    except ExportSQLiteError as e:
        Workbench.confirm(e.typ, e.message)
        return 1

    sql_text = out.getvalue()
    out.close()

    wizard = ExportSQLiteWizard(sql_text)
    wizard.run()

    return 0

Example 36

Project: hugula
Source File: PdfImagePlugin.py
View license
def _save(im, fp, filename):

    #
    # make sure image data is available
    im.load()

    xref = [0]*(5+1) # placeholders

    fp.write("%PDF-1.2\n")
    fp.write("% created by PIL PDF driver " + __version__ + "\n")

    #
    # Get image characteristics

    width, height = im.size

    # FIXME: Should replace ASCIIHexDecode with RunLengthDecode (packbits)
    # or LZWDecode (tiff/lzw compression).  Note that PDF 1.2 also supports
    # Flatedecode (zip compression).

    bits = 8
    params = None

    if im.mode == "1":
        filter = "/ASCIIHexDecode"
        colorspace = "/DeviceGray"
        procset = "/ImageB" # grayscale
        bits = 1
    elif im.mode == "L":
        filter = "/DCTDecode"
        # params = "<< /Predictor 15 /Columns %d >>" % (width-2)
        colorspace = "/DeviceGray"
        procset = "/ImageB" # grayscale
    elif im.mode == "P":
        filter = "/ASCIIHexDecode"
        colorspace = "[ /Indexed /DeviceRGB 255 <"
        palette = im.im.getpalette("RGB")
        for i in range(256):
            r = ord(palette[i*3])
            g = ord(palette[i*3+1])
            b = ord(palette[i*3+2])
            colorspace = colorspace + "%02x%02x%02x " % (r, g, b)
        colorspace = colorspace + "> ]"
        procset = "/ImageI" # indexed color
    elif im.mode == "RGB":
        filter = "/DCTDecode"
        colorspace = "/DeviceRGB"
        procset = "/ImageC" # color images
    elif im.mode == "CMYK":
        filter = "/DCTDecode"
        colorspace = "/DeviceCMYK"
        procset = "/ImageC" # color images
    else:
        raise ValueError("cannot save mode %s" % im.mode)

    #
    # catalogue

    xref[1] = fp.tell()
    _obj(fp, 1, Type = "/Catalog",
                Pages = "2 0 R")
    _endobj(fp)

    #
    # pages

    xref[2] = fp.tell()
    _obj(fp, 2, Type = "/Pages",
                Count = 1,
                Kids = "[4 0 R]")
    _endobj(fp)

    #
    # image

    op = StringIO.StringIO()

    if filter == "/ASCIIHexDecode":
        if bits == 1:
            # FIXME: the hex encoder doesn't support packed 1-bit
            # images; do things the hard way...
            data = im.tostring("raw", "1")
            im = Image.new("L", (len(data), 1), None)
            im.putdata(data)
        ImageFile._save(im, op, [("hex", (0,0)+im.size, 0, im.mode)])
    elif filter == "/DCTDecode":
        ImageFile._save(im, op, [("jpeg", (0,0)+im.size, 0, im.mode)])
    elif filter == "/FlateDecode":
        ImageFile._save(im, op, [("zip", (0,0)+im.size, 0, im.mode)])
    elif filter == "/RunLengthDecode":
        ImageFile._save(im, op, [("packbits", (0,0)+im.size, 0, im.mode)])
    else:
        raise ValueError("unsupported PDF filter (%s)" % filter)

    xref[3] = fp.tell()
    _obj(fp, 3, Type = "/XObject",
                Subtype = "/Image",
                Width = width,
                Height = height,
                Length = len(op.getvalue()),
                Filter = filter,
                BitsPerComponent = bits,
                DecodeParams = params,
                ColorSpace = colorspace)

    fp.write("stream\n")
    fp.write(op.getvalue())
    fp.write("\nendstream\n")

    _endobj(fp)

    #
    # page

    xref[4] = fp.tell()
    _obj(fp, 4)
    fp.write("<<\n/Type /Page\n/Parent 2 0 R\n"\
             "/Resources <<\n/ProcSet [ /PDF %s ]\n"\
             "/XObject << /image 3 0 R >>\n>>\n"\
             "/MediaBox [ 0 0 %d %d ]\n/Contents 5 0 R\n>>\n" %\
             (procset, width, height))
    _endobj(fp)

    #
    # page contents

    op = StringIO.StringIO()

    op.write("q %d 0 0 %d 0 0 cm /image Do Q\n" % (width, height))

    xref[5] = fp.tell()
    _obj(fp, 5, Length = len(op.getvalue()))

    fp.write("stream\n")
    fp.write(op.getvalue())
    fp.write("\nendstream\n")

    _endobj(fp)

    #
    # trailer
    startxref = fp.tell()
    fp.write("xref\n0 %d\n0000000000 65535 f \n" % len(xref))
    for x in xref[1:]:
        fp.write("%010d 00000 n \n" % x)
    fp.write("trailer\n<<\n/Size %d\n/Root 1 0 R\n>>\n" % len(xref))
    fp.write("startxref\n%d\n%%%%EOF\n" % startxref)
    fp.flush()

Example 37

Project: buildtools-BaseTools
Source File: Fv.py
View license
    def __InitializeInf__ (self, BaseAddress = None, BlockSize= None, BlockNum = None, ErasePloarity='1', VtfDict=None) :
        #
        # Create FV inf file
        #
        self.InfFileName = os.path.join(GenFdsGlobalVariable.FvDir,
                                   self.UiFvName + '.inf')
        self.FvInfFile = StringIO.StringIO()

        #
        # Add [Options]
        #
        self.FvInfFile.writelines("[options]" + T_CHAR_LF)
        if BaseAddress != None :
            self.FvInfFile.writelines("EFI_BASE_ADDRESS = " + \
                                       BaseAddress          + \
                                       T_CHAR_LF)

        if BlockSize != None:
            self.FvInfFile.writelines("EFI_BLOCK_SIZE = " + \
                                      '0x%X' %BlockSize    + \
                                      T_CHAR_LF)
            if BlockNum != None:
                self.FvInfFile.writelines("EFI_NUM_BLOCKS   = "  + \
                                      ' 0x%X' %BlockNum    + \
                                      T_CHAR_LF)
        else:
            if self.BlockSizeList == []:
                #set default block size is 1
                self.FvInfFile.writelines("EFI_BLOCK_SIZE  = 0x1" + T_CHAR_LF)
            
            for BlockSize in self.BlockSizeList :
                if BlockSize[0] != None:
                    self.FvInfFile.writelines("EFI_BLOCK_SIZE  = "  + \
                                          '0x%X' %BlockSize[0]    + \
                                          T_CHAR_LF)

                if BlockSize[1] != None:
                    self.FvInfFile.writelines("EFI_NUM_BLOCKS   = "  + \
                                          ' 0x%X' %BlockSize[1]    + \
                                          T_CHAR_LF)

        if self.BsBaseAddress != None:
            self.FvInfFile.writelines('EFI_BOOT_DRIVER_BASE_ADDRESS = ' + \
                                       '0x%X' %self.BsBaseAddress)
        if self.RtBaseAddress != None:
            self.FvInfFile.writelines('EFI_RUNTIME_DRIVER_BASE_ADDRESS = ' + \
                                      '0x%X' %self.RtBaseAddress)
        #
        # Add attribute
        #
        self.FvInfFile.writelines("[attributes]" + T_CHAR_LF)

        self.FvInfFile.writelines("EFI_ERASE_POLARITY   = "       + \
                                          ' %s' %ErasePloarity    + \
                                          T_CHAR_LF)
        if not (self.FvAttributeDict == None):
            for FvAttribute in self.FvAttributeDict.keys() :
                self.FvInfFile.writelines("EFI_"            + \
                                          FvAttribute       + \
                                          ' = '             + \
                                          self.FvAttributeDict[FvAttribute] + \
                                          T_CHAR_LF )
        if self.FvAlignment != None:
            self.FvInfFile.writelines("EFI_FVB2_ALIGNMENT_"     + \
                                       self.FvAlignment.strip() + \
                                       " = TRUE"                + \
                                       T_CHAR_LF)
                                       
        #
        # Generate FV extension header file
        #
        if self.FvNameGuid == None or self.FvNameGuid == '':
            if len(self.FvExtEntryType) > 0:
                GenFdsGlobalVariable.ErrorLogger("FV Extension Header Entries declared for %s with no FvNameGuid declaration." % (self.UiFvName))
        
        if self.FvNameGuid <> None and self.FvNameGuid <> '':
            TotalSize = 16 + 4
            Buffer = ''
            for Index in range (0, len(self.FvExtEntryType)):
                if self.FvExtEntryType[Index] == 'FILE':
                    # check if the path is absolute or relative
                    if os.path.isabs(self.FvExtEntryData[Index]):
                        FileFullPath = os.path.normpath(self.FvExtEntryData[Index])
                    else:
                        FileFullPath = os.path.normpath(os.path.join(GenFdsGlobalVariable.WorkSpaceDir, self.FvExtEntryData[Index]))
                    # check if the file path exists or not
                    if not os.path.isfile(FileFullPath):
                        GenFdsGlobalVariable.ErrorLogger("Error opening FV Extension Header Entry file %s." % (self.FvExtEntryData[Index]))
                    FvExtFile = open (FileFullPath,'rb')
                    FvExtFile.seek(0,2)
                    Size = FvExtFile.tell()
                    if Size >= 0x10000:
                        GenFdsGlobalVariable.ErrorLogger("The size of FV Extension Header Entry file %s exceeds 0x10000." % (self.FvExtEntryData[Index]))
                    TotalSize += (Size + 4)
                    FvExtFile.seek(0)
                    Buffer += pack('HH', (Size + 4), int(self.FvExtEntryTypeValue[Index], 16))
                    Buffer += FvExtFile.read() 
                    FvExtFile.close()
                if self.FvExtEntryType[Index] == 'DATA':
                    ByteList = self.FvExtEntryData[Index].split(',')
                    Size = len (ByteList)
                    if Size >= 0x10000:
                        GenFdsGlobalVariable.ErrorLogger("The size of FV Extension Header Entry data %s exceeds 0x10000." % (self.FvExtEntryData[Index]))
                    TotalSize += (Size + 4)
                    Buffer += pack('HH', (Size + 4), int(self.FvExtEntryTypeValue[Index], 16))
                    for Index1 in range (0, Size):
                        Buffer += pack('B', int(ByteList[Index1], 16))

            Guid = self.FvNameGuid.split('-')
            Buffer = pack('=LHHBBBBBBBBL', 
                        int(Guid[0], 16), 
                        int(Guid[1], 16), 
                        int(Guid[2], 16), 
                        int(Guid[3][-4:-2], 16), 
                        int(Guid[3][-2:], 16),  
                        int(Guid[4][-12:-10], 16),
                        int(Guid[4][-10:-8], 16),
                        int(Guid[4][-8:-6], 16),
                        int(Guid[4][-6:-4], 16),
                        int(Guid[4][-4:-2], 16),
                        int(Guid[4][-2:], 16),
                        TotalSize
                        ) + Buffer

            #
            # Generate FV extension header file if the total size is not zero
            #
            if TotalSize > 0:
                FvExtHeaderFileName = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName + '.ext')
                FvExtHeaderFile = StringIO.StringIO()
                FvExtHeaderFile.write(Buffer)
                Changed = SaveFileOnChange(FvExtHeaderFileName, FvExtHeaderFile.getvalue(), True)
                FvExtHeaderFile.close()
                if Changed:
                  if os.path.exists (self.InfFileName):
                    os.remove (self.InfFileName)
                self.FvInfFile.writelines("EFI_FV_EXT_HEADER_FILE_NAME = "      + \
                                           FvExtHeaderFileName                  + \
                                           T_CHAR_LF)

         
        #
        # Add [Files]
        #
        self.FvInfFile.writelines("[files]" + T_CHAR_LF)
        if VtfDict != None and self.UiFvName in VtfDict.keys():
            self.FvInfFile.writelines("EFI_FILE_NAME = "                   + \
                                       VtfDict.get(self.UiFvName)          + \
                                       T_CHAR_LF)

Example 38

Project: buildtools-BaseTools
Source File: Region.py
View license
    def AddToBuffer(self, Buffer, BaseAddress, BlockSizeList, ErasePolarity, ImageBinDict, vtfDict = None, MacroDict = {}):
        Size = self.Size
        GenFdsGlobalVariable.InfLogger('\nGenerate Region at Offset 0x%X' % self.Offset)
        GenFdsGlobalVariable.InfLogger("   Region Size = 0x%X" %Size)
        GenFdsGlobalVariable.SharpCounter = 0

        if self.RegionType == 'FV':
            #
            # Get Fv from FvDict
            #
            self.FvAddress = int(BaseAddress, 16) + self.Offset
            FvBaseAddress  = '0x%X' %self.FvAddress
            FvOffset       = 0
            for RegionData in self.RegionDataList:
                FileName = None
                if RegionData.endswith(".fv"):
                    RegionData = GenFdsGlobalVariable.MacroExtend(RegionData, MacroDict)
                    GenFdsGlobalVariable.InfLogger('   Region FV File Name = .fv : %s'%RegionData)
                    if RegionData[1] != ':' :
                        RegionData = os.path.join (GenFdsGlobalVariable.WorkSpaceDir, RegionData)
                    if not os.path.exists(RegionData):
                        EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=RegionData)

                    FileName = RegionData
                elif RegionData.upper() + 'fv' in ImageBinDict.keys():
                    GenFdsGlobalVariable.InfLogger('   Region Name = FV')
                    FileName = ImageBinDict[RegionData.upper() + 'fv']
                else:
                    #
                    # Generate FvImage.
                    #
                    FvObj = None
                    if RegionData.upper() in GenFdsGlobalVariable.FdfParser.Profile.FvDict.keys():
                        FvObj = GenFdsGlobalVariable.FdfParser.Profile.FvDict.get(RegionData.upper())

                    if FvObj != None :
                        GenFdsGlobalVariable.InfLogger('   Region Name = FV')
                        #
                        # Call GenFv tool
                        #
                        self.BlockInfoOfRegion(BlockSizeList, FvObj)
                        self.FvAddress = self.FvAddress + FvOffset
                        FvAlignValue = self.GetFvAlignValue(FvObj.FvAlignment)
                        if self.FvAddress % FvAlignValue != 0:
                            EdkLogger.error("GenFds", GENFDS_ERROR,
                                            "FV (%s) is NOT %s Aligned!" % (FvObj.UiFvName, FvObj.FvAlignment))
                        FvBuffer = StringIO.StringIO('')
                        FvBaseAddress = '0x%X' %self.FvAddress
                        BlockSize = None
                        BlockNum = None
                        FvObj.AddToBuffer(FvBuffer, FvBaseAddress, BlockSize, BlockNum, ErasePolarity, vtfDict)
                        if FvBuffer.len > Size:
                            FvBuffer.close()
                            EdkLogger.error("GenFds", GENFDS_ERROR,
                                            "Size of FV (%s) is larger than Region Size 0x%X specified." % (RegionData, Size))
                        #
                        # Put the generated image into FD buffer.
                        #
                        Buffer.write(FvBuffer.getvalue())
                        FvBuffer.close()
                        FvOffset = FvOffset + FvBuffer.len
                        Size = Size - FvBuffer.len
                        continue
                    else:
                        EdkLogger.error("GenFds", GENFDS_ERROR, "FV (%s) is NOT described in FDF file!" % (RegionData))
                #
                # Add the exist Fv image into FD buffer
                #
                if FileName != None:
                    FileLength = os.stat(FileName)[ST_SIZE]
                    if FileLength > Size:
                        EdkLogger.error("GenFds", GENFDS_ERROR,
                                        "Size of FV File (%s) is larger than Region Size 0x%X specified." \
                                        % (RegionData, Size))
                    BinFile = open (FileName, 'r+b')
                    Buffer.write(BinFile.read())
                    BinFile.close()
                    Size = Size - FileLength
            #
            # Pad the left buffer
            #
            if Size > 0:
                if (ErasePolarity == '1') :
                    PadData = 0xFF
                else :
                    PadData = 0
                for i in range(0, Size):
                    Buffer.write(pack('B', PadData))

        if self.RegionType == 'CAPSULE':
            #
            # Get Capsule from Capsule Dict
            #
            for RegionData in self.RegionDataList:
                if RegionData.endswith(".cap"):
                    RegionData = GenFdsGlobalVariable.MacroExtend(RegionData, MacroDict)
                    GenFdsGlobalVariable.InfLogger('   Region CAPSULE Image Name = .cap : %s'%RegionData)
                    if RegionData[1] != ':' :
                        RegionData = os.path.join (GenFdsGlobalVariable.WorkSpaceDir, RegionData)
                    if not os.path.exists(RegionData):
                        EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=RegionData)

                    FileName = RegionData
                elif RegionData.upper() + 'cap' in ImageBinDict.keys():
                    GenFdsGlobalVariable.InfLogger('   Region Name = CAPSULE')
                    FileName = ImageBinDict[RegionData.upper() + 'cap']
                else:
                    #
                    # Generate Capsule image and Put it into FD buffer
                    #
                    CapsuleObj = None
                    if RegionData.upper() in GenFdsGlobalVariable.FdfParser.Profile.CapsuleDict.keys():
                        CapsuleObj = GenFdsGlobalVariable.FdfParser.Profile.CapsuleDict[RegionData.upper()]

                    if CapsuleObj != None :
                        CapsuleObj.CapsuleName = RegionData.upper()
                        GenFdsGlobalVariable.InfLogger('   Region Name = CAPSULE')
                        #
                        # Call GenFv tool to generate Capsule Image
                        #
                        FileName = CapsuleObj.GenCapsule()
                        CapsuleObj.CapsuleName = None
                    else:
                        EdkLogger.error("GenFds", GENFDS_ERROR, "Capsule (%s) is NOT described in FDF file!" % (RegionData))

                #
                # Add the capsule image into FD buffer
                #
                FileLength = os.stat(FileName)[ST_SIZE]
                if FileLength > Size:
                    EdkLogger.error("GenFds", GENFDS_ERROR,
                                    "Size 0x%X of Capsule File (%s) is larger than Region Size 0x%X specified." \
                                    % (FileLength, RegionData, Size))
                BinFile = open (FileName, 'r+b')
                Buffer.write(BinFile.read())
                BinFile.close()
                Size = Size - FileLength
            #
            # Pad the left buffer
            #
            if Size > 0:
                if (ErasePolarity == '1') :
                    PadData = 0xFF
                else :
                    PadData = 0
                for i in range(0, Size):
                    Buffer.write(pack('B', PadData))

        if self.RegionType == 'FILE':
            for RegionData in self.RegionDataList:
                RegionData = GenFdsGlobalVariable.MacroExtend(RegionData, MacroDict)
                if RegionData[1] != ':' :
                    RegionData = os.path.join (GenFdsGlobalVariable.WorkSpaceDir, RegionData)
                if not os.path.exists(RegionData):
                    EdkLogger.error("GenFds", FILE_NOT_FOUND, ExtraData=RegionData)
                #
                # Add the file image into FD buffer
                #
                FileLength = os.stat(RegionData)[ST_SIZE]
                if FileLength > Size:
                    EdkLogger.error("GenFds", GENFDS_ERROR,
                                    "Size of File (%s) is larger than Region Size 0x%X specified." \
                                    % (RegionData, Size))
                GenFdsGlobalVariable.InfLogger('   Region File Name = %s'%RegionData)
                BinFile = open (RegionData, 'rb')
                Buffer.write(BinFile.read())
                BinFile.close()
                Size = Size - FileLength
            #
            # Pad the left buffer
            #
            if Size > 0:
                if (ErasePolarity == '1') :
                    PadData = 0xFF
                else :
                    PadData = 0
                for i in range(0, Size):
                    Buffer.write(pack('B', PadData))

        if self.RegionType == 'DATA' :
            GenFdsGlobalVariable.InfLogger('   Region Name = DATA')
            DataSize = 0
            for RegionData in self.RegionDataList:
                Data = RegionData.split(',')
                DataSize = DataSize + len(Data)
                if DataSize > Size:
                   EdkLogger.error("GenFds", GENFDS_ERROR, "Size of DATA is larger than Region Size ")
                else:
                    for item in Data :
                        Buffer.write(pack('B', int(item, 16)))
                Size = Size - DataSize
            #
            # Pad the left buffer
            #
            if Size > 0:
                if (ErasePolarity == '1') :
                    PadData = 0xFF
                else :
                    PadData = 0
                for i in range(0, Size):
                    Buffer.write(pack('B', PadData))

        if self.RegionType == None:
            GenFdsGlobalVariable.InfLogger('   Region Name = None')
            if (ErasePolarity == '1') :
                PadData = 0xFF
            else :
                PadData = 0
            for i in range(0, Size):
                Buffer.write(pack('B', PadData))

Example 39

Project: MakerDroid
Source File: STLImporter.py
View license
def ImportModel(filename, use_kdtree=True, callback=None, **kwargs):
    global vertices, edges, kdtree
    vertices = 0
    edges = 0
    kdtree = None

    normal_conflict_warning_seen = False

    if hasattr(filename, "read"):
        f = filename
        # useful for later error messages
        filename = "input stream"
    else:
        try:
            url_file = pycam.Utils.URIHandler(filename).open()
            # urllib.urlopen objects do not support "seek" - so we need to read
            # the whole file at once. This is ugly - anyone with a better idea?
            f = StringIO.StringIO(url_file.read())
            url_file.close()
        except IOError, err_msg:
            log.error("STLImporter: Failed to read file (%s): %s" \
                    % (filename, err_msg))
            return None
    # Read the first two lines of (potentially non-binary) input - they should
    # contain "solid" and "facet".
    header_lines = []
    while len(header_lines) < 2:
        line = f.readline(200)
        if len(line) == 0:
            # empty line (not even a line-feed) -> EOF
            log.error("STLImporter: No valid lines found in '%s'" % filename)
            return None
        # ignore comment lines
        # note: partial comments (starting within a line) are not handled
        if not line.startswith(";"):
            header_lines.append(line)
    header = "".join(header_lines)
    # read byte 80 to 83 - they contain the "numfacets" value in binary format
    f.seek(80)
    numfacets = unpack("<I", f.read(4))[0]
    binary = False

    if f.len == (84 + 50*numfacets):
        binary = True
    elif header.find("solid") >= 0 and header.find("facet") >= 0:
        binary = False
        f.seek(0)
    else:
        log.error("STLImporter: STL binary/ascii detection failed")
        return None

    if use_kdtree:
        kdtree = PointKdtree([], 3, 1, epsilon)
    model = Model(use_kdtree)

    t = None
    p1 = None
    p2 = None
    p3 = None

    if binary:
        for i in range(1, numfacets + 1): 
            if callback and callback():
                log.warn("STLImporter: load model operation cancelled")
                return None
            a1 = unpack("<f", f.read(4))[0] 
            a2 = unpack("<f", f.read(4))[0] 
            a3 = unpack("<f", f.read(4))[0] 

            n = Vector(float(a1), float(a2), float(a3))
            
            v11 = unpack("<f", f.read(4))[0] 
            v12 = unpack("<f", f.read(4))[0] 
            v13 = unpack("<f", f.read(4))[0] 

            p1 = UniqueVertex(float(v11), float(v12), float(v13))
            
            v21 = unpack("<f", f.read(4))[0] 
            v22 = unpack("<f", f.read(4))[0] 
            v23 = unpack("<f", f.read(4))[0] 

            p2 = UniqueVertex(float(v21), float(v22), float(v23))
            
            v31 = unpack("<f", f.read(4))[0] 
            v32 = unpack("<f", f.read(4))[0] 
            v33 = unpack("<f", f.read(4))[0] 
            
            p3 = UniqueVertex(float(v31), float(v32), float(v33))

            # not used
            attribs = unpack("<H", f.read(2)) 
            
            dotcross = n.dot(p2.sub(p1).cross(p3.sub(p1)))
            if a1 == a2 == a3 == 0:
                dotcross = p2.sub(p1).cross(p3.sub(p1)).z
                n = None

            if dotcross > 0:
                # Triangle expects the vertices in clockwise order
                t = Triangle(p1, p3, p2)
            elif dotcross < 0:
                if not normal_conflict_warning_seen:
                    log.warn(("Inconsistent normal/vertices found in facet " + \
                            "definition %d of '%s'. Please validate the " + \
                            "STL file!") % (i, filename))
                    normal_conflict_warning_seen = True
                t = Triangle(p1, p2, p3)
            else:
                # the three points are in a line - or two points are identical
                # usually this is caused by points, that are too close together
                # check the tolerance value in pycam/Geometry/PointKdtree.py
                log.warn("Skipping invalid triangle: %s / %s / %s " \
                        % (p1, p2, p3) + "(maybe the resolution of the model " \
                        + "is too high?)")
                continue
            if n:
                t.normal = n

            model.append(t)
    else:
        solid = re.compile(r"\s*solid\s+(\w+)\s+.*")
        endsolid = re.compile(r"\s*endsolid\s*")
        facet = re.compile(r"\s*facet\s*")
        normal = re.compile(r"\s*facet\s+normal" \
                + r"\s+(?P<x>[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?)" \
                + r"\s+(?P<y>[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?)" \
                + r"\s+(?P<z>[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?)\s+")
        endfacet = re.compile(r"\s*endfacet\s+")
        loop = re.compile(r"\s*outer\s+loop\s+")
        endloop = re.compile(r"\s*endloop\s+")
        vertex = re.compile(r"\s*vertex" \
                + r"\s+(?P<x>[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?)" \
                + r"\s+(?P<y>[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?)" \
                + r"\s+(?P<z>[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?)\s+")

        current_line = 0

        for line in f:
            if callback and callback():
                log.warn("STLImporter: load model operation cancelled")
                return None
            current_line += 1
            m = solid.match(line)
            if m:
                model.name = m.group(1)
                continue

            m = facet.match(line)
            if m:
                m = normal.match(line)
                if m:
                    n = Vector(float(m.group('x')), float(m.group('y')),
                            float(m.group('z')))
                else:
                    n = None
                continue
            m = loop.match(line)
            if m:
                continue
            m = vertex.match(line)
            if m:
                p = UniqueVertex(float(m.group('x')), float(m.group('y')),
                        float(m.group('z')))
                if p1 is None:
                    p1 = p
                elif p2 is None:
                    p2 = p
                elif p3 is None:
                    p3 = p
                else:
                    log.error("STLImporter: more then 3 points in facet " \
                            + "(line %d)" % current_line)
                continue
            m = endloop.match(line)
            if m:
                continue
            m = endfacet.match(line)
            if m:
                if p1 is None or p2 is None or p3 is None:
                    log.warn(("Invalid facet definition in line " \
                            + "%d of '%s'. Please validate the STL file!") \
                            % (current_line, filename))
                    n, p1, p2, p3 = None, None, None, None
                    continue
                if not n:
                    n = p2.sub(p1).cross(p3.sub(p1)).normalized()

                # validate the normal
                # The three vertices of a triangle in an STL file are supposed
                # to be in counter-clockwise order. This should match the
                # direction of the normal.
                if n is None:
                    # invalid triangle (zero-length vector)
                    dotcross = 0
                else:
                    # make sure the points are in ClockWise order
                    dotcross = n.dot(p2.sub(p1).cross(p3.sub(p1)))
                if dotcross > 0:
                    # Triangle expects the vertices in clockwise order
                    t = Triangle(p1, p3, p2, n)
                elif dotcross < 0:
                    if not normal_conflict_warning_seen:
                        log.warn(("Inconsistent normal/vertices found in " + \
                                "line %d of '%s'. Please validate the STL " + \
                                "file!") % (current_line, filename))
                        normal_conflict_warning_seen = True
                    t = Triangle(p1, p2, p3, n)
                else:
                    # The three points are in a line - or two points are
                    # identical. Usually this is caused by points, that are too
                    # close together. Check the tolerance value in
                    # pycam/Geometry/PointKdtree.py.
                    log.warn("Skipping invalid triangle: %s / %s / %s " \
                            % (p1, p2, p3) + "(maybe the resolution of the " \
                            + "model is too high?)")
                    n, p1, p2, p3 = (None, None, None, None)
                    continue
                n, p1, p2, p3 = (None, None, None, None)
                model.append(t)
                continue
            m = endsolid.match(line)
            if m:
                continue

    log.info("Imported STL model: %d vertices, %d edges, %d triangles" \
            % (vertices, edges, len(model.triangles())))
    vertices = 0
    edges = 0
    kdtree = None

    if not model:
        # no valid items added to the model
        return None
    else:
        return model

Example 40

Project: WAPT
Source File: cgi.py
View license
    def __init__(self, fp=None, headers=None, outerboundary="",
                 environ=os.environ, keep_blank_values=0, strict_parsing=0):
        """Constructor.  Read multipart/* until last part.

        Arguments, all optional:

        fp              : file pointer; default: sys.stdin
            (not used when the request method is GET)

        headers         : header dictionary-like object; default:
            taken from environ as per CGI spec

        outerboundary   : terminating multipart boundary
            (for internal use only)

        environ         : environment dictionary; default: os.environ

        keep_blank_values: flag indicating whether blank values in
            percent-encoded forms should be treated as blank strings.
            A true value indicates that blanks should be retained as
            blank strings.  The default false value indicates that
            blank values are to be ignored and treated as if they were
            not included.

        strict_parsing: flag indicating what to do with parsing errors.
            If false (the default), errors are silently ignored.
            If true, errors raise a ValueError exception.

        """
        method = 'GET'
        self.keep_blank_values = keep_blank_values
        self.strict_parsing = strict_parsing
        if 'REQUEST_METHOD' in environ:
            method = environ['REQUEST_METHOD'].upper()
        self.qs_on_post = None
        if method == 'GET' or method == 'HEAD':
            if 'QUERY_STRING' in environ:
                qs = environ['QUERY_STRING']
            elif sys.argv[1:]:
                qs = sys.argv[1]
            else:
                qs = ""
            fp = StringIO(qs)
            if headers is None:
                headers = {'content-type':
                           "application/x-www-form-urlencoded"}
        if headers is None:
            headers = {}
            if method == 'POST':
                # Set default content-type for POST to what's traditional
                headers['content-type'] = "application/x-www-form-urlencoded"
            if 'CONTENT_TYPE' in environ:
                headers['content-type'] = environ['CONTENT_TYPE']
            if 'QUERY_STRING' in environ:
                self.qs_on_post = environ['QUERY_STRING']
            if 'CONTENT_LENGTH' in environ:
                headers['content-length'] = environ['CONTENT_LENGTH']
        self.fp = fp or sys.stdin
        self.headers = headers
        self.outerboundary = outerboundary

        # Process content-disposition header
        cdisp, pdict = "", {}
        if 'content-disposition' in self.headers:
            cdisp, pdict = parse_header(self.headers['content-disposition'])
        self.disposition = cdisp
        self.disposition_options = pdict
        self.name = None
        if 'name' in pdict:
            self.name = pdict['name']
        self.filename = None
        if 'filename' in pdict:
            self.filename = pdict['filename']

        # Process content-type header
        #
        # Honor any existing content-type header.  But if there is no
        # content-type header, use some sensible defaults.  Assume
        # outerboundary is "" at the outer level, but something non-false
        # inside a multi-part.  The default for an inner part is text/plain,
        # but for an outer part it should be urlencoded.  This should catch
        # bogus clients which erroneously forget to include a content-type
        # header.
        #
        # See below for what we do if there does exist a content-type header,
        # but it happens to be something we don't understand.
        if 'content-type' in self.headers:
            ctype, pdict = parse_header(self.headers['content-type'])
        elif self.outerboundary or method != 'POST':
            ctype, pdict = "text/plain", {}
        else:
            ctype, pdict = 'application/x-www-form-urlencoded', {}
        self.type = ctype
        self.type_options = pdict
        self.innerboundary = ""
        if 'boundary' in pdict:
            self.innerboundary = pdict['boundary']
        clen = -1
        if 'content-length' in self.headers:
            try:
                clen = int(self.headers['content-length'])
            except ValueError:
                pass
            if maxlen and clen > maxlen:
                raise ValueError, 'Maximum content length exceeded'
        self.length = clen

        self.list = self.file = None
        self.done = 0
        if ctype == 'application/x-www-form-urlencoded':
            self.read_urlencoded()
        elif ctype[:10] == 'multipart/':
            self.read_multi(environ, keep_blank_values, strict_parsing)
        else:
            self.read_single()

Example 41

Project: pelisalacarta
Source File: plugintools.py
View license
def read_body_and_headers(url, post=None, headers=[], follow_redirects=False, timeout=30):
    _log("read_body_and_headers "+url)

    if post is not None:
        _log("read_body_and_headers post="+post)

    if len(headers)==0:
        headers.append(["User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:18.0) Gecko/20100101 Firefox/18.0"])

    # Start cookie lib
    ficherocookies = os.path.join( get_data_path(), 'cookies.dat' )
    _log("read_body_and_headers cookies_file="+ficherocookies)

    cj = None
    ClientCookie = None
    cookielib = None

    # Let's see if cookielib is available
    try:
        _log("read_body_and_headers importing cookielib")
        import cookielib
    except ImportError:
        _log("read_body_and_headers cookielib no disponible")
        # If importing cookielib fails
        # let's try ClientCookie
        try:
            _log("read_body_and_headers importing ClientCookie")
            import ClientCookie
        except ImportError:
            _log("read_body_and_headers ClientCookie not available")
            # ClientCookie isn't available either
            urlopen = urllib2.urlopen
            Request = urllib2.Request
        else:
            _log("read_body_and_headers ClientCookie available")
            # imported ClientCookie
            urlopen = ClientCookie.urlopen
            Request = ClientCookie.Request
            cj = ClientCookie.MozillaCookieJar()

    else:
        _log("read_body_and_headers cookielib available")
        # importing cookielib worked
        urlopen = urllib2.urlopen
        Request = urllib2.Request
        cj = cookielib.MozillaCookieJar()
        # This is a subclass of FileCookieJar
        # that has useful load and save methods

    if cj is not None:
    # we successfully imported
    # one of the two cookie handling modules
        _log("read_body_and_headers Cookies enabled")

        if os.path.isfile(ficherocookies):
            _log("read_body_and_headers Reading cookie file")
            # if we have a cookie file already saved
            # then load the cookies into the Cookie Jar
            try:
                cj.load(ficherocookies, ignore_discard=True)
            except:
                _log("read_body_and_headers Wrong cookie file, deleting...")
                os.remove(ficherocookies)

        # Now we need to get our Cookie Jar
        # installed in the opener;
        # for fetching URLs
        if cookielib is not None:
            _log("read_body_and_headers opener using urllib2 (cookielib)")
            # if we use cookielib
            # then we get the HTTPCookieProcessor
            # and install the opener in urllib2
            if not follow_redirects:
                opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=http_debug_log_enabled),urllib2.HTTPCookieProcessor(cj),NoRedirectHandler())
            else:
                opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=http_debug_log_enabled),urllib2.HTTPCookieProcessor(cj))
            urllib2.install_opener(opener)

        else:
            _log("read_body_and_headers opener using ClientCookie")
            # if we use ClientCookie
            # then we get the HTTPCookieProcessor
            # and install the opener in ClientCookie
            opener = ClientCookie.build_opener(ClientCookie.HTTPCookieProcessor(cj))
            ClientCookie.install_opener(opener)

    # -------------------------------------------------
    # Cookies instaladas, lanza la petición
    # -------------------------------------------------

    # Contador
    inicio = time.clock()

    # Diccionario para las cabeceras
    txheaders = {}

    # Construye el request
    if post is None:
        _log("read_body_and_headers GET request")
    else:
        _log("read_body_and_headers POST request")
    
    # Añade las cabeceras
    _log("read_body_and_headers ---------------------------")
    for header in headers:
        _log("read_body_and_headers header %s=%s" % (str(header[0]),str(header[1])) )
        txheaders[header[0]]=header[1]
    _log("read_body_and_headers ---------------------------")

    req = Request(url, post, txheaders)
    if timeout is None:
        handle=urlopen(req)
    else:        
        #Disponible en python 2.6 en adelante --> handle = urlopen(req, timeout=timeout)
        #Para todas las versiones:
        try:
            import socket
            deftimeout = socket.getdefaulttimeout()
            socket.setdefaulttimeout(timeout)
            handle=urlopen(req)            
            socket.setdefaulttimeout(deftimeout)
        except:
            import sys
            for line in sys.exc_info():
                _log( "%s" % line )
    
    # Actualiza el almacén de cookies
    cj.save(ficherocookies, ignore_discard=True)

    # Lee los datos y cierra
    if handle.info().get('Content-Encoding') == 'gzip':
        buf = StringIO( handle.read())
        f = gzip.GzipFile(fileobj=buf)
        data = f.read()
    else:
        data=handle.read()

    info = handle.info()
    _log("read_body_and_headers Response")

    returnheaders=[]
    _log("read_body_and_headers ---------------------------")
    for header in info:
        _log("read_body_and_headers "+header+"="+info[header])
        returnheaders.append([header,info[header]])
    handle.close()
    _log("read_body_and_headers ---------------------------")

    '''
    # Lanza la petición
    try:
        response = urllib2.urlopen(req)
    # Si falla la repite sustituyendo caracteres especiales
    except:
        req = urllib2.Request(url.replace(" ","%20"))
    
        # Añade las cabeceras
        for header in headers:
            req.add_header(header[0],header[1])

        response = urllib2.urlopen(req)
    '''
    
    # Tiempo transcurrido
    fin = time.clock()
    _log("read_body_and_headers Downloaded in %d seconds " % (fin-inicio+1))
    _log("read_body_and_headers body="+data)

    return data,returnheaders

Example 42

Project: pelisalacarta
Source File: _Util.py
View license
    def isleap(year):
    return year % 4 == 0 and (year % 100 <> 0 or year % 400 == 0)

    # Return number of leap years in range [y1, y2)
    # Assume y1 <= y2 and no funny (non-leap century) years
    def leapdays(y1, y2):
    return (y2+3)/4 - (y1+3)/4

    EPOCH = 1970
    def timegm(tuple):
        """Unrelated but handy function to calculate Unix timestamp from GMT."""
        year, month, day, hour, minute, second = tuple[:6]
        assert year >= EPOCH
        assert 1 <= month <= 12
        days = 365*(year-EPOCH) + leapdays(EPOCH, year)
        for i in range(1, month):
            days = days + mdays[i]
        if month > 2 and isleap(year):
            days = days + 1
        days = days + day - 1
        hours = days*24 + hour
        minutes = hours*60 + minute
        seconds = minutes*60L + second
        return seconds


# Date/time conversion routines for formats used by the HTTP protocol.

EPOCH = 1970
def my_timegm(tt):
    year, month, mday, hour, min, sec = tt[:6]
    if ((year >= EPOCH) and (1 <= month <= 12) and (1 <= mday <= 31) and
        (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
        return timegm(tt)
    else:
        return None

days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
          "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
months_lower = []
for month in months: months_lower.append(string.lower(month))


def time2isoz(t=None):
    """Return a string representing time in seconds since epoch, t.

    If the function is called without an argument, it will use the current
    time.

    The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
    representing Universal Time (UTC, aka GMT).  An example of this format is:

    1994-11-24 08:49:37Z

    """
    if t is None: t = time.time()
    year, mon, mday, hour, min, sec = time.gmtime(t)[:6]
    return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
        year, mon, mday, hour, min, sec)

def time2netscape(t=None):
    """Return a string representing time in seconds since epoch, t.

    If the function is called without an argument, it will use the current
    time.

    The format of the returned string is like this:

    Wed, DD-Mon-YYYY HH:MM:SS GMT

    """
    if t is None: t = time.time()
    year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7]
    return "%s %02d-%s-%04d %02d:%02d:%02d GMT" % (
        days[wday], mday, months[mon-1], year, hour, min, sec)


UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}

timezone_re = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$")
def offset_from_tz_string(tz):
    offset = None
    if UTC_ZONES.has_key(tz):
        offset = 0
    else:
        m = timezone_re.search(tz)
        if m:
            offset = 3600 * int(m.group(2))
            if m.group(3):
                offset = offset + 60 * int(m.group(3))
            if m.group(1) == '-':
                offset = -offset
    return offset

def _str2time(day, mon, yr, hr, min, sec, tz):
    # translate month name to number
    # month numbers start with 1 (January)
    try:
        mon = months_lower.index(string.lower(mon))+1
    except ValueError:
        # maybe it's already a number
        try:
            imon = int(mon)
        except ValueError:
            return None
        if 1 <= imon <= 12:
            mon = imon
        else:
            return None

    # make sure clock elements are defined
    if hr is None: hr = 0
    if min is None: min = 0
    if sec is None: sec = 0

    yr = int(yr)
    day = int(day)
    hr = int(hr)
    min = int(min)
    sec = int(sec)

    if yr < 1000:
    # find "obvious" year
    cur_yr = time.localtime(time.time())[0]
    m = cur_yr % 100
    tmp = yr
    yr = yr + cur_yr - m
    m = m - tmp
        if abs(m) > 50:
            if m > 0: yr = yr + 100
            else: yr = yr - 100

    # convert UTC time tuple to seconds since epoch (not timezone-adjusted)
    t = my_timegm((yr, mon, day, hr, min, sec, tz))

    if t is not None:
        # adjust time using timezone string, to get absolute time since epoch
        if tz is None:
            tz = "UTC"
        tz = string.upper(tz)
        offset = offset_from_tz_string(tz)
        if offset is None:
            return None
        t = t - offset

    return t


strict_re = re.compile(r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) (\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$")
wkday_re = re.compile(
    r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I)
loose_http_re = re.compile(
    r"""^
    (\d\d?)            # day
       (?:\s+|[-\/])
    (\w+)              # month
        (?:\s+|[-\/])
    (\d+)              # year
    (?:
      (?:\s+|:)    # separator before clock
       (\d\d?):(\d\d)  # hour:min
       (?::(\d\d))?    # optional seconds
    )?                 # optional clock
       \s*
    ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
       \s*
    (?:\(\w+\))?       # ASCII representation of timezone in parens.
       \s*$""", re.X)
def http2time(text):
    """Returns time in seconds since epoch of time represented by a string.

    Return value is an integer.

    None is returned if the format of str is unrecognized, the time is outside
    the representable range, or the timezone string is not recognized.  If the
    string contains no timezone, UTC is assumed.

    The timezone in the string may be numerical (like "-0800" or "+0100") or a
    string timezone (like "UTC", "GMT", "BST" or "EST").  Currently, only the
    timezone strings equivalent to UTC (zero offset) are known to the function.

    The function loosely parses the following formats:

    Wed, 09 Feb 1994 22:23:32 GMT       -- HTTP format
    Tuesday, 08-Feb-94 14:15:29 GMT     -- old rfc850 HTTP format
    Tuesday, 08-Feb-1994 14:15:29 GMT   -- broken rfc850 HTTP format
    09 Feb 1994 22:23:32 GMT            -- HTTP format (no weekday)
    08-Feb-94 14:15:29 GMT              -- rfc850 format (no weekday)
    08-Feb-1994 14:15:29 GMT            -- broken rfc850 format (no weekday)

    The parser ignores leading and trailing whitespace.  The time may be
    absent.

    If the year is given with only 2 digits, the function will select the
    century that makes the year closest to the current date.

    """
    # fast exit for strictly conforming string
    m = strict_re.search(text)
    if m:
        g = m.groups()
        mon = months_lower.index(string.lower(g[1])) + 1
        tt = (int(g[2]), mon, int(g[0]),
              int(g[3]), int(g[4]), float(g[5]))
        return my_timegm(tt)

    # No, we need some messy parsing...

    # clean up
    text = string.lstrip(text)
    text = wkday_re.sub("", text, 1)  # Useless weekday

    # tz is time zone specifier string
    day, mon, yr, hr, min, sec, tz = [None]*7

    # loose regexp parse
    m = loose_http_re.search(text)
    if m is not None:
        day, mon, yr, hr, min, sec, tz = m.groups()
    else:
        return None  # bad format

    return _str2time(day, mon, yr, hr, min, sec, tz)


iso_re = re.compile(
    """^
    (\d{4})              # year
       [-\/]?
    (\d\d?)              # numerical month
       [-\/]?
    (\d\d?)              # day
   (?:
         (?:\s+|[-:Tt])  # separator before clock
      (\d\d?):?(\d\d)    # hour:min
      (?::?(\d\d(?:\.\d*)?))?  # optional seconds (and fractional)
   )?                    # optional clock
      \s*
   ([-+]?\d\d?:?(:?\d\d)?
    |Z|z)?               # timezone  (Z is "zero meridian", i.e. GMT)
      \s*$""", re.X)
def iso2time(text):
    """
    As for http2time, but parses the ISO 8601 formats:

    1994-02-03 14:15:29 -0100    -- ISO 8601 format
    1994-02-03 14:15:29          -- zone is optional
    1994-02-03                   -- only date
    1994-02-03T14:15:29          -- Use T as separator
    19940203T141529Z             -- ISO 8601 compact format
    19940203                     -- only date

    """
    # clean up
    text = string.lstrip(text)

    # tz is time zone specifier string
    day, mon, yr, hr, min, sec, tz = [None]*7

    # loose regexp parse
    m = iso_re.search(text)
    if m is not None:
        # XXX there's an extra bit of the timezone I'm ignoring here: is
        #   this the right thing to do?
        yr, mon, day, hr, min, sec, tz, _ = m.groups()
    else:
        return None  # bad format

    return _str2time(day, mon, yr, hr, min, sec, tz)



# XXX Andrew Dalke kindly sent me a similar class in response to my request on
# comp.lang.python, which I then proceeded to lose.  I wrote this class
# instead, but I think he's released his code publicly since, could pinch the
# tests from it, at least...
class seek_wrapper:
    """Adds a seek method to a file object.

    This is only designed for seeking on readonly file-like objects.

    Wrapped file-like object must have a read method.  The readline method is
    only supported if that method is present on the wrapped object.  The
    readlines method is always supported.  xreadlines and iteration are
    supported only for Python 2.2 and above.

    Public attribute: wrapped (the wrapped file object).

    WARNING: All other attributes of the wrapped object (ie. those that are not
    one of wrapped, read, readline, readlines, xreadlines, __iter__ and next)
    are passed through unaltered, which may or may not make sense for your
    particular file object.

    """
    # General strategy is to check that cache is full enough, then delegate
    # everything to the cache (self._cache, which is a StringIO.StringIO
    # instance.  Seems to be some cStringIO.StringIO problem on 1.5.2 -- I
    # get a StringOobject, with no readlines method.

    # Invariant: the end of the cache is always at the same place as the
    # end of the wrapped file:
    # self.wrapped.tell() == self.__cache.tell()

    def __init__(self, wrapped):
        self.wrapped = wrapped
        self.__have_readline = hasattr(self.wrapped, "readline")
        self.__cache = StringIO()

    def __getattr__(self, name):
        wrapped = self.__dict__.get("wrapped")
        if wrapped:
            return getattr(wrapped, name)
        return getattr(self.__class__, name)

    def seek(self, offset, whence=0):
        # make sure we have read all data up to the point we are seeking to
        pos = self.__cache.tell()
        if whence == 0:  # absolute
            to_read = offset - pos
        elif whence == 1:  # relative to current position
            to_read = offset
        elif whence == 2:  # relative to end of *wrapped* file
            # since we don't know yet where the end of that file is, we must
            # read everything
            to_read = None
        if to_read is None or to_read >= 0:
            if to_read is None:
                self.__cache.write(self.wrapped.read())
            else:
                self.__cache.write(self.wrapped.read(to_read))
            self.__cache.seek(pos)

        return self.__cache.seek(offset, whence)

    def tell(self):
        return self.__cache.tell()

    def read(self, size=-1):
        pos = self.__cache.tell()

        self.__cache.seek(pos)

        end = len(self.__cache.getvalue())
        available = end - pos

        # enough data already cached?
        if size <= available and size != -1:
            return self.__cache.read(size)

        # no, so read sufficient data from wrapped file and cache it
        to_read = size - available
        assert to_read > 0 or size == -1
        self.__cache.seek(0, 2)
        if size == -1:
            self.__cache.write(self.wrapped.read())
        else:
            self.__cache.write(self.wrapped.read(to_read))
        self.__cache.seek(pos)

        return self.__cache.read(size)

    def readline(self, size=-1):
        if not self.__have_readline:
            raise NotImplementedError("no readline method on wrapped object")

        # line we're about to read might not be complete in the cache, so
        # read another line first
        pos = self.__cache.tell()
        self.__cache.seek(0, 2)
        self.__cache.write(self.wrapped.readline())
        self.__cache.seek(pos)

        data = self.__cache.readline()
        if size != -1:
            r = data[:size]
            self.__cache.seek(pos+size)
        else:
            r = data
        return r

    def readlines(self, sizehint=-1):
        pos = self.__cache.tell()
        self.__cache.seek(0, 2)
        self.__cache.write(self.wrapped.read())
        self.__cache.seek(pos)
        try:
            return self.__cache.readlines(sizehint)
        except TypeError:  # 1.5.2 hack
            return self.__cache.readlines()

    def __iter__(self): return self
    def next(self):
        line = self.readline()
        if line == "": raise StopIteration
        return line

    xreadlines = __iter__

    def __repr__(self):
        return ("<%s at %s whose wrapped object = %s>" %
                (self.__class__.__name__, `id(self)`, `self.wrapped`))

    def close(self):
        self._cache = None
        self.read = None
        self.readline = None
        self.readlines = None
        self.seek = None
        if self.wrapped: self.wrapped.close()
        self.wrapped = None

class eoffile:
    # file-like object that always claims to be at end-of-file
    def read(self, size=-1): return ""
    def readline(self, size=-1): return ""

class response_seek_wrapper(seek_wrapper):
    """Avoids unnecessarily clobbering methods on .close().

    Also supports pickling.

    Class name is for historical reasons.

    """

    def close(self):
        self.wrapped.close()
        self.wrapped = eoffile()

    def __getstate__(self):
        # There are three obvious options here:
        # 1. truncate
        # 2. read to end
        # 3. close socket, pickle state including read position, then open
        #    again on unpickle and use Range header

        # 2 breaks pickle protocol, because one expects the original object
        # to be left unscathed by pickling.  3 is too complicated and
        # surprising (and too much work ;-) to happen in a sane __getstate__.
        # So we do 1.

        state = self.__dict__.copy()
        state["wrapped"] = eoffile()
        return state

Example 43

Project: oleviewdotnet
Source File: cgi.py
View license
    def __init__(self, fp=None, headers=None, outerboundary="",
                 environ=os.environ, keep_blank_values=0, strict_parsing=0):
        """Constructor.  Read multipart/* until last part.

        Arguments, all optional:

        fp              : file pointer; default: sys.stdin
            (not used when the request method is GET)

        headers         : header dictionary-like object; default:
            taken from environ as per CGI spec

        outerboundary   : terminating multipart boundary
            (for internal use only)

        environ         : environment dictionary; default: os.environ

        keep_blank_values: flag indicating whether blank values in
            percent-encoded forms should be treated as blank strings.
            A true value indicates that blanks should be retained as
            blank strings.  The default false value indicates that
            blank values are to be ignored and treated as if they were
            not included.

        strict_parsing: flag indicating what to do with parsing errors.
            If false (the default), errors are silently ignored.
            If true, errors raise a ValueError exception.

        """
        method = 'GET'
        self.keep_blank_values = keep_blank_values
        self.strict_parsing = strict_parsing
        if 'REQUEST_METHOD' in environ:
            method = environ['REQUEST_METHOD'].upper()
        self.qs_on_post = None
        if method == 'GET' or method == 'HEAD':
            if 'QUERY_STRING' in environ:
                qs = environ['QUERY_STRING']
            elif sys.argv[1:]:
                qs = sys.argv[1]
            else:
                qs = ""
            fp = StringIO(qs)
            if headers is None:
                headers = {'content-type':
                           "application/x-www-form-urlencoded"}
        if headers is None:
            headers = {}
            if method == 'POST':
                # Set default content-type for POST to what's traditional
                headers['content-type'] = "application/x-www-form-urlencoded"
            if 'CONTENT_TYPE' in environ:
                headers['content-type'] = environ['CONTENT_TYPE']
            if 'QUERY_STRING' in environ:
                self.qs_on_post = environ['QUERY_STRING']
            if 'CONTENT_LENGTH' in environ:
                headers['content-length'] = environ['CONTENT_LENGTH']
        self.fp = fp or sys.stdin
        self.headers = headers
        self.outerboundary = outerboundary

        # Process content-disposition header
        cdisp, pdict = "", {}
        if 'content-disposition' in self.headers:
            cdisp, pdict = parse_header(self.headers['content-disposition'])
        self.disposition = cdisp
        self.disposition_options = pdict
        self.name = None
        if 'name' in pdict:
            self.name = pdict['name']
        self.filename = None
        if 'filename' in pdict:
            self.filename = pdict['filename']

        # Process content-type header
        #
        # Honor any existing content-type header.  But if there is no
        # content-type header, use some sensible defaults.  Assume
        # outerboundary is "" at the outer level, but something non-false
        # inside a multi-part.  The default for an inner part is text/plain,
        # but for an outer part it should be urlencoded.  This should catch
        # bogus clients which erroneously forget to include a content-type
        # header.
        #
        # See below for what we do if there does exist a content-type header,
        # but it happens to be something we don't understand.
        if 'content-type' in self.headers:
            ctype, pdict = parse_header(self.headers['content-type'])
        elif self.outerboundary or method != 'POST':
            ctype, pdict = "text/plain", {}
        else:
            ctype, pdict = 'application/x-www-form-urlencoded', {}
        self.type = ctype
        self.type_options = pdict
        self.innerboundary = ""
        if 'boundary' in pdict:
            self.innerboundary = pdict['boundary']
        clen = -1
        if 'content-length' in self.headers:
            try:
                clen = int(self.headers['content-length'])
            except ValueError:
                pass
            if maxlen and clen > maxlen:
                raise ValueError, 'Maximum content length exceeded'
        self.length = clen

        self.list = self.file = None
        self.done = 0
        if ctype == 'application/x-www-form-urlencoded':
            self.read_urlencoded()
        elif ctype[:10] == 'multipart/':
            self.read_multi(environ, keep_blank_values, strict_parsing)
        else:
            self.read_single()

Example 44

Project: mystic
Source File: scripts.py
View license
def model_plotter(model, logfile=None, **kwds):
    """
generate surface contour plots for model, specified by full import path
generate model trajectory from logfile (or solver restart file), if provided

Available from the command shell as:
  mystic_model_plotter.py model (filename) [options]

or as a function call as:
  mystic.model_plotter(model, filename=None, **options)

The option "bounds" takes an indicator string, where the bounds should
be given as comma-separated slices. For example, using bounds = "-1:10, 0:20"
will set the lower and upper bounds for x to be (-1,10) and y to be (0,20).
The "step" can also be given, to control the number of lines plotted in the
grid. Thus "-1:10:.1, 0:20" would set the bounds as above, but use increments
of .1 along x and the default step along y.  For models with > 2D, the bounds
can be used to specify 2 dimensions plus fixed values for remaining dimensions.
Thus, "-1:10, 0:20, 1.0" would plot the 2D surface where the z-axis was fixed
at z=1.0.  When called from a script, slice objects can be used instead of a
string, thus "-1:10:.1, 0:20, 1.0" becomes (slice(-1,10,.1), slice(20), 1.0).

The option "label" takes comma-separated strings. For example, label = "x,y,"
will place 'x' on the x-axis, 'y' on the y-axis, and nothing on the z-axis.
LaTeX is also accepted. For example, label = "$ h $, $ {\\alpha}$, $ v$" will
label the axes with standard LaTeX math formatting. Note that the leading
space is required, while a trailing space aligns the text with the axis
instead of the plot frame.

The option "reduce" can be given to reduce the output of a model to a scalar,
thus converting 'model(params)' to 'reduce(model(params))'. A reducer is given
by the import path (e.g. 'numpy.add'). The option "scale" will convert the plot
to log-scale, and scale the cost by 'z=log(4*z*scale+1)+2'. This is useful for
visualizing small contour changes around the minimium. If using log-scale
produces negative numbers, the option "shift" can be used to shift the cost
by 'z=z+shift'. Both shift and scale are intended to help visualize contours.

Required Inputs:
  model               full import path for the model (e.g. mystic.models.rosen)

Additional Inputs:
  filename            name of the convergence logfile (e.g. log.txt)
"""
    #FIXME: should be able to:
    # - apply a constraint as a region of NaN -- apply when 'xx,yy=x[ij],y[ij]'
    # - apply a penalty by shifting the surface (plot w/alpha?) -- as above
    # - build an appropriately-sized default grid (from logfile info)
    # - move all mulit-id param/cost reading into read_history
    #FIXME: current issues:
    # - 1D slice and projection work for 2D function, but aren't "pretty"
    # - 1D slice and projection for 1D function, is it meaningful and correct?
    # - should be able to plot from solver.genealogy (multi-monitor?) [1D,2D,3D?]
    # - should be able to scale 'z-axis' instead of scaling 'z' itself
    #   (see https://github.com/matplotlib/matplotlib/issues/209)
    # - if trajectory outside contour grid, will increase bounds
    #   (see support_hypercube.py for how to fix bounds)
    import shlex
    global __quit
    __quit = False
    _model = None
    _reducer = None
    _solver = None

    instance = None
    # handle the special case where list is provided by sys.argv
    if isinstance(model, (list,tuple)) and not logfile and not kwds:
        cmdargs = model # (above is used by script to parse command line)
    elif isinstance(model, basestring) and not logfile and not kwds:
        cmdargs = shlex.split(model)
    # 'everything else' is essentially the functional interface
    else:
        cmdargs = kwds.get('kwds', '')
        if not cmdargs:
            out = kwds.get('out', None)
            bounds = kwds.get('bounds', None)
            label = kwds.get('label', None)
            nid = kwds.get('nid', None)
            iter = kwds.get('iter', None)
            reduce = kwds.get('reduce', None)
            scale = kwds.get('scale', None)
            shift = kwds.get('shift', None)
            fill = kwds.get('fill', False)
            depth = kwds.get('depth', False)
            dots = kwds.get('dots', False)
            join = kwds.get('join', False)
            verb = kwds.get('verb', False)

            # special case: bounds passed as list of slices
            if not isinstance(bounds, (basestring, type(None))):
                cmdargs = ''
                for b in bounds:
                    if isinstance(b, slice):
                        cmdargs += "{}:{}:{}, ".format(b.start, b.stop, b.step)
                    else:
                        cmdargs += "{}, ".format(b)
                bounds = cmdargs[:-2]
                cmdargs = ''

            if callable(reduce): _reducer, reduce = reduce, None

        # special case: model passed as model instance
       #model.__doc__.split('using::')[1].split()[0].strip()
        if callable(model): _model, model = model, "None"

        # handle logfile if given
        if logfile:
            if isinstance(logfile, basestring):
                model += ' ' + logfile
            else: # special case of passing in monitor instance
                instance = logfile

        # process "commandline" arguments
        if not cmdargs:
            cmdargs = ''
            cmdargs += '' if out is None else '--out={} '.format(out)
            cmdargs += '' if bounds is None else '--bounds="{}" '.format(bounds)
            cmdargs += '' if label is None else '--label={} '.format(label)
            cmdargs += '' if nid is None else '--nid={} '.format(nid)
            cmdargs += '' if iter is None else '--iter={} '.format(iter)
            cmdargs += '' if reduce is None else '--reduce={} '.format(reduce)
            cmdargs += '' if scale is None else '--scale={} '.format(scale)
            cmdargs += '' if shift is None else '--shift={} '.format(shift)
            cmdargs += '' if fill == False else '--fill '
            cmdargs += '' if depth == False else '--depth '
            cmdargs += '' if dots == False else '--dots '
            cmdargs += '' if join == False else '--join '
            cmdargs += '' if verb == False else '--verb '
        else:
            cmdargs = ' ' + cmdargs
        cmdargs = model.split() + shlex.split(cmdargs)

    #XXX: note that 'argparse' is new as of python2.7
    from optparse import OptionParser
    def _exit(self, errno=None, msg=None):
      global __quit
      __quit = True
      if errno or msg:
        msg = msg.split(': error: ')[-1].strip()
        raise IOError(msg)
    OptionParser.exit = _exit

    parser = OptionParser(usage=model_plotter.__doc__.split('\n\nOptions:')[0])
    parser.add_option("-u","--out",action="store",dest="out",\
                      metavar="STR",default=None,
                      help="filepath to save generated plot")
    parser.add_option("-b","--bounds",action="store",dest="bounds",\
                      metavar="STR",default="-5:5:.1, -5:5:.1",
                      help="indicator string to set plot bounds and density")
    parser.add_option("-l","--label",action="store",dest="label",\
                      metavar="STR",default=",,",
                      help="string to assign label to axis")
    parser.add_option("-n","--nid",action="store",dest="id",\
                      metavar="INT",default=None,
                      help="id # of the nth simultaneous points to plot")
    parser.add_option("-i","--iter",action="store",dest="stop",\
                      metavar="STR",default=":",
                      help="string for smallest:largest iterations to plot")
    parser.add_option("-r","--reduce",action="store",dest="reducer",\
                      metavar="STR",default="None",
                      help="import path of output reducer function")
    parser.add_option("-x","--scale",action="store",dest="zscale",\
                      metavar="INT",default=0.0,
                      help="scale plotted cost by z=log(4*z*scale+1)+2")
    parser.add_option("-z","--shift",action="store",dest="zshift",\
                      metavar="INT",default=0.0,
                      help="shift plotted cost by z=z+shift")
    parser.add_option("-f","--fill",action="store_true",dest="fill",\
                      default=False,help="plot using filled contours")
    parser.add_option("-d","--depth",action="store_true",dest="surface",\
                      default=False,help="plot contours showing depth in 3D")
    parser.add_option("-o","--dots",action="store_true",dest="dots",\
                      default=False,help="show trajectory points in plot")
    parser.add_option("-j","--join",action="store_true",dest="line",\
                      default=False,help="connect trajectory points in plot")
    parser.add_option("-v","--verb",action="store_true",dest="verbose",\
                      default=False,help="print model documentation string")

#   import sys
#   if 'mystic_model_plotter.py' not in sys.argv:
    from StringIO import StringIO
    f = StringIO()
    parser.print_help(file=f)
    f.seek(0)
    if 'Options:' not in model_plotter.__doc__:
      model_plotter.__doc__ += '\nOptions:%s' % f.read().split('Options:')[-1]
    f.close()

    try:
      parsed_opts, parsed_args = parser.parse_args(cmdargs)
    except UnboundLocalError:
      pass
    if __quit: return

    # get the import path for the model
    model = parsed_args[0]  # e.g. 'mystic.models.rosen'
    if "None" == model: model = None

    try: # get the name of the parameter log file
      source = parsed_args[1]  # e.g. 'log.txt'
    except:
      source = None

    try: # select the bounds
      options = parsed_opts.bounds  # format is "-1:10:.1, -1:10:.1, 1.0"
    except:
      options = "-5:5:.1, -5:5:.1"

    try: # plot using filled contours
      fill = parsed_opts.fill
    except:
      fill = False

    try: # plot contours showing depth in 3D
      surface = parsed_opts.surface
    except:
      surface = False

    #XXX: can't do '-x' with no argument given  (use T/F instead?)
    try: # scale plotted cost by z=log(4*z*scale+1)+2
      scale = float(parsed_opts.zscale)
      if not scale: scale = False
    except:
      scale = False

    #XXX: can't do '-z' with no argument given
    try: # shift plotted cost by z=z+shift
      shift = float(parsed_opts.zshift)
      if not shift: shift = False
    except:
      shift = False

    try: # import path of output reducer function
      reducer = parsed_opts.reducer  # e.g. 'numpy.add'
      if "None" == reducer: reducer = None
    except:
      reducer = None

    style = '-' # default linestyle
    if parsed_opts.dots:
      mark = 'o' # marker=mark
      # when using 'dots', also can turn off 'line'
      if not parsed_opts.line:
        style = '' # linestyle='None'
    else:
      mark = ''
    color = 'w' if fill else 'k'
    style = color + style + mark

    try: # select labels for the axes
      label = parsed_opts.label.split(',')  # format is "x, y, z"
    except:
      label = ['','','']

    try: # select which 'id' to plot results for
      ids = (int(parsed_opts.id),) #XXX: allow selecting more than one id ?
    except:
      ids = None # i.e. 'all'

    try: # select which iteration to stop plotting at
      stop = parsed_opts.stop  # format is "1:10:1"
      stop = stop if ":" in stop else ":"+stop
    except:
      stop = ":"

    try: # select whether to be verbose about model documentation
      verbose = bool(parsed_opts.verbose)
    except:
      verbose = False

    #################################################
    solver = None  # set to 'mystic.solvers.fmin' (or similar) for 'live' fits
    #NOTE: 'live' runs constrain params explicitly in the solver, then reduce
    #      dimensions appropriately so results can be 2D contour plotted.
    #      When working with legacy results that have more than 2 params,
    #      the trajectory WILL NOT follow the masked surface generated
    #      because the masked params were NOT fixed when the solver was run.
    #################################################

    from mystic.tools import reduced, masked, partial

    # process inputs
    if _model: model = _model
    if _reducer: reducer = _reducer
    if _solver: solver = _solver
    select, spec, mask = _parse_input(options)
    x,y = _parse_axes(spec, grid=True) # grid=False for 1D plots
    #FIXME: does grid=False still make sense here...?
    if reducer: reducer = _reducer or _get_instance(reducer)
    if solver and (not source or not model): #XXX: not instance?
        raise RuntimeError('a model and results filename are required')
    elif not source and not model and not instance:
        raise RuntimeError('a model or a results file is required')
    if model:
        model = _model or _get_instance(model)
        if verbose: print model.__doc__
        # need a reducer if model returns an array
        if reducer: model = reduced(reducer, arraylike=False)(model)

    if solver:
        # if 'live'... pick a solver
        solver = 'mystic.solvers.fmin'
        solver = _solver or _get_instance(solver)
        xlen = len(select)+len(mask)
        if solver.__name__.startswith('diffev'):
            initial = [(-1,1)]*xlen
        else:
            initial = [0]*xlen
        from mystic.monitors import VerboseLoggingMonitor
        if instance:
            itermon = VerboseLoggingMonitor(new=True)
            itermon.prepend(instance)
        else:
            itermon = VerboseLoggingMonitor(filename=source, new=True)
        # explicitly constrain parameters
        model = partial(mask)(model)
        # solve
        sol = solver(model, x0=initial, itermon=itermon)

        #-OVERRIDE-INPUTS-#
        import numpy
        # read trajectories from monitor (comment out to use logfile)
        source = itermon
        # if negative minimum, shift by the 'solved minimum' plus an epsilon
        shift = max(-numpy.min(itermon.y), 0.0) + 0.5 # a good guess
        #-----------------#

    if model: # for plotting, implicitly constrain by reduction
        model = masked(mask)(model)

       ## plot the surface in 1D
       #if solver: v=sol[-1]
       #elif source: v=cost[-1]
       #else: v=None
       #fig0 = _draw_slice(model, x=x, y=v, scale=scale, shift=shift)
        # plot the surface in 2D or 3D
        fig = _draw_contour(model, x, y, surface=surface, fill=fill, scale=scale, shift=shift)
    else:
       #fig0 = None
        fig = None

    if instance: source = instance
    if source:
        # params are the parameter trajectories
        # cost is the solution trajectory
        params, cost = _get_history(source, ids)
        if len(cost) > 1: style = style[1:] # 'auto-color' #XXX: or grayscale?

        for p,c in zip(params, cost):
           ## project trajectory on a 1D slice of model surface #XXX: useful?
           #s = select[0] if len(select) else 0
           #px = p[int(s)] # _draw_projection requires one parameter
           ## ignore everything after 'stop'
           #_c = eval('c[%s]' % stop)
           #_x = eval('px[%s]' % stop)
           #fig0 = _draw_projection(_x,_c, style=style, scale=scale, shift=shift, figure=fig0)

            # plot the trajectory on the model surface (2D or 3D)
            # get two selected params #XXX: what if len(select)<2? or len(p)<2?
            p = [p[int(i)] for i in select[:2]]
            px,py = p # _draw_trajectory requires two parameters
            # ignore everything after 'stop'
            _x = eval('px[%s]' % stop)
            _y = eval('py[%s]' % stop)
            _c = eval('c[%s]' % stop) if surface else None
            fig = _draw_trajectory(_x,_y,_c, style=style, scale=scale, shift=shift, figure=fig)

    import matplotlib.pyplot as plt
    from mpl_toolkits.mplot3d import axes3d
    # add labels to the axes
    if surface: kwds = {'projection':'3d'} # 3D
    else: kwds = {}                        # 2D
    ax = fig.gca(**kwds)
    ax.set_xlabel(label[0])
    ax.set_ylabel(label[1])
    if surface: ax.set_zlabel(label[2])

    if not parsed_opts.out:
        plt.show()
    else:
        fig.savefig(parsed_opts.out)

Example 45

Project: mystic
Source File: scripts.py
View license
def collapse_plotter(filename, **kwds):
    """
generate cost convergence rate plots from file written with 'write_support_file'

Available from the command shell as:
  mystic_collapse_plotter.py filename [options]

or as a function call as:
  mystic.collapse_plotter(filename, **options)

The option "col" takes a string of comma-separated integers indicating
iteration numbers where parameter collapse has occurred.  If a second set of
integers is provided (delineated by a semicolon), the additional set of integers
will be plotted with a different linestyle (to indicate a different type of
collapse).

The option "label" takes a label string. For example, label = "y"
will label the y-axis plot with 'y'. LaTeX is also accepted. For example,
label = " log-cost, $ log_{10}(\hat{P} - \hat{P}_{max})$" will label the
y-axis with standard LaTeX math formatting. Note that the leading space is
required, and the text is aligned along the axis.

Required Inputs:
  filename            name of the python convergence logfile (e.g paramlog.py)
"""
    import shlex
    global __quit
    __quit = False

    instance = None
    # handle the special case where list is provided by sys.argv
    if isinstance(filename, (list,tuple)) and not kwds:
        cmdargs = filename # (above is used by script to parse command line)
    elif isinstance(filename, basestring) and not kwds:
        cmdargs = shlex.split(filename)
    # 'everything else' is essentially the functional interface
    else:
        cmdargs = kwds.get('kwds', '')
        if not cmdargs:
            out = kwds.get('out', None)
            dots = kwds.get('dots', False)
           #line = kwds.get('line', False)
            linear = kwds.get('linear', False)
            iter = kwds.get('iter', None)
            label = kwds.get('label', None)
            col = kwds.get('col', None)

            # process "commandline" arguments
            cmdargs = ''
            cmdargs += '' if out is None else '--out={} '.format(out)
            cmdargs += '' if dots == False else '--dots '
            cmdargs += '' if linear == False else '--linear '
           #cmdargs += '' if line == False else '--line '
            cmdargs += '' if iter is None else '--iter={} '.format(iter)
            cmdargs += '' if label == None else '--label={} '.format(label)
            cmdargs += '' if col is None else '--col="{}" '.format(col)
        else:
            cmdargs = ' ' + cmdargs
        if isinstance(filename, basestring):
            cmdargs = filename.split() + shlex.split(cmdargs)
        else: # special case of passing in monitor instance
            instance = filename
            cmdargs = shlex.split(cmdargs)

    #XXX: note that 'argparse' is new as of python2.7
    from optparse import OptionParser
    def _exit(self, errno=None, msg=None):
      global __quit
      __quit = True
      if errno or msg:
        msg = msg.split(': error: ')[-1].strip()
        raise IOError(msg)
    OptionParser.exit = _exit

    parser = OptionParser(usage=collapse_plotter.__doc__.split('\n\nOptions:')[0])
    parser.add_option("-d","--dots",action="store_true",dest="dots",\
                      default=False,help="show data points in plot")
    #parser.add_option("-l","--line",action="store_true",dest="line",\
    #                  default=False,help="connect data points with a line")
    parser.add_option("-y","--linear",action="store_true",dest="linear",\
                      default=False,help="plot y-axis in linear scale")
    parser.add_option("-u","--out",action="store",dest="out",\
                      metavar="STR",default=None,
                      help="filepath to save generated plot")
    parser.add_option("-i","--iter",action="store",dest="stop",metavar="INT",\
                      default=None,help="the largest iteration to plot")
    parser.add_option("-l","--label",action="store",dest="label",\
                      metavar="STR",default="",\
                      help="string to assign label to y-axis")
    parser.add_option("-c","--col",action="store",dest="collapse",\
                      metavar="STR",default="",
                      help="string to indicate collapse indices")
#   import sys
#   if 'mystic_collapse_plotter.py' not in sys.argv:
    from StringIO import StringIO
    f = StringIO()
    parser.print_help(file=f)
    f.seek(0)
    if 'Options:' not in collapse_plotter.__doc__:
      collapse_plotter.__doc__ += '\nOptions:%s' % f.read().split('Options:')[-1]
    f.close()

    try:
      parsed_opts, parsed_args = parser.parse_args(cmdargs)
    except UnboundLocalError:
      pass
    if __quit: return

    style = '-' # default linestyle
    if parsed_opts.dots:
      mark = 'o'
      # when using 'dots', also turn off 'line'
      #if not parsed_opts.line:
      #  style = 'None'
    else:
      mark = ''

    try: # select labels for the axes
      label = parsed_opts.label  # format is "x" or " $x$"
    except:
      label = 'log-cost, $log_{10}(y - y_{min})$'

    try: # get logfile name
      filename = parsed_args[0]
    except:
      raise IOError, "please provide log file name"

    try: # select which iteration to stop plotting at
      stop = int(parsed_opts.stop)
    except:
      stop = None

    try: # select collapse boundaries to plot
      collapse = parsed_opts.collapse.split(';')  # format is "2, 3; 4, 5, 6; 7"
      collapse = [eval("(%s,)" % i) if i.strip() else () for i in collapse]
    except:
      collapse = []

    # read file
    from mystic.munge import read_history
    params, cost = read_history(filename)

    # ignore everything after 'stop'
    cost = cost[:stop]
    params = params[:stop]

    # get the minimum cost
    import numpy as np
    cost_min = min(cost)

    # convert to log scale
    x = np.arange(len(cost))
    settings = np.seterr(all='ignore')
    if parsed_opts.linear:
      y = np.array(cost)
     #y = np.abs(cost_min - np.array(cost))
    else:
      y = np.log10(np.abs(cost_min - np.array(cost)))
    np.seterr(**settings)

    import matplotlib.pyplot as plt
    fig = plt.figure()
    plt.plot(x, y, linestyle=style, marker=mark, markersize=1)

    colors = ['orange','red','brown','pink']
    linestyles = ['--','-.',':','-']

    for param,color,style in zip(collapse,colors,linestyles):
      for clps in set(param):
        plt.axvline(x=clps, ymin=-10, ymax=1, hold=None, linestyle=style, linewidth=param.count(clps), color=color)

    if label:
        #plt.title('convergence rate')
        plt.xlabel('iteration number, $n$')
        plt.ylabel(label)
        #plt.ylabel('$log-error,\; log_{10}(\hat{P} - \hat{P}_{max})$')

    if not parsed_opts.out:
        plt.show()
    else:
        fig.savefig(parsed_opts.out)

Example 46

Project: mystic
Source File: scripts.py
View license
def log_reader(filename, **kwds):
    """
plot parameter convergence from file written with 'LoggingMonitor'

Available from the command shell as:
  mystic_log_reader.py filename [options]

or as a function call as:
  mystic.log_reader(filename, **options)

The option "param" takes an indicator string. The indicator string is built
from comma-separated array slices. For example, params = ":" will plot all
parameters.  Alternatively, params = ":2, 3:" will plot all parameters except
for the third parameter, while params = "0" will only plot the first parameter.

Required Inputs:
  filename            name of the convergence logfile (e.g log.txt)
"""
    import shlex
    global __quit
    __quit = False

    instance = None
    # handle the special case where list is provided by sys.argv
    if isinstance(filename, (list,tuple)) and not kwds:
        cmdargs = filename # (above is used by script to parse command line)
    elif isinstance(filename, basestring) and not kwds:
        cmdargs = shlex.split(filename)
    # 'everything else' is essentially the functional interface
    else:
        cmdargs = kwds.get('kwds', '')
        if not cmdargs:
            out = kwds.get('out', None)
            dots = kwds.get('dots', False)
            line = kwds.get('line', False)
            iter = kwds.get('iter', None)
            legend = kwds.get('legend', False)
            nid = kwds.get('nid', None)
            param = kwds.get('param', None)

            # process "commandline" arguments
            cmdargs = ''
            cmdargs += '' if out is None else '--out={} '.format(out)
            cmdargs += '' if dots == False else '--dots '
            cmdargs += '' if line == False else '--line '
            cmdargs += '' if iter is None else '--iter={} '.format(iter)
            cmdargs += '' if legend == False else '--legend '
            cmdargs += '' if nid is None else '--nid={} '.format(nid)
            cmdargs += '' if param is None else '--param="{}" '.format(param)
        else:
            cmdargs = ' ' + cmdargs
        if isinstance(filename, basestring):
            cmdargs = filename.split() + shlex.split(cmdargs)
        else: # special case of passing in monitor instance
            instance = filename
            cmdargs = shlex.split(cmdargs)

    #XXX: note that 'argparse' is new as of python2.7
    from optparse import OptionParser
    def _exit(self, errno=None, msg=None):
      global __quit
      __quit = True
      if errno or msg:
        msg = msg.split(': error: ')[-1].strip()
        raise IOError(msg)
    OptionParser.exit = _exit

    parser = OptionParser(usage=log_reader.__doc__.split('\n\nOptions:')[0])
    parser.add_option("-u","--out",action="store",dest="out",\
                      metavar="STR",default=None,
                      help="filepath to save generated plot")
    parser.add_option("-d","--dots",action="store_true",dest="dots",\
                      default=False,help="show data points in plot")
    parser.add_option("-l","--line",action="store_true",dest="line",\
                      default=False,help="connect data points in plot with a line")
    parser.add_option("-i","--iter",action="store",dest="stop",metavar="INT",\
                      default=None,help="the largest iteration to plot")
    parser.add_option("-g","--legend",action="store_true",dest="legend",\
                      default=False,help="show the legend")
    parser.add_option("-n","--nid",action="store",dest="id",\
                      metavar="INT",default=None,
                      help="id # of the nth simultaneous points to plot")
    parser.add_option("-p","--param",action="store",dest="param",\
                      metavar="STR",default=":",
                      help="indicator string to select parameters")
    #parser.add_option("-f","--file",action="store",dest="filename",metavar="FILE",\
    #                  default='log.txt',help="log file name")

#   import sys
#   if 'mystic_log_reader.py' not in sys.argv:
    from StringIO import StringIO
    f = StringIO()
    parser.print_help(file=f)
    f.seek(0)
    if 'Options:' not in log_reader.__doc__:
      log_reader.__doc__ += '\nOptions:%s' % f.read().split('Options:')[-1]
    f.close()

    try:
      parsed_opts, parsed_args = parser.parse_args(cmdargs)
    except UnboundLocalError:
      pass
    if __quit: return

    style = '-' # default linestyle
    if parsed_opts.dots:
      mark = 'o'
      # when using 'dots', also can turn off 'line'
      if not parsed_opts.line:
        style = 'None'
    else:
      mark = ''

    try: # get logfile name
      if instance:
        filename = instance
      else:
        filename = parsed_args[0]
    except:
      raise IOError("please provide log file name")

    try: # select which iteration to stop plotting at
      stop = int(parsed_opts.stop)
    except:
      stop = None

    try: # select which 'id' to plot results for
      runs = (int(parsed_opts.id),) #XXX: allow selecting more than one id ?
    except:
      runs = None # i.e. 'all' **or** use id=0, which should be 'best' energy ?

    try: # select which parameters to plot
      select = parsed_opts.param.split(',')  # format is ":2, 2:4, 5, 6:"
    except:
      select = [':']

    # ensure all terms of select have a ":"
    for i in range(len(select)):
      if isinstance(select[i], int): select[i] = str(select[i])
      if select[i] == '-1': select[i] = 'len(params)-1:len(params)'
      elif not select[i].count(':'):
        select[i] += ':' + str(int(select[i])+1)


    # == Possible results ==
    # iter = (i,id) or (i,) 
    # split => { (i,) then (i+1,) } or { (i,) then (0,) }
    # y x = { float list } or { list [list1, ...] }

    # == Use Cases ==
    # (i,id) + { (i,) then (i+1,) } + { float list }
    # (i,) + { (i,) then (i+1,) } + { float list }
    # (i,id) + { (i,) then (i+1,) } + { list [list1, ...] }
    # (i,) + { (i,) then (i+1,) } + { list [list1, ...] }
    # (i,id) + { (i,) then (0,) } + { float list }
    # (i,) + { (i,) then (0,) } + { float list }
    # (i,id) + { (i,) then (0,) } + { list [list1, ...] }
    # (i,) + { (i,) then (0,) } + { list [list1, ...] }
    # NOTES:
    #   Legend is different for list versus [list1,...]
    #   Plot should be discontinuous for (i,) then (0,)

    # parse file contents to get (i,id), cost, and parameters
    try:
        instance = instance if instance else filename
        from mystic.munge import read_trajectories
        step, param, cost = read_trajectories(instance)
    except SyntaxError:
        from mystic.munge import read_raw_file
        read_raw_file(filename)
        msg = "incompatible file format, try 'support_convergence.py'"
        raise SyntaxError(msg)

    # ignore everything after 'stop'
    step = step[:stop]
    cost = cost[:stop]
    param = param[:stop]

    # split (i,id) into iteration and id
    multinode = len(step[0]) - 1  #XXX: what if step = []?
    iter = [i[0] for i in step]
    if multinode:
      id = [i[1] for i in step]
    else:
      id = [0 for i in step]

    # build the list of selected parameters
    params = range(len(param[0]))
    selected = []
    for i in select:
      selected.extend(eval("params[%s]" % i))
    selected = list(set(selected))

    results = [[] for i in range(max(id) + 1)]

    # populate results for each id with the corresponding (iter,cost,param)
    for i in range(len(id)):
      if runs is None or id[i] in runs: # take only the selected 'id'
        results[id[i]].append((iter[i],cost[i],param[i]))
    # NOTE: for example...  results = [[(0,...)],[(0,...),(1,...)],[],[(0,...)]]

    # build list of parameter (and cost) convergences for each id
    conv = []; cost_conv = []; iter_conv = []
    for i in range(len(results)):
      conv.append([])#; cost_conv.append([]); iter_conv.append([])
      if len(results[i]):
        for k in range(len(results[i][0][2])):
          conv[i].append([results[i][j][2][k] for j in range(len(results[i]))])
        cost_conv.append([results[i][j][1] for j in range(len(results[i]))])
        iter_conv.append([results[i][j][0] for j in range(len(results[i]))])
      else:
        conv[i] = [[] for k in range(len(param[0]))]
        cost_conv.append([])
        iter_conv.append([])

    #print "iter_conv = %s" % iter_conv
    #print "cost_conv = %s" % cost_conv
    #print "conv = %s" % conv

    import matplotlib.pyplot as plt
    fig = plt.figure()

    #FIXME: These may fail when conv[i][j] = [[],[],[]] and cost = []. Verify this.
    ax1 = fig.add_subplot(2,1,1)
    for i in range(len(conv)):
      if runs is None or i in runs: # take only the selected 'id'
        for j in range(len(param[0])):
          if j in selected: # take only the selected 'params'
            tag = "%d,%d" % (j,i) # label is 'parameter,id'
            ax1.plot(iter_conv[i],conv[i][j],label="%s" % tag,marker=mark,linestyle=style)
    if parsed_opts.legend: plt.legend()

    ax2 = fig.add_subplot(2,1,2)
    for i in range(len(conv)):
      if runs is None or i in runs: # take only the selected 'id'
        tag = "%d" % i # label is 'cost id'
        ax2.plot(iter_conv[i],cost_conv[i],label='cost %s' % tag,marker=mark,linestyle=style)
    if parsed_opts.legend: plt.legend()

    if not parsed_opts.out:
        plt.show()
    else:
        fig.savefig(parsed_opts.out)

Example 47

Project: fwlite
Source File: cgi.py
View license
    def __init__(self, fp=None, headers=None, outerboundary="",
                 environ=os.environ, keep_blank_values=0, strict_parsing=0):
        """Constructor.  Read multipart/* until last part.

        Arguments, all optional:

        fp              : file pointer; default: sys.stdin
            (not used when the request method is GET)

        headers         : header dictionary-like object; default:
            taken from environ as per CGI spec

        outerboundary   : terminating multipart boundary
            (for internal use only)

        environ         : environment dictionary; default: os.environ

        keep_blank_values: flag indicating whether blank values in
            percent-encoded forms should be treated as blank strings.
            A true value indicates that blanks should be retained as
            blank strings.  The default false value indicates that
            blank values are to be ignored and treated as if they were
            not included.

        strict_parsing: flag indicating what to do with parsing errors.
            If false (the default), errors are silently ignored.
            If true, errors raise a ValueError exception.

        """
        method = 'GET'
        self.keep_blank_values = keep_blank_values
        self.strict_parsing = strict_parsing
        if 'REQUEST_METHOD' in environ:
            method = environ['REQUEST_METHOD'].upper()
        self.qs_on_post = None
        if method == 'GET' or method == 'HEAD':
            if 'QUERY_STRING' in environ:
                qs = environ['QUERY_STRING']
            elif sys.argv[1:]:
                qs = sys.argv[1]
            else:
                qs = ""
            fp = StringIO(qs)
            if headers is None:
                headers = {'content-type':
                           "application/x-www-form-urlencoded"}
        if headers is None:
            headers = {}
            if method == 'POST':
                # Set default content-type for POST to what's traditional
                headers['content-type'] = "application/x-www-form-urlencoded"
            if 'CONTENT_TYPE' in environ:
                headers['content-type'] = environ['CONTENT_TYPE']
            if 'QUERY_STRING' in environ:
                self.qs_on_post = environ['QUERY_STRING']
            if 'CONTENT_LENGTH' in environ:
                headers['content-length'] = environ['CONTENT_LENGTH']
        self.fp = fp or sys.stdin
        self.headers = headers
        self.outerboundary = outerboundary

        # Process content-disposition header
        cdisp, pdict = "", {}
        if 'content-disposition' in self.headers:
            cdisp, pdict = parse_header(self.headers['content-disposition'])
        self.disposition = cdisp
        self.disposition_options = pdict
        self.name = None
        if 'name' in pdict:
            self.name = pdict['name']
        self.filename = None
        if 'filename' in pdict:
            self.filename = pdict['filename']

        # Process content-type header
        #
        # Honor any existing content-type header.  But if there is no
        # content-type header, use some sensible defaults.  Assume
        # outerboundary is "" at the outer level, but something non-false
        # inside a multi-part.  The default for an inner part is text/plain,
        # but for an outer part it should be urlencoded.  This should catch
        # bogus clients which erroneously forget to include a content-type
        # header.
        #
        # See below for what we do if there does exist a content-type header,
        # but it happens to be something we don't understand.
        if 'content-type' in self.headers:
            ctype, pdict = parse_header(self.headers['content-type'])
        elif self.outerboundary or method != 'POST':
            ctype, pdict = "text/plain", {}
        else:
            ctype, pdict = 'application/x-www-form-urlencoded', {}
        self.type = ctype
        self.type_options = pdict
        self.innerboundary = ""
        if 'boundary' in pdict:
            self.innerboundary = pdict['boundary']
        clen = -1
        if 'content-length' in self.headers:
            try:
                clen = int(self.headers['content-length'])
            except ValueError:
                pass
            if maxlen and clen > maxlen:
                raise ValueError, 'Maximum content length exceeded'
        self.length = clen

        self.list = self.file = None
        self.done = 0
        if ctype == 'application/x-www-form-urlencoded':
            self.read_urlencoded()
        elif ctype[:10] == 'multipart/':
            self.read_multi(environ, keep_blank_values, strict_parsing)
        else:
            self.read_single()

Example 48

Project: wikiteam
Source File: uploader.py
View license
def upload(wikis, config={}):
    headers = {'User-Agent': dumpgenerator.getUserAgent()}

    for wiki in wikis:
        print "#"*73
        print "# Uploading", wiki
        print "#"*73
        wiki = wiki.lower()
        prefix = dumpgenerator.domain2prefix(config={'api': wiki})

        wikiname = prefix.split('-')[0]
        dumps = []
        for dirname, dirnames, filenames in os.walk('.'):
            if dirname == '.':
                for f in filenames:
                    if f.startswith('%s-' % (wikiname)) and (f.endswith('-wikidump.7z') or f.endswith('-history.xml.7z')):
                        dumps.append(f)
                break

        c = 0
        for dump in dumps:
            wikidate = dump.split('-')[1]
            item = get_item('wiki-' + wikiname)
            if dump in uploadeddumps:
                if config['prune-directories']:
                    rmline='rm -rf %s-%s-wikidump/' % (wikiname, wikidate)
                    # With -f the deletion might have happened before and we won't know
                    if not os.system(rmline):
                        print 'DELETED %s-%s-wikidump/' % (wikiname, wikidate)
                if config['prune-wikidump'] and dump.endswith('wikidump.7z'):
                        # Simplistic quick&dirty check for the presence of this file in the item
                        stdout, stderr = subprocess.Popen(["md5sum", dump], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
                        dumphash = re.sub(' +.+\n?', '', stdout)

                        if dumphash in map(lambda x: x['md5'], item.files):
                            log(wiki, dump, 'verified')
                            rmline='rm -rf %s' % dump
                            if not os.system(rmline):
                                print 'DELETED ' + dump
                            print '%s was uploaded before, skipping...' % (dump)
                            continue
                        else:
                            print 'ERROR: The online item misses ' + dump
                            log(wiki, dump, 'missing')
                            # We'll exit this if and go upload the dump
                else:
                    print '%s was uploaded before, skipping...' % (dump)
                    continue

            time.sleep(0.1)
            wikidate_text = wikidate[0:4]+'-'+wikidate[4:6]+'-'+wikidate[6:8]
            print wiki, wikiname, wikidate, dump

            # Does the item exist already?
            ismissingitem = not item.exists

            # Logo path
            logourl = ''

            if ismissingitem or config['update']:
                #get metadata from api.php
                #first sitename and base url
                params = {'action': 'query', 'meta': 'siteinfo', 'format': 'xml'}
                data = urllib.urlencode(params)
                req = urllib2.Request(url=wiki, data=data, headers=headers)
                xml = ''
                try:
                    f = urllib2.urlopen(req)
                    xml = f.read()
                    f.close()
                except:
                    pass

                sitename = ''
                baseurl = ''
                lang = ''
                try:
                    sitename = re.findall(ur"sitename=\"([^\"]+)\"", xml)[0]
                except:
                    pass
                try:
                    baseurl = re.findall(ur"base=\"([^\"]+)\"", xml)[0]
                except:
                    pass
                try:
                    lang = re.findall(ur"lang=\"([^\"]+)\"", xml)[0]
                except:
                    pass

                if not sitename:
                    sitename = wikiname
                if not baseurl:
                    baseurl = re.sub(ur"(?im)/api\.php", ur"", wiki)
                if lang:
                    lang = convertlang.has_key(lang.lower()) and convertlang[lang.lower()] or lang.lower()

                #now copyright info from API
                params = {'action': 'query', 'siprop': 'general|rightsinfo', 'format': 'xml'}
                data = urllib.urlencode(params)
                req = urllib2.Request(url=wiki, data=data, headers=headers)
                xml = ''
                try:
                    f = urllib2.urlopen(req)
                    xml = f.read()
                    f.close()
                except:
                    pass

                rightsinfourl = ''
                rightsinfotext = ''
                try:
                    rightsinfourl = re.findall(ur"rightsinfo url=\"([^\"]+)\"", xml)[0]
                    rightsinfotext = re.findall(ur"text=\"([^\"]+)\"", xml)[0]
                except:
                    pass

                raw = ''
                try:
                    f = urllib.urlopen(baseurl)
                    raw = f.read()
                    f.close()
                except:
                    pass

                #or copyright info from #footer in mainpage
                if baseurl and not rightsinfourl and not rightsinfotext:
                    rightsinfotext = ''
                    rightsinfourl = ''
                    try:
                        rightsinfourl = re.findall(ur"<link rel=\"copyright\" href=\"([^\"]+)\" />", raw)[0]
                    except:
                        pass
                    try:
                        rightsinfotext = re.findall(ur"<li id=\"copyright\">([^\n\r]*?)</li>", raw)[0]
                    except:
                        pass
                    if rightsinfotext and not rightsinfourl:
                        rightsinfourl = baseurl + '#footer'
                try:
                    logourl = re.findall(ur'p-logo["\'][^>]*>\s*<a [^>]*background-image:\s*(?:url\()?([^;)"]+)', raw)[0]
                except:
                    pass
                print logourl

                #retrieve some info from the wiki
                wikititle = "Wiki - %s" % (sitename) # Wiki - ECGpedia
                wikidesc = "<a href=\"%s\">%s</a> dumped with <a href=\"https://github.com/WikiTeam/wikiteam\" rel=\"nofollow\">WikiTeam</a> tools." % (baseurl, sitename)# "<a href=\"http://en.ecgpedia.org/\" rel=\"nofollow\">ECGpedia,</a>: a free electrocardiography (ECG) tutorial and textbook to which anyone can contribute, designed for medical professionals such as cardiac care nurses and physicians. Dumped with <a href=\"https://github.com/WikiTeam/wikiteam\" rel=\"nofollow\">WikiTeam</a> tools."
                wikikeys = ['wiki', 'wikiteam', 'MediaWiki', sitename, wikiname] # ecg; ECGpedia; wiki; wikiteam; MediaWiki
                if not rightsinfourl and not rightsinfotext:
                    wikikeys.append('unknowncopyright')

                wikilicenseurl = rightsinfourl # http://creativecommons.org/licenses/by-nc-sa/3.0/
                wikirights = rightsinfotext # e.g. http://en.ecgpedia.org/wiki/Frequently_Asked_Questions : hard to fetch automatically, could be the output of API's rightsinfo if it's not a usable licenseurl or "Unknown copyright status" if nothing is found.
                wikiurl = wiki # we use api here http://en.ecgpedia.org/api.php
            else:
                print 'Item already exists.'
                lang = 'foo'
                wikititle = 'foo'
                wikidesc = 'foo'
                wikikeys = 'foo'
                wikilicenseurl = 'foo'
                wikirights = 'foo'
                wikiurl = 'foo'

            if c == 0:
                # Item metadata
                md = {
                    'mediatype': 'web',
                    'collection': config['collection'],
                    'title': wikititle,
                    'description': wikidesc,
                    'language': lang,
                    'last-updated-date': wikidate_text,
                    'subject': '; '.join(wikikeys), # Keywords should be separated by ; but it doesn't matter much; the alternative is to set one per field with subject[0], subject[1], ...
                    'licenseurl': wikilicenseurl and urlparse.urljoin(wiki, wikilicenseurl),
                    'rights': wikirights,
                    'originalurl': wikiurl,
                }

            #Upload files and update metadata
            try:
                item.upload(dump, metadata=md, access_key=accesskey, secret_key=secretkey, verbose=True)
                item.modify_metadata(md) # update
                print 'You can find it in https://archive.org/details/wiki-%s' % (wikiname)
                if logourl:
                    logo = StringIO.StringIO(urllib.urlopen(urlparse.urljoin(wiki, logourl)).read())
                    logoextension = logourl.split('.')[-1] if logourl.split('.') else 'unknown'
                    logo.name = 'wiki-' + wikiname + '_logo.' + logoextension
                    item.upload(logo, access_key=accesskey, secret_key=secretkey, verbose=True)
                uploadeddumps.append(dump)
                log(wiki, dump, 'ok')
            except:
                print wiki, dump, 'error when uploading?'

            c += 1

Example 49

Project: bigmler
Source File: args.py
View license
def get_output_args(api, command_args, resume):
    """Returns the output args needed for the main bigmler computation process

    """
    try:
        if command_args.train_stdin:
            if command_args.test_stdin:
                sys.exit("The standard input can't be used both for training "
                         "and testing. Choose one of them")
            command_args.training_set = StringIO(sys.stdin.read())
        elif command_args.test_stdin:
            command_args.test_set = StringIO(sys.stdin.read())
    except AttributeError:
        pass

    try:
        if command_args.objective_field:
            objective = command_args.objective_field
            try:
                command_args.objective_field = int(objective)
            except ValueError:
                if not command_args.train_header:
                    sys.exit("The %s has been set as objective field but"
                             " the file has not been marked as containing"
                             " headers.\nPlease set the --train-header flag if"
                             " the file has headers or use a column number"
                             " to set the objective field." % objective)
    except AttributeError:
        pass

    command_args.resume_ = resume

    # Reads description if provided.
    try:
        if command_args.description:
            description_arg = u.read_description(command_args.description)
            command_args.description_ = description_arg
        else:
            command_args.description_ = DEFAULT_DESCRIPTION
    except AttributeError:
        pass

    # Parses fields if provided.
    try:
        if command_args.field_attributes:
            field_attributes_arg = (
                u.read_field_attributes(command_args.field_attributes))
            command_args.field_attributes_ = field_attributes_arg
        else:
            command_args.field_attributes_ = []
    except AttributeError:
        pass
    try:
        if command_args.test_field_attributes:
            field_attributes_arg = (
                u.read_field_attributes(command_args.test_field_attributes))
            command_args.test_field_attributes_ = field_attributes_arg
        else:
            command_args.test_field_attributes_ = []
    except AttributeError:
        pass

    # Parses types if provided.
    try:
        if command_args.types:
            types_arg = u.read_types(command_args.types)
            command_args.types_ = types_arg
        else:
            command_args.types_ = None
        if command_args.test_types:
            types_arg = u.read_types(command_args.test_types)
            command_args.test_types_ = types_arg
        else:
            command_args.test_types_ = None
    except AttributeError:
        pass


    # Parses dataset fields if provided.
    try:
        if command_args.dataset_fields:
            dataset_fields_arg = [
                field.strip() for field in command_args.dataset_fields.split(
                    command_args.args_separator)]
            command_args.dataset_fields_ = dataset_fields_arg
        else:
            command_args.dataset_fields_ = []
    except AttributeError:
        pass

    # Parses model input fields if provided.
    try:
        if command_args.model_fields:
            model_fields_arg = [
                field.strip() for field in command_args.model_fields.split(
                    command_args.args_separator)]
            command_args.model_fields_ = model_fields_arg
        else:
            command_args.model_fields_ = []
    except AttributeError:
        pass

    # Parses cluster input fields if provided.
    try:
        if command_args.cluster_fields:
            cluster_fields_arg = [
                field.strip() for field in command_args.cluster_fields.split(
                    command_args.args_separator)]
            command_args.cluster_fields_ = cluster_fields_arg
        else:
            command_args.cluster_fields_ = []
    except AttributeError:
        pass


    # Parses association input fields if provided.
    try:
        if command_args.association_fields:
            association_fields_arg = [
                field.strip() for field in \
                command_args.association_fields.split( \
                command_args.args_separator)]
            command_args.association_fields_ = association_fields_arg
        else:
            command_args.association_fields_ = []
    except AttributeError:
        pass

    # Parses anomaly input fields if provided.
    try:
        if command_args.anomaly_fields:
            anomaly_fields_arg = [
                field.strip() for field in command_args.anomaly_fields.split(
                    command_args.args_separator)]
            command_args.anomaly_fields_ = anomaly_fields_arg
        else:
            command_args.anomaly_fields_ = []
    except AttributeError:
        pass

    # Parses logistic regression input fields if provided.
    try:
        if command_args.logistic_fields:
            logistic_fields_arg = [
                field.strip() for field in command_args.logistic_fields.split(
                    command_args.args_separator)]
            command_args.logistic_fields_ = logistic_fields_arg
        else:
            command_args.logistic_fields_ = []
    except AttributeError:
        pass

    # Parses field_codings for logistic regressions
    try:
        if command_args.field_codings:
            command_args.field_codings_ = u.read_json(
                command_args.field_codings)
        else:
            command_args.field_codings_ = []
    except AttributeError:
        pass

    # Parses imports for scripts and libraries.
    try:
        if command_args.imports:
            imports_arg = [
                field.strip() for field in command_args.imports.split(
                    command_args.args_separator)]
            command_args.imports_ = imports_arg
        else:
            command_args.imports_ = []
    except AttributeError:
        pass

    # Parses parameters for scripts.
    try:
        if command_args.declare_inputs:
            command_args.parameters_ = u.read_json(command_args.declare_inputs)
        else:
            command_args.parameters_ = []
    except AttributeError:
        pass

    # Parses creation_defaults for executions.
    try:
        if command_args.creation_defaults:
            command_args.creation_defaults_ = u.read_json(
                command_args.creation_defaults)
        else:
            command_args.creation_defaults_ = {}
    except AttributeError:
        pass

    # Parses arguments for executions.
    try:
        if command_args.inputs:
            command_args.arguments_ = u.read_json(command_args.inputs)
        else:
            command_args.arguments_ = []
    except AttributeError:
        pass

    # Parses input maps for executions.
    try:
        if command_args.input_maps:
            command_args.input_maps_ = u.read_json(command_args.input_maps)
        else:
            command_args.input_maps_ = []
    except AttributeError:
        pass

    # Parses outputs for executions.
    try:
        if command_args.outputs:
            command_args.outputs_ = u.read_json(command_args.outputs)
        else:
            command_args.outputs_ = []
    except AttributeError:
        pass

    # Parses outputs for scripts.
    try:
        if command_args.declare_outputs:
            command_args.declare_outputs_ = \
                u.read_json(command_args.declare_outputs)
        else:
            command_args.declare_outputs_ = []
    except AttributeError:
        pass

    model_ids = []
    try:
        # Parses model/ids if provided.
        if command_args.models:
            model_ids = u.read_resources(command_args.models)
        command_args.model_ids_ = model_ids
    except AttributeError:
        pass

    # Retrieve model/ids if provided.
    try:
        if command_args.model_tag:
            model_ids = (model_ids +
                         u.list_ids(api.list_models,
                                    "tags__in=%s" % command_args.model_tag))
        command_args.model_ids_ = model_ids
    except AttributeError:
        pass

    # Reads votes files in the provided directories.
    try:
        if command_args.votes_dirs:
            dirs = [
                directory.strip() for directory in
                command_args.votes_dirs.split(
                    command_args.args_separator)]
            votes_path = os.path.dirname(command_args.predictions)
            votes_files = u.read_votes_files(dirs, votes_path)
            command_args.votes_files_ = votes_files
        else:
            command_args.votes_files_ = []
    except AttributeError:
        pass

    # Parses fields map if provided.
    try:
        if command_args.fields_map:
            fields_map_arg = u.read_fields_map(command_args.fields_map)
            command_args.fields_map_ = fields_map_arg
        else:
            command_args.fields_map_ = None
    except AttributeError:
        pass

    cluster_ids = []
    try:
        # Parses cluster/ids if provided.
        if command_args.clusters:
            cluster_ids = u.read_resources(command_args.clusters)
        command_args.cluster_ids_ = cluster_ids
    except AttributeError:
        pass

    # Retrieve cluster/ids if provided.
    try:
        if command_args.cluster_tag:
            cluster_ids = (cluster_ids +
                           u.list_ids(api.list_clusters,
                                      "tags__in=%s" %
                                      command_args.cluster_tag))
        command_args.cluster_ids_ = cluster_ids
    except AttributeError:
        pass

    association_ids = []
    try:
        # Parses association/ids if provided.
        if command_args.associations:
            association_ids = u.read_resources(command_args.associations)
        command_args.association_ids_ = association_ids
    except AttributeError:
        pass

    # Retrieve association/ids if provided.
    try:
        if command_args.association_tag:
            association_ids = (association_ids +
                               u.list_ids(api.list_associations,
                                          "tags__in=%s" %
                                          command_args.association_tag))
        command_args.association_ids_ = association_ids
    except AttributeError:
        pass

    logistic_regression_ids = []
    try:
        # Parses logisticregression/ids if provided.
        if command_args.logistic_regressions:
            logistic_regression_ids = u.read_resources( \
                command_args.logistic_regressions)
        command_args.logistic_regression_ids_ = logistic_regression_ids
    except AttributeError:
        pass

    # Retrieve logisticregression/ids if provided.
    try:
        if command_args.logistic_tag:
            logistic_regression_ids = (logistic_regression_ids + \
                u.list_ids(api.list_logistic_regressions,
                           "tags__in=%s" % command_args.logistic_tag))
        command_args.logistic_regression_ids_ = logistic_regression_ids
    except AttributeError:
        pass

    # Parses cluster names to generate datasets if provided
    try:
        if command_args.cluster_datasets:
            cluster_datasets_arg = [
                dataset.strip() for dataset in
                command_args.cluster_datasets.split(
                    command_args.args_separator)]
            command_args.cluster_datasets_ = cluster_datasets_arg
        else:
            command_args.cluster_datasets_ = []
    except AttributeError:
        pass

    # Parses cluster names to generate models if provided
    try:
        if command_args.cluster_models:
            cluster_models_arg = [
                model.strip() for model in
                command_args.cluster_models.split(
                    command_args.args_separator)]
            command_args.cluster_models_ = cluster_models_arg
        else:
            command_args.cluster_models_ = []
    except AttributeError:
        pass

    # Parses summary_fields to exclude from the clustering algorithm
    try:
        if command_args.summary_fields:
            summary_fields_arg = [
                field.strip() for field in
                command_args.summary_fields.split(
                    command_args.args_separator)]
            command_args.summary_fields_ = summary_fields_arg
        else:
            command_args.summary_fields_ = []
    except AttributeError:
        pass

    anomaly_ids = []
    try:
        # Parses anomaly/ids if provided.
        if command_args.anomalies:
            anomaly_ids = u.read_resources(command_args.anomalies)
        command_args.anomaly_ids_ = anomaly_ids
    except AttributeError:
        pass

    # Retrieve anomaly/ids if provided.
    try:
        if command_args.anomaly_tag:
            anomaly_ids = (anomaly_ids +
                           u.list_ids(api.list_anomalies,
                                      "tags__in=%s" %
                                      command_args.anomaly_tag))
        command_args.anomaly_ids_ = anomaly_ids
    except AttributeError:
        pass

    sample_ids = []
    try:
        # Parses sample/ids if provided.
        if command_args.samples:
            sample_ids = u.read_resources(command_args.samples)
        command_args.sample_ids_ = sample_ids
    except AttributeError:
        pass

    # Retrieve sample/ids if provided.
    try:
        if command_args.sample_tag:
            sample_ids = (
                sample_ids + u.list_ids(api.list_samples,
                                        "tags__in=%s" %
                                        command_args.sample_tag))
        command_args.sample_ids_ = sample_ids
    except AttributeError:
        pass

    # Parses sample row fields
    try:
        if command_args.row_fields:
            row_fields_arg = [field.strip() for field in
                              command_args.row_fields.split(
                                  command_args.args_separator)]
            command_args.row_fields_ = row_fields_arg
        else:
            command_args.row_fields_ = []
    except AttributeError:
        pass

    # Parses sample stat_fields
    try:
        if command_args.stat_fields:
            stat_fields_arg = [field.strip() for field in
                               command_args.stat_fields.split(
                                   command_args.args_separator)]
            command_args.stat_fields_ = stat_fields_arg
        else:
            command_args.stat_fields_ = []
    except AttributeError:
        pass

    return {"api": api, "args": command_args}

Example 50

Project: python
Source File: bigmlconnection.py
View license
    def _download(self, url, filename=None, wait_time=10, retries=10,
                  counter=0):
        """Retrieves a remote file.

        Uses HTTP GET to download a file object with a BigML `url`.
        """
        code = HTTP_INTERNAL_SERVER_ERROR
        file_object = None

        # if retries for the creation and download have been exhausted,
        # return None
        if counter > 2 * retries:
            LOGGER.error("Retries exhausted trying to download the file.")
            return file_object
        if GAE_ENABLED:
            try:
                req_options = {
                    'url': url + self.auth,
                    'method': urlfetch.GET,
                    'validate_certificate': self.verify
                }
                response = urlfetch.fetch(**req_options)
            except urlfetch.Error, exception:
                LOGGER.error("HTTP request error: %s",
                             str(exception))
                return file_object
        else:
            try:
                response = requests.get(url + self.auth,
                                        verify=self.verify, stream=True)
            except (requests.ConnectionError,
                    requests.Timeout,
                    requests.RequestException), exc:
                LOGGER.error("HTTP request error: %s", str(exc))
                return file_object
        try:
            code = response.status_code
            if code == HTTP_OK:
                # starting the dataset export procedure
                if response.headers.get("content-type") == JSON_TYPE:
                    try:
                        if counter < retries:
                            download_status = json_load(response.content)
                            if download_status and isinstance(download_status,
                                                              dict):
                                if download_status['status']['code'] != 5:
                                    time.sleep(get_exponential_wait(wait_time,
                                                                    counter))
                                    counter += 1
                                    return self._download(url,
                                                          filename=filename,
                                                          wait_time=wait_time,
                                                          retries=retries,
                                                          counter=counter)
                                else:
                                    return self._download(url,
                                                          filename=filename,
                                                          wait_time=wait_time,
                                                          retries=retries,
                                                          counter=retries + 1)
                        elif counter == retries:
                            LOGGER.error("The maximum number of retries "
                                         " for the download has been "
                                         " exceeded. You can retry your "
                                         " command again in"
                                         " a while.")
                            return None
                    except ValueError:
                        LOGGER.error("Failed getting a valid JSON structure.")
                else:
                    # When download starts, content-type is no longer a
                    # JSON object.
                    if filename is not None and GAE_ENABLED:
                        LOGGER.error("No support for downloading"
                                     " to local files in Google App Engine.")
                        filename = None
                    if filename is None:
                        if GAE_ENABLED:
                            file_object = StringIO.StringIO(response.content)
                        else:
                            file_object = response.raw
                    else:
                        try:
                            total_size = int(
                                response.headers.get("content-length"))
                        except ValueError:
                            total_size = None
                        file_size = stream_copy(response, filename)
                        if file_size == 0:
                            LOGGER.error("Error copying file to %s", filename)
                        else:
                            file_object = filename
                        # if transient connection errors prevent the download,
                        # retry
                        if total_size is None or file_size < total_size:
                            LOGGER.error("Error downloading: "
                                         "total size=%s, %s downloaded",
                                         total_size, file_size)
                            time.sleep(get_exponential_wait(wait_time,
                                                            counter))
                            return self._download(url, filename=filename,
                                                  wait_time=wait_time,
                                                  retries=retries,
                                                  counter=counter + 1)
            elif code in [HTTP_BAD_REQUEST,
                          HTTP_UNAUTHORIZED,
                          HTTP_NOT_FOUND,
                          HTTP_TOO_MANY_REQUESTS]:
                error = response.content
                LOGGER.error("Error downloading: %s", error)
            else:
                LOGGER.error("Unexpected error (%s)", code)
                code = HTTP_INTERNAL_SERVER_ERROR
        except ValueError:
            LOGGER.error("Malformed response")

        return file_object