os.walk

Here are the examples of the python api os.walk taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: scandir
Source File: benchmark.py
View license
def benchmark(path, get_size=False):
    sizes = {}

    if get_size:
        def do_os_walk():
            size = 0
            for root, dirs, files in os.walk(path):
                for filename in files:
                    fullname = os.path.join(root, filename)
                    st = os.lstat(fullname)
                    if not stat.S_ISLNK(st.st_mode):
                        size += st.st_size
            sizes['os_walk'] = size

        def do_scandir_walk():
            sizes['scandir_walk'] = get_tree_size(path)

    else:
        def do_os_walk():
            for root, dirs, files in os.walk(path):
                pass

        def do_scandir_walk():
            for root, dirs, files in scandir.walk(path):
                pass

    # Run this once first to cache things, so we're not benchmarking I/O
    print("Priming the system's cache...")
    do_scandir_walk()

    # Use the best of 3 time for each of them to eliminate high outliers
    os_walk_time = 1000000
    scandir_walk_time = 1000000
    N = 3
    for i in range(N):
        print('Benchmarking walks on {0}, repeat {1}/{2}...'.format(
            path, i + 1, N))
        os_walk_time = min(os_walk_time, timeit.timeit(do_os_walk, number=1))
        scandir_walk_time = min(scandir_walk_time,
                                timeit.timeit(do_scandir_walk, number=1))

    if get_size:
        if sizes['os_walk'] == sizes['scandir_walk']:
            equality = 'equal'
        else:
            equality = 'NOT EQUAL!'
        print('os.walk size {0}, scandir.walk size {1} -- {2}'.format(
            sizes['os_walk'], sizes['scandir_walk'], equality))

    print('os.walk took {0:.3f}s, scandir.walk took {1:.3f}s -- {2:.1f}x as fast'.format(
          os_walk_time, scandir_walk_time, os_walk_time / scandir_walk_time))

Example 2

Project: pootle
Source File: project_tree.py
View license
def translation_project_dir_exists(language, project):
    """Tests if there are translation files corresponding to the given
    :param:`language` and :param:`project`.
    """
    if project.get_treestyle() == "gnu":
        # GNU style projects are tricky

        if language.code == 'templates':
            # Language is template look for template files
            for dirpath_, dirnames, filenames in os.walk(
                    project.get_real_path()):
                for filename in filenames:
                    if (project.file_belongs_to_project(filename,
                                                        match_templates=True)
                            and match_template_filename(project, filename)):
                        return True
        else:
            # find files with the language name in the project dir
            for dirpath_, dirnames, filenames in os.walk(
                    project.get_real_path()):
                for filename in filenames:
                    # FIXME: don't reuse already used file
                    filename_matches = (
                        project.file_belongs_to_project(
                            filename,
                            match_templates=False)
                        and direct_language_match_filename(
                            project.lang_mapper.get_upstream_code(language.code),
                            filename))
                    if filename_matches:
                        return True
    else:
        # find directory with the language name in the project dir
        try:
            dirpath_, dirnames, filename = os.walk(
                project.get_real_path()).next()
            lang_code = project.lang_mapper.get_upstream_code(language.code)
            if lang_code in dirnames:
                return True
        except StopIteration:
            pass

    return False

Example 3

Project: Veil-Evasion
Source File: shellcode.py
View license
    def LoadModules(self):
        """
        Crawls the metasploit install tree and extracts available payloads
        and their associated required options for langauges specified.

        """

        # Variable changed for compatibility with  non-root and non-Kali users
        # Thanks to Tim Medin for the patch
        msfFolder = settings.METASPLOIT_PATH

        # I can haz multiple platforms?
        platforms = ["windows"]

        for platform in platforms:
            self.payloadTree[platform] = {}

            stagesX86 = list()
            stagersX86 = list()
            stagesX64 = list()
            stagersX64 = list()

            # load up all the stages (meterpreter/vnc/etc.)
            # TODO: detect Windows and modify the paths appropriately
            for root, dirs, files in os.walk(settings.METASPLOIT_PATH + "/modules/payloads/stages/" + platform + "/"):
                for f in files:
                    stageName = f.split(".")[0]
                    if "x64" in root:
                        stagesX64.append(f.split(".")[0])
                        if "x64" not in self.payloadTree[platform]:
                            self.payloadTree[platform]["x64"] = {}
                        self.payloadTree[platform]["x64"][stageName] = {}
                    elif "x86" in root: # linux payload structure format
                        stagesX86.append(f.split(".")[0])
                        if "x86" not in self.payloadTree[platform]:
                            self.payloadTree[platform]["x86"] = {}
                        self.payloadTree[platform]["x86"][stageName] = {}
                    else: # windows payload structure format
                        stagesX86.append(f.split(".")[0])
                        if stageName not in self.payloadTree[platform]:
                            self.payloadTree[platform][stageName] = {}

            # load up all the stagers (reverse_tcp, bind_tcp, etc.)
            # TODO: detect Windows and modify the paths appropriately
            for root, dirs, files in os.walk(settings.METASPLOIT_PATH + "/modules/payloads/stagers/" + platform + "/"):
                for f in files:

                    if ".rb" in f:
                        extraOptions = list()
                        moduleName = f.split(".")[0]
                        lines = open(root + "/" + f).readlines()
                        for line in lines:
                            if "OptString" in line.strip() and "true" in line.strip():
                                cmd = line.strip().split(",")[0].replace("OptString.new(","")[1:-1]
                                extraOptions.append(cmd)
                        if "bind" in f:
                            if "x64" in root:
                                for stage in stagesX64:
                                    self.payloadTree[platform]["x64"][stage][moduleName] = ["LPORT"] + extraOptions
                            elif "x86" in root:
                                for stage in stagesX86:
                                    self.payloadTree[platform]["x86"][stage][moduleName] = ["LPORT"] + extraOptions
                            else:
                                for stage in stagesX86:
                                    self.payloadTree[platform][stage][moduleName] = ["LPORT"] + extraOptions
                        if "reverse" in f:
                            if "x64" in root:
                                for stage in stagesX64:
                                    self.payloadTree[platform]["x64"][stage][moduleName] = ["LHOST", "LPORT"] + extraOptions
                            elif "x86" in root:
                                for stage in stagesX86:
                                    self.payloadTree[platform]["x86"][stage][moduleName] = ["LHOST", "LPORT"] + extraOptions
                            else:
                                for stage in stagesX86:
                                    self.payloadTree[platform][stage][moduleName] = ["LHOST", "LPORT"] + extraOptions

            # load up any payload singles
            # TODO: detect Windows and modify the paths appropriately
            for root, dirs, files in os.walk(settings.METASPLOIT_PATH + "/modules/payloads/singles/" + platform + "/"):
                for f in files:

                    if ".rb" in f:

                        lines = open(root + "/" + f).readlines()
                        totalOptions = list()
                        moduleName = f.split(".")[0]

                        for line in lines:
                            if "OptString" in line.strip() and "true" in line.strip():
                                cmd = line.strip().split(",")[0].replace("OptString.new(","")[1:-1]
                                totalOptions.append(cmd)
                        if "bind" in f:
                            totalOptions.append("LPORT")
                        if "reverse" in f:
                            totalOptions.append("LHOST")
                            totalOptions.append("LPORT")
                        if "x64" in root:
                            self.payloadTree[platform]["x64"][moduleName] = totalOptions
                        elif "x86" in root:
                            self.payloadTree[platform]["x86"][moduleName] = totalOptions
                        else:
                            self.payloadTree[platform][moduleName] = totalOptions

Example 4

Project: wikiteam
Source File: launcher.py
View license
def main():
    if len(sys.argv) < 2:
        print 'python script.py file-with-apis.txt'
        sys.exit()
    
    print 'Reading list of APIs from', sys.argv[1]
    wikis = open(sys.argv[1], 'r').read().splitlines()
    print '%d APIs found' % (len(wikis))
    
    for wiki in wikis:
        print "#"*73
        print "# Downloading", wiki
        print "#"*73
        wiki = wiki.lower()
        # Make the prefix in standard way; api and index must be defined, not important which is which
        prefix = dumpgenerator.domain2prefix(config={'api': wiki, 'index': wiki})
        
        #check if compressed, in that case dump was finished previously
        compressed = False
        for dirname, dirnames, filenames in os.walk('.'):
            if dirname == '.':
                for f in filenames:
                    if f.startswith(prefix) and f.endswith('.7z'):
                        compressed = True
                        zipfilename = f
                break #stop searching, dot not explore subdirectories
        
        if compressed:
            print 'Skipping... This wiki was downloaded and compressed before in', zipfilename
            # Get the archive's file list.
            if ( ( ( sys.version_info[0] == 3 ) and ( sys.version_info[1] > 0 ) ) or ( ( sys.version_info[0] == 2 ) and ( sys.version_info[1] > 6 ) ) ):
                archivecontent = subprocess.check_output (['7z', 'l', zipfilename])
                if re.search(ur"%s.+-history\.xml" % (prefix), archivecontent) is None:
                    # We should perhaps not create an archive in this case, but we continue anyway.
                    print "ERROR: The archive contains no history!"
                if re.search(ur"Special:Version\.html", archivecontent) is None:
                    print "WARNING: The archive doesn't contain Special:Version.html, this may indicate that download didn't finish."
            else:
                print "WARNING: Content of the archive not checked, we need python 2.7+ or 3.1+."
                # TODO: Find a way like grep -q below without doing a 7z l multiple times?
            continue
        
        #download
        started = False #was this wiki download started before? then resume
        wikidir = ''
        for dirname, dirnames, filenames in os.walk('.'):
            if dirname == '.':
                for d in dirnames:
                    if d.startswith(prefix):
                        wikidir = d
                        started = True
                break #stop searching, dot not explore subdirectories
        
        # time.sleep(60)
        # Uncomment what above and add --delay=60 in the dumpgenerator.py calls below for broken wiki farms
        # such as editthis.info, wiki-site.com, wikkii (adjust the value as needed;
        # typically they don't provide any crawl-delay value in their robots.txt).
        if started and wikidir: #then resume
            print 'Resuming download, using directory', wikidir
            subprocess.call('./dumpgenerator.py --api=%s --xml --images --resume --path=%s' % (wiki, wikidir), shell=True)
        else: #download from scratch
            subprocess.call('./dumpgenerator.py --api=%s --xml --images' % wiki, shell=True)
            started = True
            #save wikidir now
            for dirname, dirnames, filenames in os.walk('.'):
                if dirname == '.':
                    for d in dirnames:
                        if d.startswith(prefix):
                            wikidir = d
                    break #stop searching, dot not explore subdirectories
        
        prefix = wikidir.split('-wikidump')[0]
        
        finished = False
        if started and wikidir and prefix:
            if (subprocess.call (['tail -n 1 %s/%s-history.xml | grep -q "</mediawiki>"' % (wikidir, prefix)], shell=True) ):
                print "No </mediawiki> tag found: dump failed, needs fixing; resume didn't work. Exiting."
            else:
                finished = True
        # You can also issue this on your working directory to find all incomplete dumps:
        # tail -n 1 */*-history.xml | grep -Ev -B 1 "</page>|</mediawiki>|==|^$"
        
        #compress
        if finished:
            time.sleep(1)
            os.chdir(wikidir)
            print 'Changed directory to', os.getcwd()
            # Basic integrity check for the xml. The script doesn't actually do anything, so you should check if it's broken. Nothing can be done anyway, but redownloading.
            subprocess.call('grep "<title>" *.xml -c;grep "<page>" *.xml -c;grep "</page>" *.xml -c;grep "<revision>" *.xml -c;grep "</revision>" *.xml -c', shell=True)
            # Make a non-solid archive with all the text and metadata at default compression. You can also add config.txt if you don't care about your computer and user names being published or you don't use full paths so that they're not stored in it.
            subprocess.call('7z' + ' a -ms=off ../%s-history.xml.7z.tmp %s-history.xml %s-titles.txt index.html Special:Version.html errors.log siteinfo.json' % (prefix, prefix, prefix), shell=True)
            subprocess.call('mv' + ' ../%s-history.xml.7z.tmp ../%s-history.xml.7z' % (prefix, prefix), shell=True)
            # Now we add the images, if there are some, to create another archive, without recompressing everything, at the min compression rate, higher doesn't compress images much more.
            subprocess.call('cp' + ' ../%s-history.xml.7z ../%s-wikidump.7z.tmp' % (prefix, prefix), shell=True)
            subprocess.call('7z' + ' a -ms=off -mx=1 ../%s-wikidump.7z.tmp %s-images.txt images/' % (prefix, prefix), shell=True)
            subprocess.call('mv' + ' ../%s-wikidump.7z.tmp ../%s-wikidump.7z' % (prefix, prefix), shell=True)
            os.chdir('..')
            print 'Changed directory to', os.getcwd()
            time.sleep(1)

Example 5

Project: odoo
Source File: setup.py
View license
def py2exe_datafiles():
    data_files = {}
    data_files['Microsoft.VC90.CRT'] = glob('C:\Microsoft.VC90.CRT\*.*')

    for root, dirnames, filenames in os.walk('openerp'):
        for filename in filenames:
            if not re.match(r'.*(\.pyc|\.pyo|\~)$', filename):
                data_files.setdefault(root, []).append(join(root, filename))

    import babel
    data_files['babel/localedata'] = glob(join(dirname(babel.__file__), 'localedata', '*'))
    others = ['global.dat', 'numbers.py', 'support.py', 'plural.py']
    data_files['babel'] = map(lambda f: join(dirname(babel.__file__), f), others)
    others = ['frontend.py', 'mofile.py']
    data_files['babel/messages'] = map(lambda f: join(dirname(babel.__file__), 'messages', f), others)

    import pytz
    tzdir = dirname(pytz.__file__)
    for root, _, filenames in os.walk(join(tzdir, 'zoneinfo')):
        base = join('pytz', root[len(tzdir) + 1:])
        data_files[base] = [join(root, f) for f in filenames]

    import docutils
    dudir = dirname(docutils.__file__)
    for root, _, filenames in os.walk(dudir):
        base = join('docutils', root[len(dudir) + 1:])
        data_files[base] = [join(root, f) for f in filenames if not f.endswith(('.py', '.pyc', '.pyo'))]

    import passlib
    pl = dirname(passlib.__file__)
    for root, _, filenames in os.walk(pl):
        base = join('passlib', root[len(pl) + 1:])
        data_files[base] = [join(root, f) for f in filenames if not f.endswith(('.py', '.pyc', '.pyo'))]

    return data_files.items()

Example 6

Project: gsutil
Source File: wildcard_iterator.py
View license
  def _IterDir(self, directory, wildcard):
    """An iterator over the specified dir and wildcard."""
    # UTF8-encode directory before passing it to os.walk() so if there are
    # non-valid UTF8 chars in the file name (e.g., that can happen if the file
    # originated on Windows) os.walk() will not attempt to decode and then die
    # with a "codec can't decode byte" error, and instead we can catch the error
    # at yield time and print a more informative error message.
    for dirpath, unused_dirnames, filenames in os.walk(directory.encode(UTF8)):
      for f in fnmatch.filter(filenames, wildcard):
        try:
          yield os.path.join(dirpath,
                             FixWindowsEncodingIfNeeded(f)).decode(UTF8)
        except UnicodeDecodeError:
          # Note: We considered several ways to deal with this, but each had
          # problems:
          # 1. Raise an exception and try to catch in a higher layer (the
          #    gsutil cp command), so we can properly support the gsutil cp -c
          #    option. That doesn't work because raising an exception during
          #    iteration terminates the generator.
          # 2. Accumulate a list of bad filenames and skip processing each
          #    during iteration, then raise at the end, with exception text
          #    printing the bad paths. That doesn't work because iteration is
          #    wrapped in PluralityCheckableIterator, so it's possible there
          #    are not-yet-performed copy operations at the time we reach the
          #    end of the iteration and raise the exception - which would cause
          #    us to skip copying validly named files. Moreover, the gsutil
          #    cp command loops over argv, so if you run the command gsutil cp
          #    -rc dir1 dir2 gs://bucket, an invalid unicode name inside dir1
          #    would cause dir2 never to be visited.
          # 3. Print the invalid pathname and skip it during iteration. That
          #    would work but would mean gsutil cp could exit with status 0
          #    even though some files weren't copied.
          # 4. Change the WildcardIterator to include an error status along with
          #    the result. That would solve the problem but would be a
          #    substantial change (WildcardIterator is used in many parts of
          #    gsutil), and we didn't feel that magnitude of change was
          #    warranted by this relatively uncommon corner case.
          # Instead we chose to abort when one such file is encountered, and
          # require the user to remove or rename the files and try again.
          raise CommandException('\n'.join(textwrap.wrap(
              _UNICODE_EXCEPTION_TEXT % repr(os.path.join(dirpath, f)))))

Example 7

Project: invesalius3
Source File: dicom_reader.py
View license
def yGetDicomGroups(directory, recursive=True, gui=True):
    """
    Return all full paths to DICOM files inside given directory.
    """
    nfiles = 0
    # Find total number of files
    if recursive:
        for dirpath, dirnames, filenames in os.walk(directory):
            nfiles += len(filenames)
    else:
        dirpath, dirnames, filenames = os.walk(directory)
        nfiles = len(filenames)

    counter = 0
    grouper = dicom_grouper.DicomPatientGrouper() 
    #q = Queue.Queue()
    #l = threading.Lock()
    #threads = []
    #for i in xrange(cpu_count()):
    #    t = LoadDicom(grouper, q, l)
    #    t.start()
    #    threads.append(t)
    # Retrieve only DICOM files, splited into groups
    if recursive:
        for dirpath, dirnames, filenames in os.walk(directory):
            for name in filenames:
                filepath = os.path.join(dirpath, name)
                counter += 1
                if gui:
                    yield (counter,nfiles)
                LoadDicom(grouper, filepath)
    else:
        dirpath, dirnames, filenames = os.walk(directory)
        for name in filenames:
            filepath = str(os.path.join(dirpath, name))
            counter += 1
            if gui:
                yield (counter,nfiles)
            #q.put(filepath)

    #for t in threads:
    #    q.put(0)

    #for t in threads:
    #    t.join()

    #TODO: Is this commented update necessary?
    #grouper.Update()
    yield grouper.GetPatientsGroups()

Example 8

Project: mantaray
Source File: get_system_paths.py
View license
def get_system_paths(value, prnt, outfile, mount_point):

	#initialize variables
	system_hive_path = "NONE"
	system_hive_regback_path = "NONE"
	sam_hive_path = "NONE"
	sam_hive_regback_path = "NONE"
	software_hive_path = "NONE"
	software_hive_regback_path = "NONE"
	security_hive_path = "NONE"
	security_hive_regback_path = "NONE"
	paths = []
	paths = ["NONE", "NONE", "NONE", "NONE","NONE", "NONE","NONE", "NONE"]


	#process Windows systems
	if (value == "hfs+"):
		print ("Don't need to look for registry hives since this partition is hfs+")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is hfs+\n")
	elif (value == "ext"):
		print ("Don't need to look for registry hives since this partition is ext")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is ext2\n")
	elif (value == "ext3"):
		print ("Don't need to look for registry hives since this partition is ext3")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is ext3\n")
	elif (value == "ext4"):
		print ("Don't need to look for registry hives since this partition is ext4")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is ext4\n")
	else: 
		if(value == "fat32"):
			print ("Look for registry hives since this partition is Windows and formatted fat32")
		if(outfile != "NONE"):
			outfile.write("Look for registry hives since this partition is Windows and formatted fat32\n")
		if(value == "ntfs"):
			print ("Look for registry hives since this partition is Windows and formatted ntfs")
		if(outfile != "NONE"):
			outfile.write("Look for registry hives since this partition is Windows and formatted ntfs\n")
		
		#set up print border
		if(prnt == "YES"):
			print("***********REGISTRY FILE LOCATIONS***********************************")
		if(outfile != "NONE"):
			outfile.write("***********REGISTRY FILE LOCATIONS***********************************\n")


		#get path to system hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/system'):
					system_hive_path = os.path.join(root,filenames)
					paths[0] = system_hive_path
					if(prnt == "YES"):					
						print("SYSTEM HIVE: " + system_hive_path)
					if (outfile != "NONE"):
						outfile.write("SYSTEM HIVE: " + system_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/system'):
					system_hive_regback_path = os.path.join(root,filenames)
					paths[1] = system_hive_regback_path
					if(prnt == "YES"):					
						print("SYSTEM HIVE REGBACK: " + system_hive_regback_path)
					if (outfile != "NONE"):
						outfile.write("SYSTEM HIVE REGBACK: " + system_hive_regback_path + "\n")
		#get path to sam hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/sam'):
					sam_hive_path = os.path.join(root,filenames)
					paths[2] = sam_hive_path
					if(prnt == "YES"):				
						print("SAM HIVE: " + sam_hive_path)
					if(outfile != "NONE"):
						outfile.write("SAM HIVE: " + sam_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/sam'):
					sam_hive_regback_path = os.path.join(root,filenames)
					paths[3] = sam_hive_regback_path
					if(prnt == "YES"):
						print("SAM HIVE REGBACK: " + sam_hive_regback_path)
					if(outfile != "NONE"):
						outfile.write("SAM HIVE REGBACK: " + sam_hive_regback_path + "\n")
		#get path to software hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/software'):
					software_hive_path = os.path.join(root,filenames)
					paths[4] = software_hive_path
					if(prnt == "YES"):
						print("SOFTWARE HIVE: " + software_hive_path)
					if(outfile != "NONE"):
						outfile.write("SOFTWARE HIVE: " + software_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/software'):
					software_hive_regback_path = os.path.join(root,filenames)
					paths[5] = software_hive_regback_path
					if(prnt == "YES"):
						print("SOFTWARE HIVE REGBACK: " + software_hive_regback_path)
					if(outfile != "NONE"):
						outfile.write("SOFTWARE HIVE REGBACK: " + software_hive_regback_path + "\n")

		#get path to security hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/security'):
					security_hive_path = os.path.join(root,filenames)
					paths[6] = security_hive_path
					if(prnt == "YES"):
						print("SECURITY HIVE: " + security_hive_path)
					if(outfile != "NONE"):
						outfile.write("SECURITY HIVE: " + security_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/security'):
					security_hive_regback_path = os.path.join(root,filenames)
					paths[7] = security_hive_regback_path
					if(prnt == "YES"):
						print("SECURITY HIVE REGBACK: " + security_hive_regback_path)
					if(outfile != "NONE"):
						outfile.write("SECURITY HIVE REGBACK: " + security_hive_regback_path + "\n")

		#close print border
		if(prnt == "YES"):		
			print("***********REGISTRY FILE LOCATIONS***********************************\n\n")
		if(outfile != "NONE"):
			outfile.write("***********REGISTRY FILE LOCATIONS************************************\n\n")

		return paths

Example 9

Project: mantaray
Source File: get_system_paths.py
View license
def get_system_paths(value, prnt, outfile, mount_point):

	#initialize variables
	system_hive_path = "NONE"
	system_hive_regback_path = "NONE"
	sam_hive_path = "NONE"
	sam_hive_regback_path = "NONE"
	software_hive_path = "NONE"
	software_hive_regback_path = "NONE"
	security_hive_path = "NONE"
	security_hive_regback_path = "NONE"
	paths = []
	paths = ["NONE", "NONE", "NONE", "NONE","NONE", "NONE","NONE", "NONE"]


	#process Windows systems
	if (value == "hfs+"):
		print ("Don't need to look for registry hives since this partition is hfs+")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is hfs+\n")
	elif (value == "ext"):
		print ("Don't need to look for registry hives since this partition is ext")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is ext2\n")
	elif (value == "ext3"):
		print ("Don't need to look for registry hives since this partition is ext3")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is ext3\n")
	elif (value == "ext4"):
		print ("Don't need to look for registry hives since this partition is ext4")
		if(outfile != "NONE"):
			outfile.write("Don't need to look for registry hives since this partition is ext4\n")
	else: 
		if(value == "fat32"):
			print ("Look for registry hives since this partition is Windows and formatted fat32")
		if(outfile != "NONE"):
			outfile.write("Look for registry hives since this partition is Windows and formatted fat32\n")
		if(value == "ntfs"):
			print ("Look for registry hives since this partition is Windows and formatted ntfs")
		if(outfile != "NONE"):
			outfile.write("Look for registry hives since this partition is Windows and formatted ntfs\n")
		
		#set up print border
		if(prnt == "YES"):
			print("***********REGISTRY FILE LOCATIONS***********************************")
		if(outfile != "NONE"):
			outfile.write("***********REGISTRY FILE LOCATIONS***********************************\n")


		#get path to system hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/system'):
					system_hive_path = os.path.join(root,filenames)
					paths[0] = system_hive_path
					if(prnt == "YES"):					
						print("SYSTEM HIVE: " + system_hive_path)
					if (outfile != "NONE"):
						outfile.write("SYSTEM HIVE: " + system_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/system'):
					system_hive_regback_path = os.path.join(root,filenames)
					paths[1] = system_hive_regback_path
					if(prnt == "YES"):					
						print("SYSTEM HIVE REGBACK: " + system_hive_regback_path)
					if (outfile != "NONE"):
						outfile.write("SYSTEM HIVE REGBACK: " + system_hive_regback_path + "\n")
		#get path to sam hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/sam'):
					sam_hive_path = os.path.join(root,filenames)
					paths[2] = sam_hive_path
					if(prnt == "YES"):				
						print("SAM HIVE: " + sam_hive_path)
					if(outfile != "NONE"):
						outfile.write("SAM HIVE: " + sam_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/sam'):
					sam_hive_regback_path = os.path.join(root,filenames)
					paths[3] = sam_hive_regback_path
					if(prnt == "YES"):
						print("SAM HIVE REGBACK: " + sam_hive_regback_path)
					if(outfile != "NONE"):
						outfile.write("SAM HIVE REGBACK: " + sam_hive_regback_path + "\n")
		#get path to software hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/software'):
					software_hive_path = os.path.join(root,filenames)
					paths[4] = software_hive_path
					if(prnt == "YES"):
						print("SOFTWARE HIVE: " + software_hive_path)
					if(outfile != "NONE"):
						outfile.write("SOFTWARE HIVE: " + software_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/software'):
					software_hive_regback_path = os.path.join(root,filenames)
					paths[5] = software_hive_regback_path
					if(prnt == "YES"):
						print("SOFTWARE HIVE REGBACK: " + software_hive_regback_path)
					if(outfile != "NONE"):
						outfile.write("SOFTWARE HIVE REGBACK: " + software_hive_regback_path + "\n")

		#get path to security hive
		for root,dirs,files in os.walk(mount_point):
			for filenames in files:
				if (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/security'):
					security_hive_path = os.path.join(root,filenames)
					paths[6] = security_hive_path
					if(prnt == "YES"):
						print("SECURITY HIVE: " + security_hive_path)
					if(outfile != "NONE"):
						outfile.write("SECURITY HIVE: " + security_hive_path + "\n")
				elif (os.path.join(root,filenames).lower() == mount_point + '/windows/system32/config/regback/security'):
					security_hive_regback_path = os.path.join(root,filenames)
					paths[7] = security_hive_regback_path
					if(prnt == "YES"):
						print("SECURITY HIVE REGBACK: " + security_hive_regback_path)
					if(outfile != "NONE"):
						outfile.write("SECURITY HIVE REGBACK: " + security_hive_regback_path + "\n")

		#close print border
		if(prnt == "YES"):		
			print("***********REGISTRY FILE LOCATIONS***********************************\n\n")
		if(outfile != "NONE"):
			outfile.write("***********REGISTRY FILE LOCATIONS************************************\n\n")

		return paths

Example 10

Project: Radium-Keylogger
Source File: Radiumkeylogger.py
View license
def DriveTree():
    file_dir1 = 'C:\Users\Public\Intel\Logs\Dir_View.txt'   #The drive hierarchy will be saved in this file
    drives = win32api.GetLogicalDriveStrings()
    drives = drives.split('\000')[:-1]
    no_of_drives = len(drives)
    file_dir_O = open(file_dir1, "w")

    for d in range(no_of_drives):
        try:
            file_dir_O.write(str(drives[d]) + "\n")
            directories = os.walk(drives[d])
            next_dir = next(directories)

            next_directories = next_dir[1]
            next_files = next_dir[2]

            next_final_dir = next_directories + next_files

            for nd in next_final_dir:
                file_dir_O.write("	" + str(nd) + "\n")
                try:
                    sub_directories = os.walk(drives[d] + nd)

                    next_sub_dir = next(sub_directories)[1]
                    next_sub_sub_file = next(sub_directories)[2]

                    next_final_final_dir = next_sub_dir + next_sub_sub_file

                    for nsd in next_final_final_dir:
                        file_dir_O.write("		" + str(nsd) + "\n")

                        try:
                            sub_sub_directories = os.walk(drives[d] + nd + '\\' + nsd)

                            next_sub_sub_dir = next(sub_sub_directories)[1]
                            next_sub_sub_sub_file = next(sub_sub_directories)[2]

                            next_final_final_final_dir = next_sub_sub_dir + next_sub_sub_sub_file

                            for nssd in next_final_final_final_dir:
                                file_dir_O.write("			" + str(nssd) + "\n")
                        except Exception as e:
                            pass

                except Exception as e:
                    pass
        except Exception as e:
            pass

    file_dir_O.close()
    return True

Example 11

Project: Radium-Keylogger
Source File: Radiumkeylogger.py
View license
def DriveTree():
    file_dir1 = 'C:\Users\Public\Intel\Logs\Dir_View.txt'   #The drive hierarchy will be saved in this file
    drives = win32api.GetLogicalDriveStrings()
    drives = drives.split('\000')[:-1]
    no_of_drives = len(drives)
    file_dir_O = open(file_dir1, "w")

    for d in range(no_of_drives):
        try:
            file_dir_O.write(str(drives[d]) + "\n")
            directories = os.walk(drives[d])
            next_dir = next(directories)

            next_directories = next_dir[1]
            next_files = next_dir[2]

            next_final_dir = next_directories + next_files

            for nd in next_final_dir:
                file_dir_O.write("	" + str(nd) + "\n")
                try:
                    sub_directories = os.walk(drives[d] + nd)

                    next_sub_dir = next(sub_directories)[1]
                    next_sub_sub_file = next(sub_directories)[2]

                    next_final_final_dir = next_sub_dir + next_sub_sub_file

                    for nsd in next_final_final_dir:
                        file_dir_O.write("		" + str(nsd) + "\n")

                        try:
                            sub_sub_directories = os.walk(drives[d] + nd + '\\' + nsd)

                            next_sub_sub_dir = next(sub_sub_directories)[1]
                            next_sub_sub_sub_file = next(sub_sub_directories)[2]

                            next_final_final_final_dir = next_sub_sub_dir + next_sub_sub_sub_file

                            for nssd in next_final_final_final_dir:
                                file_dir_O.write("			" + str(nssd) + "\n")
                        except Exception as e:
                            pass

                except Exception as e:
                    pass
        except Exception as e:
            pass

    file_dir_O.close()
    return True

Example 12

Project: crowdin-cli
Source File: connection.py
View license
    def get_files_source(self):
        sources = []
        for f in self.files_source:
            f['source'] = f['source'].replace('^', '!')
            if f['source'].count('**') > 1:
                logger.info("Source pattern {0} is not valid. The mask `**` "
                            "can be used only once in the source pattern.".format(f['source']))
                exit()
            ignore_list = []
            parameters = {}

            if 'titles' in f:
                parameters['titles'] = f['titles']

            if 'type' in f:
                parameters['type'] = f['type']

            if 'translate_content' in f:
                parameters['translate_content'] = f['translate_content']

            if 'translate_attributes' in f:
                parameters['translate_attributes'] = f['translate_attributes']

            if 'content_segmentation' in f:
                parameters['content_segmentation'] = f['content_segmentation']

            if 'translatable_elements' in f:
                parameters['translatable_elements'] = f['translatable_elements']

            if 'update_option' in f:
                parameters['update_option'] = f['update_option']

            if 'first_line_contains_header' in f:
                parameters['first_line_contains_header'] = f['first_line_contains_header']

            if 'scheme' in f:
                parameters['scheme'] = f['scheme']

            if 'multilingual_spreadsheet' in f:
                parameters['multilingual_spreadsheet'] = f['multilingual_spreadsheet']

            if 'import_duplicates' in f:
                parameters['import_duplicates'] = f['import_duplicates']

            if 'import_eq_suggestions' in f:
                parameters['import_eq_suggestions'] = f['import_eq_suggestions']

            if 'auto_approve_imported' in f:
                parameters['auto_approve_imported'] = f['auto_approve_imported']

            if 'languages_mapping' in f:
                parameters['languages_mapping'] = f['languages_mapping']

            if 'dest' in f:
                parameters['dest'] = f['dest']

            if 'escape_quotes' in f:
                parameters['escape_quotes'] = f['escape_quotes']

            file_name = f['source'][1:][f['source'].rfind("/"):]
            if 'ignore' in f:
                for ign in f['ignore']:
                    if '*' in ign or '?' in ign or '[' in ign:
                        ign = ign.replace('^', '!')
                        root, items, fg = self.get_doubled_asterisk(ign)
                        # walk through folders and file in local directories
                        for dp, dn, filenames in os.walk(items):
                            for ff in filenames:
                                if fnmatch.fnmatch(ff, self.metacharacter(ign[ign.rfind('/'):][1:])):
                                    ignore_list.append(os.path.join(dp.replace(root, ''), ff).replace("\\", r'/'))
                    else:
                        ignore_list.append(ign.replace("\\", r'/'))

            root, items, fg = self.get_doubled_asterisk(f['source'])
            if '*' in file_name or '?' in file_name or '[' in file_name:
                    if '**' in f['source']:

                        for dp, dn, filenames in os.walk(items):
                            for ff in filenames:

                                if fnmatch.fnmatch(ff, self.metacharacter(f['source'][f['source'].rfind('/'):][1:])):
                                    if fg in dp.replace("\\", r'/'):
                                        fgg = ''
                                        if fg:
                                            fgg = '/' + fg
                                        value = os.path.join(dp.replace(root, ''), ff).replace("\\", r'/')

                                        if not [s for s in ignore_list if s in value]:
                                            sources.append(value)
                                            f['translation'] = f['translation'].replace("\\", r'/')
                                            fgg = fgg.replace("\\", r'/')
                                            items = items.replace("\\", r'/')
                                            dp = dp.replace("\\", r'/')
                                            sources.append(
                                                f['translation'].replace('**', dp.replace(items, '').replace(fgg, ''))
                                            )
                                            sources.append(parameters)

                    else:
                        # print items
                        for dp, dn, filenames in os.walk(items):
                            for ff in filenames:
                                # if os.path.splitext(ff)[1] == os.path.splitext(f['source'])[1]:
                                if fnmatch.fnmatch(ff, self.metacharacter(f['source'][f['source'].rfind('/'):][1:])):
                                    value = os.path.join(dp.replace(root, ''), ff).replace("\\", r'/')

                                    if not [s for s in ignore_list if s in value]:
                                        sources.append(value)
                                        sources.append(f['translation'])
                                        sources.append(parameters)
                            break

            elif '**' in f['source']:
                for dp, dn, filenames in os.walk(items):
                    for ff in filenames:
                        if ff == f['source'][f['source'].rfind('/'):][1:]:
                            if fg in dp.replace("\\", r'/'):
                                fgg = ''
                                if fg:
                                    fgg = '/' + fg
                                value = os.path.join(dp.replace(root, ''), ff).replace("\\", r'/')
                                if not [s for s in ignore_list if s in value]:
                                    sources.append(value)
                                    f['translation'] = f['translation'].replace("\\", r'/')
                                    fgg = fgg.replace("\\", r'/')
                                    items = items.replace("\\", r'/')
                                    dp = dp.replace("\\", r'/')
                                    sources.append(
                                        f['translation'].replace('**', dp.replace(items, '').replace(fgg, ''))
                                    )
                                    sources.append(parameters)

            else:
                if not [s for s in ignore_list if s in f['source']]:
                    sources.append(f['source'])
                    sources.append(f['translation'])
                    sources.append(parameters)
        if not sources:
            print('It seems that there are none files to upload. Please check your configuration')
        return sources

Example 13

Project: CorpusTools
Source File: multiple_files.py
View license
def load_directory_multiple_files(corpus_name, path, dialect,
                                    annotation_types = None,
                                    feature_system_path = None,
                                    stop_check = None, call_back = None):
    """
    Loads a directory of corpus standard files (separated into words files
    and phones files)

    Parameters
    ----------
    corpus_name : str
        Name of corpus
    path : str
        Path to directory of text files
    dialect : str
        One of 'buckeye' or 'timit'
    annotation_types : list of AnnotationType, optional
        List of AnnotationType specifying how to parse the glosses.
        Auto-generated based on dialect.
    feature_system_path : str, optional
        File path of FeatureMatrix binary to specify segments
    stop_check : callable or None
        Optional function to check whether to gracefully terminate early
    call_back : callable or None
        Optional function to supply progress information during the loading

    Returns
    -------
    SpontaneousSpeechCorpus
        Corpus containing Discourses corresponding to the text files
    """
    if call_back is not None:
        call_back('Finding  files...')
        call_back(0, 0)
    file_tuples = []
    for root, subdirs, files in os.walk(path):
        for filename in files:
            if stop_check is not None and stop_check():
                return
            if not (filename.lower().endswith('.words') or filename.lower().endswith('.wrd')):
                continue
            file_tuples.append((root, filename))
    if call_back is not None:
        call_back('Parsing files...')
        call_back(0,len(file_tuples))
        cur = 0
    corpus = SpontaneousSpeechCorpus(corpus_name, path)
    for i, t in enumerate(file_tuples):
        if stop_check is not None and stop_check():
            return
        if call_back is not None:
            call_back('Parsing file {} of {}...'.format(i+1, len(file_tuples)))
            call_back(i)
        root, filename = t
        name,ext = os.path.splitext(filename)
        if ext == '.words':
            phone_ext = '.phones'
        else:
            phone_ext = '.phn'
        word_path = os.path.join(root,filename)
        phone_path = os.path.splitext(word_path)[0] + phone_ext
        d = load_discourse_multiple_files(name, word_path, phone_path,
                                            dialect, annotation_types,
                                            corpus.lexicon, None,
                                            stop_check, None)
        corpus.add_discourse(d)

    if feature_system_path is not None:
        feature_matrix = load_binary(feature_system_path)
        corpus.lexicon.set_feature_matrix(feature_matrix)
    return corpus

Example 14

Project: picoCTF-Platform-2
Source File: api_manager.py
View license
def load_problems(args):
    problem_dir = args.problems_directory[0]
    grader_dir = args.graders_directory[0]
    static_dir = args.static_directory[0]

    if not path.exists(static_dir):
        logging.debug("No directory {}. Creating...".format(static_dir))
        makedirs(static_dir)

    if not path.exists(problem_dir):
        logging.critical("No such directory: {}".format(problem_dir))
        return

    for (dirpath, dirnames, filenames) in walk(problem_dir):
        if "problem.json" in filenames:
            json_file = path.join(dirpath, 'problem.json')
            contents = open(json_file, "r").read()

            try:
                data = json_util.loads(contents)
            except ValueError as e:
                logging.warning("Invalid JSON format in file {filename} ({exception})".format(filename=json_file,
                                                                                              exception=e))
                continue

            if not isinstance(data, dict):
                logging.warning("Invalid JSON format in file {}".format(json_file))
                continue

            if 'name' not in data:
                logging.warning("Invalid problem format in file {}".format(json_file))
                continue

            problem_name = data['name']
            relative_path = path.relpath(dirpath, problem_dir)
            logging.info("Found problem '{}'".format(problem_name))

            if 'grader' not in dirnames:
                logging.warning("Problem '{}' appears to have no grader folder. Skipping...".format(problem_name))
                continue

            grader_path = path.join(grader_dir, relative_path)
            if path.exists(grader_path):
                shutil.rmtree(grader_path)

            shutil.copytree(path.join(dirpath, 'grader'), grader_path)
            logging.info("Graders updated for problem {}".format(problem_name))

            try:
                api.problem.insert_problem(data)
            except api.common.WebException as e:
                logging.info("Problem '{}' was not added to the database. Reason: {}".format(problem_name, e))

            if 'static' in dirnames:
                logging.info("Found a static directory for '{}'. Copying...".format(problem_name))
                static_path = path.join(static_dir, relative_path)
                if path.exists(static_path):
                    shutil.rmtree(static_path)
                shutil.copytree(path.join(dirpath, 'static'), static_path)


    errors = api.problem.analyze_problems()
    for error in errors:
        logging.warning(error)

Example 15

Project: picoCTF-web
Source File: api_manager.py
View license
def load_problems(args):
    problem_dir = args.problems_directory[0]
    grader_dir = args.graders_directory[0]
    static_dir = args.static_directory[0]

    if not path.exists(static_dir):
        logging.debug("No directory {}. Creating...".format(static_dir))
        makedirs(static_dir)

    if not path.exists(problem_dir):
        logging.critical("No such directory: {}".format(problem_dir))
        return

    for (dirpath, dirnames, filenames) in walk(problem_dir):
        if "problem.json" in filenames:
            json_file = path.join(dirpath, 'problem.json')
            contents = open(json_file, "r").read()

            try:
                data = json_util.loads(contents)
            except ValueError as e:
                logging.warning("Invalid JSON format in file {filename} ({exception})".format(filename=json_file,
                                                                                              exception=e))
                continue

            if not isinstance(data, dict):
                logging.warning("Invalid JSON format in file {}".format(json_file))
                continue

            if 'name' not in data:
                logging.warning("Invalid problem format in file {}".format(json_file))
                continue

            problem_name = data['name']
            relative_path = path.relpath(dirpath, problem_dir)
            logging.info("Found problem '{}'".format(problem_name))

            if 'grader' not in dirnames:
                logging.warning("Problem '{}' appears to have no grader folder. Skipping...".format(problem_name))
                continue

            grader_path = path.join(grader_dir, relative_path)
            if path.exists(grader_path):
                shutil.rmtree(grader_path)

            shutil.copytree(path.join(dirpath, 'grader'), grader_path)
            logging.info("Graders updated for problem {}".format(problem_name))

            try:
                api.problem.insert_problem(data)
            except api.common.WebException as e:
                logging.info("Problem '{}' was not added to the database. Reason: {}".format(problem_name, e))

            if 'static' in dirnames:
                logging.info("Found a static directory for '{}'. Copying...".format(problem_name))
                static_path = path.join(static_dir, relative_path)
                if path.exists(static_path):
                    shutil.rmtree(static_path)
                shutil.copytree(path.join(dirpath, 'static'), static_path)


    errors = api.problem.analyze_problems()
    for error in errors:
        logging.warning(error)

Example 16

Project: picrust
Source File: all_tests.py
View license
def main():
    option_parser, opts, args =\
       parse_command_line_parameters(**script_info)

    if (opts.suppress_unit_tests and opts.suppress_script_usage_tests):
       option_parser.error("You're suppressing both test types. Nothing to run.")

    test_dir = abspath(dirname(__file__))

    unittest_good_pattern = re.compile('OK\s*$')
    application_not_found_pattern = re.compile('ApplicationNotFoundError')
    python_name = 'python'
    bad_tests = []
    missing_application_tests = []

    # Run through all of PICRUSt's unit tests, and keep track of any files which
    # fail unit tests.
    if not opts.suppress_unit_tests:
        unittest_names = []
        if not opts.unittest_glob:
            for root, dirs, files in walk(test_dir):
                for name in files:
                    if name.startswith('test_') and name.endswith('.py'):
                        unittest_names.append(join(root,name))
        else:
            for fp in glob(opts.unittest_glob):
                fn = split(fp)[1]
                if fn.startswith('test_') and fn.endswith('.py'):
                    unittest_names.append(abspath(fp))

        unittest_names.sort()

        for unittest_name in unittest_names:
            print "Testing %s:\n" % unittest_name
            command = '%s %s -v' % (python_name, unittest_name)
            stdout, stderr, return_value = system_call(command)
            print stderr
            if not unittest_good_pattern.search(stderr):
                if application_not_found_pattern.search(stderr):
                    missing_application_tests.append(unittest_name)
                else:
                    bad_tests.append(unittest_name)

    if not opts.suppress_script_usage_tests:
        try:
            from qiime.test import run_script_usage_tests
        except ImportError:
            print "QIIME not installed so not running script tests."
            opts.suppress_script_usage_tests=True
        else:
            test_data_dir = join(get_picrust_project_dir(),'picrust_test_data')
            scripts_dir  = join(get_picrust_project_dir(),'scripts')
            if opts.script_usage_tests != None:
                script_usage_tests = opts.script_usage_tests.split(',')
            else:
                script_usage_tests = None

            # Run the script usage testing functionality
            script_usage_result_summary, num_script_usage_example_failures = \
                    run_script_usage_tests(
                    test_data_dir=test_data_dir,
                    scripts_dir=scripts_dir,
                    working_dir='/tmp/',
                    verbose=True,
                    tests=script_usage_tests,
                    force_overwrite=True,
                    timeout=300)

    print "==============\nResult summary\n=============="

    if not opts.suppress_unit_tests:
        print "\nUnit test result summary\n------------------------\n"
        if bad_tests:
            print "\nFailed the following unit tests.\n%s" % '\n'.join(bad_tests)

        if missing_application_tests:
            print "\nFailed the following unit tests, in part or whole due "+\
            "to missing external applications.\nDepending on the PICRUSt features "+\
            "you plan to use, this may not be critical.\n%s"\
             % '\n'.join(missing_application_tests)

        if not (missing_application_tests or bad_tests):
            print "\nAll unit tests passed.\n\n"

    if not opts.suppress_script_usage_tests:
        print "\nScript usage test result summary\n------------------------------------\n"
        print script_usage_result_summary
        print ""

    # If script usage tests weren't suppressed,we can't have any failures.
    script_usage_tests_success = (opts.suppress_script_usage_tests or
                                  num_script_usage_example_failures == 0)

    # If any of the unit tests or script usage tests fail, or if we have any
    # missing application errors, use return code 1 (as python's unittest
    # module does to indicate one or more failures).
    return_code = 1
    if (len(bad_tests) == 0 and len(missing_application_tests) == 0 and
        script_usage_tests_success):
        return_code = 0
    return return_code

Example 17

Project: SourcePawnCompletions
Source File: inotify_c.py
View license
    def read_events(self, event_buffer_size=DEFAULT_EVENT_BUFFER_SIZE):
        """
        Reads events from inotify and yields them.
        """
        # HACK: We need to traverse the directory path
        # recursively and simulate events for newly
        # created subdirectories/files. This will handle
        # mkdir -p foobar/blah/bar; touch foobar/afile

        def _recursive_simulate(src_path):
            events = []
            for root, dirnames, filenames in os.walk(src_path):
                for dirname in dirnames:
                    try:
                        full_path = os.path.join(root, dirname)
                        wd_dir = self._add_watch(full_path, self._event_mask)
                        e = InotifyEvent(
                            wd_dir, InotifyConstants.IN_CREATE | InotifyConstants.IN_ISDIR, 0, dirname, full_path)
                        events.append(e)
                    except OSError:
                        pass
                for filename in filenames:
                    full_path = os.path.join(root, filename)
                    wd_parent_dir = self._wd_for_path[os.path.dirname(full_path)]
                    e = InotifyEvent(
                        wd_parent_dir, InotifyConstants.IN_CREATE, 0, filename, full_path)
                    events.append(e)
            return events

        while True:
            try:
                event_buffer = os.read(self._inotify_fd, event_buffer_size)
            except OSError as e:
                if e.errno == errno.EINTR:
                    continue
            break

        with self._lock:
            event_list = []
            for wd, mask, cookie, name in Inotify._parse_event_buffer(event_buffer):
                if wd == -1:
                    continue
                wd_path = self._path_for_wd[wd]
                src_path = os.path.join(wd_path, name) if name else wd_path #avoid trailing slash
                inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)

                if inotify_event.is_moved_from:
                    self.remember_move_from_event(inotify_event)
                elif inotify_event.is_moved_to:
                    move_src_path = self.source_for_move(inotify_event)
                    if move_src_path in self._wd_for_path:
                        moved_wd = self._wd_for_path[move_src_path]
                        del self._wd_for_path[move_src_path]
                        self._wd_for_path[inotify_event.src_path] = moved_wd
                        self._path_for_wd[moved_wd] = inotify_event.src_path
                    src_path = os.path.join(wd_path, name)
                    inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)

                if inotify_event.is_ignored:
                    # Clean up book-keeping for deleted watches.
                    self._remove_watch_bookkeeping(src_path)
                    continue

                event_list.append(inotify_event)

                if (self.is_recursive and inotify_event.is_directory and
                        inotify_event.is_create):

                    # TODO: When a directory from another part of the
                    # filesystem is moved into a watched directory, this
                    # will not generate events for the directory tree.
                    # We need to coalesce IN_MOVED_TO events and those
                    # IN_MOVED_TO events which don't pair up with
                    # IN_MOVED_FROM events should be marked IN_CREATE
                    # instead relative to this directory.
                    try:
                        self._add_watch(src_path, self._event_mask)
                    except OSError:
                        continue

                    event_list.extend(_recursive_simulate(src_path))

        return event_list

Example 18

View license
def gather_raw_data(site_folder, yaml_outpath, scan_type, \
                        include_sites=False, subject_inclusion=None):

    import os
    import yaml
    
    sub_dict = {}
    inclusion_list = []
    

    # create subject inclusion list
    if subject_inclusion != None:
        with open(subject_inclusion, "r") as f:
            inclusion_list = f.readlines()
        # remove any /n's
        inclusion_list = map(lambda s: s.strip(), inclusion_list)   
    
    
    for root, folders, files in os.walk(site_folder):
    
        for filename in files:
        
            fullpath = os.path.join(root, filename)
            
            if ".nii" in fullpath:
            
                # /path_to_site_folder/subject_id/session_id/scan_id/..
                
                second_half = fullpath.split(site_folder)[1]
                
                second_half_list = second_half.split("/")

                try:
                    second_half_list.remove("")
                except:
                    pass

                if include_sites:

                    try:

                        site_id = second_half_list[-5]
                        subject_id = second_half_list[-4]
                        session_id = second_half_list[-3]
                        scan_id = second_half_list[-2]

                    except:

                        err = "\n\n[!] Could not parse the data directory " \
                              "structure. Is it in the correct format?\n\n" \
                              "Your directory structure:\n%s\n\nIt should " \
                              "be something like this:\n/site_folder/subject"\
                              "_id/session_id/scan_id/file.nii.gz\n\n" \
                              % second_half
                        raise Exception(err)

                else:

                    try:

                        subject_id = second_half_list[-4]
                        session_id = second_half_list[-3]
                        scan_id = second_half_list[-2]

                    except:
                        
                        err = "\n\n[!] Could not parse the data directory " \
                              "structure. Is it in the correct format?\n\n" \
                              "Your directory structure:\n%s\n\nIt should " \
                              "be something like this:\n/subject_id" \
                              "/session_id/scan_id/file.nii.gz\n\n" \
                              % second_half
                        raise Exception(err)


                if subject_inclusion == None:
                    inclusion_list.append(subject_id)
                

                # assign default scan names if the file structure of the data
                # does not have a directory level for scan
                if ".nii" in scan_id:
                    if scan_type == "anat":
                        scan_id = "anat_1"
                    elif scan_type == "func":
                        scan_id = "rest_1"
                
                #sub_info = (subject_id, session_id, scan_id)
                
                if ("anat" in scan_id) or ("anat" in filename) or \
                    ("mprage" in filename):
                    
                    resource = "anatomical_scan"
                    
                if ("rest" in scan_id) or ("rest" in filename) or \
                    ("func" in scan_id) or ("func" in filename):
                    
                    resource = "functional_scan"

                
                if (scan_type == "anat") and \
                    (resource == "anatomical_scan") and \
                        (subject_id in inclusion_list):

                    if subject_id not in sub_dict.keys():
                        sub_dict[subject_id] = {}
                    
                    if session_id not in sub_dict[subject_id].keys():
                        sub_dict[subject_id][session_id] = {}
                    
                    if resource not in sub_dict[subject_id][session_id].keys():
                        sub_dict[subject_id][session_id][resource] = {}

                        if include_sites:
                            sub_dict[subject_id][session_id]["site_name"] = \
                                site_id
                                                       
                    if scan_id not in sub_dict[subject_id][session_id][resource].keys():
                        sub_dict[subject_id][session_id][resource][scan_id] = fullpath
                    
                        
                if (scan_type == "func") and \
                     (resource == "functional_scan") and \
                         (subject_id in inclusion_list):
                
                    if subject_id not in sub_dict.keys():
                        sub_dict[subject_id] = {}
                    
                    if session_id not in sub_dict[subject_id].keys():
                        sub_dict[subject_id][session_id] = {}
                    
                    if resource not in sub_dict[subject_id][session_id].keys():
                        sub_dict[subject_id][session_id][resource] = {}
                        
                        if include_sites:
                            sub_dict[subject_id][session_id]["site_name"] = \
                                site_id
                                    
                    if scan_id not in sub_dict[subject_id][session_id][resource].keys():
                        sub_dict[subject_id][session_id][resource][scan_id] = fullpath
 
 
                
    with open(yaml_outpath,"wt") as f:
        f.write(yaml.dump(sub_dict))

Example 19

Project: ochothon
Source File: init.py
View license
def init(args):

    tmp = tempfile.mkdtemp()
    try:

        #
        # - git clone the template repo from the opaugam org.
        #
        kind = args[0] if args else 'default'
        repo = 'ochothon-template-%s' % kind
        code, _ = shell('git clone https://github.com/opaugam/%s' % repo, cwd=tmp)
        assert code == 0, 'unable to find template "%s" in git' % kind

        #
        # - ask a few questions
        #
        tag = raw_input('> enter a short identifier (e.g web or database): ')
        image = raw_input('> enter the docker repo/image: ')

        #
        # - strip non-alpha characters from the tag
        #
        bad = ''.join(c for c in map(chr, range(256)) if not c.isalnum() and c not in ['-'])
        tag = tag.translate(None, bad)

        mappings = \
            {
                'tag': tag,
                'image': image
            }

        renderable = \
            [
                'Dockerfile',
                'README*',
                '*.py',
                '*.yml',
                '*.conf'
            ]

        #
        # - walk through the cloned repo
        # - render all templates
        #
        l = len(tmp) + 1
        env = Environment(loader=FileSystemLoader(tmp))
        for root, sub, items in os.walk(tmp):
            for item in items:
                absolute = path.join(root, item)
                if not '.git' in absolute:
                    for regex in renderable:
                        if fnmatch.fnmatch(item, regex):
                            rendered = env.get_template(absolute[l:]).render(mappings)
                            import codecs
                            with codecs.open(absolute, 'wb', 'utf-8') as f:
                                f.write(rendered)
                            break

        #
        # - copy the whole thing to where the script is invoked from
        #
        local = 'ochopod-marathon-%s' % tag
        code, _ = shell('mkdir %s && cp -r %s/%s/* %s' % (local, tmp, repo, local))
        print 'template ready in %s/' % local

    except KeyboardInterrupt:
        exit(0)

    except Exception as failure:
        print('internal failure <- %s' % str(failure))
        exit(1)

    finally:

        #
        # - cleanup the temporary directory
        #
        shutil.rmtree(tmp)

Example 20

Project: pyspace
Source File: setup.py
View license
def generate_blacklist():
    """
    This function tries to import all the pySPACE node modules available.
    If a certain module does not have the necessary dependencies installed,
    it will be blacklisted and not imported in further usages of the software

    Once the user installs the missing dependency, he or she can run the setup
    script with the soft option enabled i.e.::

        python setup.py --blacklist

    which will only refresh the blacklisted nodes.
    """
    blacklist = []
    missing_dependencies = []
    import pySPACE
    import pySPACE.missions.nodes
    for dirpath, dirnames, files in os.walk('pySPACE'):
        if 'nodes' not in dirpath or 'missions' not in dirpath:
            continue

        for filename in files:
            if "__init__" in filename or filename.endswith(".py") == False:
                continue

            the_module = os.path.join(dirpath, filename)
            if CURRENTOS == 'Windows':
                the_module = the_module.replace('\\', '.')[:-3]
            else:
                the_module = the_module.replace('/', '.')[:-3]
            try:
                __import__(the_module)
            except Exception, e:
                missing_dependencies.append(e.message)
                blacklist.append(filename + ", " + dirpath)
            except:
                pass

    try:
        latest_config = home + os.sep + 'pySPACEcenter' + os.sep + 'config.yaml'
        config_file = open(latest_config, 'r+')
    except:
        import glob
        print "The default config file was not found. Modifying the latest" \
              "version available."
        latest_config = max(glob.iglob(
            home + os.sep + 'pySPACEcenter' + os.sep + '*.yaml'),
            key=os.path.getctime)
        config_file = open(latest_config, 'r+')

    content = config_file.read()
    blacklist_wrote_to_file = False
    new_content = ""
    for line in content.split('\n'):
        if "blacklisted_nodes" in line:
            new_content += "blacklisted_nodes: " + str(blacklist)
            blacklist_wrote_to_file = True
        else:
            new_content += line + "\n"
    if not blacklist_wrote_to_file:
        new_content += "\nblacklisted_nodes: "+str(blacklist)

    if len(blacklist) > 0:
        print "--> The following nodes are blacklisted and henceforth" \
              " excluded from usage within pySPACE"
        print "\n       > "+"\n       > ".join(blacklist)

        print "\n  > If you want to use the above modules, please fix the " \
              "following errors"
        print "\n       --> "+"\n       --> ".join(missing_dependencies) + "\n"



    config_file.seek(0)
    config_file.write(new_content)
    config_file.truncate()
    config_file.close()

Example 21

Project: headphones
Source File: helpers.py
View license
def expand_subfolders(f):
    """
    Try to expand a given folder and search for subfolders containing media
    files. This should work for discographies indexed per album in the same
    root, possibly with folders per CD (if any).

    This algorithm will return nothing if the result is only one folder. In this
    case, normal post processing will be better.
    """

    from headphones import logger

    # Find all folders with media files in them
    media_folders = []

    for root, dirs, files in os.walk(f):
        for file in files:
            extension = os.path.splitext(file)[1].lower()[1:]

            if extension in headphones.MEDIA_FORMATS:
                if root not in media_folders:
                    media_folders.append(root)

    # Stop here if nothing found
    if len(media_folders) == 0:
        return

    # Split into path components
    media_folders = [split_path(media_folder) for media_folder in media_folders]

    # Correct folder endings such as CD1 etc.
    for index, media_folder in enumerate(media_folders):
        if RE_CD.match(media_folder[-1]):
            media_folders[index] = media_folders[index][:-1]

    # Verify the result by computing path depth relative to root.
    path_depths = [len(media_folder) for media_folder in media_folders]
    difference = max(path_depths) - min(path_depths)

    if difference > 0:
        logger.info(
            "Found %d media folders, but depth difference between lowest and deepest media folder is %d (expected zero). If this is a discography or a collection of albums, make sure albums are per folder.",
            len(media_folders), difference)

        # While already failed, advice the user what he could try. We assume the
        # directory may contain separate CD's and maybe some extra's. The
        # structure may look like X albums at same depth, and (one or more)
        # extra folders with a higher depth.
        extra_media_folders = [media_folder[:min(path_depths)] for media_folder in media_folders if
                               len(media_folder) > min(path_depths)]
        extra_media_folders = list(
            set([os.path.join(*media_folder) for media_folder in extra_media_folders]))

        logger.info(
            "Please look at the following folder(s), since they cause the depth difference: %s",
            extra_media_folders)
        return

    # Convert back to paths and remove duplicates, which may be there after
    # correcting the paths
    media_folders = list(set([os.path.join(*media_folder) for media_folder in media_folders]))

    # Don't return a result if the number of subfolders is one. In this case,
    # this algorithm will not improve processing and will likely interfere
    # with other attempts such as MusicBrainz release group IDs.
    if len(media_folders) == 1:
        logger.debug("Did not expand subfolder, as it resulted in one folder.")
        return

    logger.debug("Expanded subfolders in folder: %s", media_folders)
    return media_folders

Example 22

Project: headphones
Source File: helpers.py
View license
def extract_metadata(f):
    """
    Scan all files in the given directory and decide on an artist, album and
    year based on the metadata. A decision is based on the number of different
    artists, albums and years found in the media files.
    """

    from headphones import logger

    # Walk directory and scan all media files
    results = []
    count = 0

    for root, dirs, files in os.walk(f):
        for file in files:
            # Count the number of potential media files
            extension = os.path.splitext(file)[1].lower()[1:]

            if extension in headphones.MEDIA_FORMATS:
                count += 1

            # Try to read the file info
            try:
                media_file = MediaFile(os.path.join(root, file))
            except (FileTypeError, UnreadableFileError):
                # Probably not a media file
                continue

            # Append metadata to file
            artist = media_file.albumartist or media_file.artist
            album = media_file.album
            year = media_file.year

            if artist and album and year:
                results.append((artist.lower(), album.lower(), year))

    # Verify results
    if len(results) == 0:
        logger.info("No metadata in media files found, ignoring.")
        return (None, None, None)

    # Require that some percentage of files have tags
    count_ratio = 0.75

    if count < (count_ratio * len(results)):
        logger.info("Counted %d media files, but only %d have tags, ignoring.", count, len(results))
        return (None, None, None)

    # Count distinct values
    artists = list(set([x[0] for x in results]))
    albums = list(set([x[1] for x in results]))
    years = list(set([x[2] for x in results]))

    # Remove things such as CD2 from album names
    if len(albums) > 1:
        new_albums = list(albums)

        # Replace occurences of e.g. CD1
        for index, album in enumerate(new_albums):
            if RE_CD_ALBUM.search(album):
                old_album = new_albums[index]
                new_albums[index] = RE_CD_ALBUM.sub("", album).strip()

                logger.debug("Stripped albumd number identifier: %s -> %s", old_album,
                             new_albums[index])

        # Remove duplicates
        new_albums = list(set(new_albums))

        # Safety check: if nothing has merged, then ignore the work. This can
        # happen if only one CD of a multi part CD is processed.
        if len(new_albums) < len(albums):
            albums = new_albums

    # All files have the same metadata, so it's trivial
    if len(artists) == 1 and len(albums) == 1:
        return (artists[0], albums[0], years[0])

    # (Lots of) different artists. Could be a featuring album, so test for this.
    if len(artists) > 1 and len(albums) == 1:
        split_artists = [RE_FEATURING.split(x) for x in artists]
        featurings = [len(split_artist) - 1 for split_artist in split_artists]
        logger.info("Album seem to feature %d different artists", sum(featurings))

        if sum(featurings) > 0:
            # Find the artist of which the least splits have been generated.
            # Ideally, this should be 0, which should be the album artist
            # itself.
            artist = split_artists[featurings.index(min(featurings))][0]

            # Done
            return (artist, albums[0], years[0])

    # Not sure what to do here.
    logger.info("Found %d artists, %d albums and %d years in metadata, so ignoring", len(artists),
                len(albums), len(years))
    logger.debug("Artists: %s, Albums: %s, Years: %s", artists, albums, years)

    return (None, None, None)

Example 23

Project: scons
Source File: javac.py
View license
def emit_java_classes(target, source, env):
    """Create and return lists of source java files
    and their corresponding target class files.
    """
    java_suffix = env.get('JAVASUFFIX', '.java')
    class_suffix = env.get('JAVACLASSSUFFIX', '.class')

    target[0].must_be_same(SCons.Node.FS.Dir)
    classdir = target[0]

    s = source[0].rentry().disambiguate()
    if isinstance(s, SCons.Node.FS.File):
        sourcedir = s.dir.rdir()
    elif isinstance(s, SCons.Node.FS.Dir):
        sourcedir = s.rdir()
    else:
        raise SCons.Errors.UserError("Java source must be File or Dir, not '%s'" % s.__class__)

    slist = []
    js = _my_normcase(java_suffix)
    for entry in source:
        entry = entry.rentry().disambiguate()
        if isinstance(entry, SCons.Node.FS.File):
            slist.append(entry)
        elif isinstance(entry, SCons.Node.FS.Dir):
            result = SCons.Util.OrderedDict()
            dirnode = entry.rdir()
            def find_java_files(arg, dirpath, filenames):
                java_files = sorted([n for n in filenames
                                       if _my_normcase(n).endswith(js)])
                mydir = dirnode.Dir(dirpath)
                java_paths = [mydir.File(f) for f in java_files]
                for jp in java_paths:
                     arg[jp] = True
            for dirpath, dirnames, filenames in os.walk(dirnode.get_abspath()):
               find_java_files(result, dirpath, filenames)
            entry.walk(find_java_files, result)

            slist.extend(list(result.keys()))
        else:
            raise SCons.Errors.UserError("Java source must be File or Dir, not '%s'" % entry.__class__)

    version = env.get('JAVAVERSION', '1.4')
    full_tlist = []
    for f in slist:
        tlist = []
        source_file_based = True
        pkg_dir = None
        if not f.is_derived():
            pkg_dir, classes = parse_java_file(f.rfile().get_abspath(), version)
            if classes:
                source_file_based = False
                if pkg_dir:
                    d = target[0].Dir(pkg_dir)
                    p = pkg_dir + os.sep
                else:
                    d = target[0]
                    p = ''
                for c in classes:
                    t = d.File(c + class_suffix)
                    t.attributes.java_classdir = classdir
                    t.attributes.java_sourcedir = sourcedir
                    t.attributes.java_classname = classname(p + c)
                    tlist.append(t)

        if source_file_based:
            base = f.name[:-len(java_suffix)]
            if pkg_dir:
                t = target[0].Dir(pkg_dir).File(base + class_suffix)
            else:
                t = target[0].File(base + class_suffix)
            t.attributes.java_classdir = classdir
            t.attributes.java_sourcedir = f.dir
            t.attributes.java_classname = classname(base)
            tlist.append(t)

        for t in tlist:
            t.set_specific_source([f])

        full_tlist.extend(tlist)

    return full_tlist, slist

Example 24

Project: lorax
Source File: filediff.py
View license
def main(args):
    try:
        sourcedir, targetdir = args[1], args[2]
    except IndexError:
        print("invalid argument count")
        print("usage: python {0} sourcetree targettree".format(args[0]))
        sys.exit(2)

    if sourcedir.endswith("/"):
        sourcedir = sourcedir[:-1]
    if targetdir.endswith("/"):
        targetdir = targetdir[:-1]

    # parse sourcedir and targetdir
    sourcetree, targettree = {}, {}
    for tree, d in [[sourcetree, sourcedir], [targettree, targetdir]]:
        for root, _dnames, fnames in os.walk(d):
            for fname in fnames:
                fpath = os.path.join(root, fname)
                rpath = fpath.replace(d, "", 1)
                tree[rpath] = fpath

    # set up magic
    m = magic.open(magic.MAGIC_NONE)
    m.load()

    # get files missing in source
    sys.stderr.write("getting files missing in source\n")
    for rpath in sorted(targettree.keys()):
        fpath = targettree[rpath]

        targetfile = fpath
        try:
            sourcefile = sourcetree[rpath]
        except KeyError:
            sys.stdout.write('Missing: %s\n' % rpath)
            continue

        # skip broken links
        if os.path.islink(targetfile) and not os.path.exists(targetfile):
            continue

        # check stat
        #sourcemode = os.stat(sourcefile).st_mode
        #targetmode = os.stat(targetfile).st_mode
        #if sourcemode != targetmode:
        #    sys.stdout.write('Stat differ: %s\n' % rpath)

        # diff only text files
        ftype = m.file(fpath)
        if ftype not in ["ASCII text"]:
            continue

        with open(targetfile, "r") as fobj:
            target = fobj.readlines()
        with open(sourcefile) as fobj:
            source = fobj.readlines()

        # do the file diff
        for line in difflib.unified_diff(source, target,
                                         fromfile=sourcefile,
                                         tofile=targetfile):

            sys.stdout.write(line)

    # set up yum

    # XXX HACK
    # we don't want yum's stuff in the output
    # so we redirect stdout to /dev/null for a while...
    stdout = os.dup(1)
    null = open("/dev/null", "w")
    os.dup2(null.fileno(), 1)

    # here yum prints out some stuff we really don't care about
    yb = yum.YumBase()
    yb.doSackSetup()

    # give the stdout back
    os.dup2(stdout, 1)
    null.close()

    # get excessive files in source
    sys.stderr.write("getting excessive files in source\n")
    sizedict, pkgdict = {}, {}
    for rpath, fpath in sourcetree.items():
        # if file in target, skip it
        if rpath in targettree:
            continue

        # get file size
        try:
            sizeinbytes = os.path.getsize(fpath)
        except OSError:
            sizeinbytes = 0

        # set link size to 0
        islink = os.path.islink(fpath)
        if islink:
            sizeinbytes = 0

        pkglist = yb.whatProvides(rpath, None, None)
        pkglist = set(map(lambda pkgobj: pkgobj.name, pkglist))

        for pkg in pkglist:
            sizedict[pkg] = sizedict.get(pkg, 0) + sizeinbytes
            pkgdict[pkg] = pkgdict.get(pkg, []) + \
                           [(rpath, sizeinbytes, islink)]

    # sort by size
    for pkg, _size in sorted(sizedict.items(), key=operator.itemgetter(1),
                            reverse=True):

        for item in sorted(pkgdict[pkg]):
            sys.stdout.write("%s\t%s\n" % (pkg, item))

Example 25

Project: RocAlphaGo
Source File: game_converter.py
View license
def run_game_converter(cmd_line_args=None):
    """Run conversions. command-line args may be passed in as a list
    """
    import argparse
    import sys

    parser = argparse.ArgumentParser(
        description='Prepare SGF Go game files for training the neural network model.',
        epilog="Available features are: board, ones, turns_since, liberties,\
        capture_size, self_atari_size, liberties_after, sensibleness, and zeros.\
        Ladder features are not currently implemented")
    parser.add_argument("--features", "-f", help="Comma-separated list of features to compute and store or 'all'", default='all')  # noqa: E501
    parser.add_argument("--outfile", "-o", help="Destination to write data (hdf5 file)", required=True)  # noqa: E501
    parser.add_argument("--recurse", "-R", help="Set to recurse through directories searching for SGF files", default=False, action="store_true")  # noqa: E501
    parser.add_argument("--directory", "-d", help="Directory containing SGF files to process. if not present, expects files from stdin", default=None)  # noqa: E501
    parser.add_argument("--size", "-s", help="Size of the game board. SGFs not matching this are discarded with a warning", type=int, default=19)  # noqa: E501
    parser.add_argument("--verbose", "-v", help="Turn on verbose mode", default=False, action="store_true")  # noqa: E501

    if cmd_line_args is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(cmd_line_args)

    if args.features.lower() == 'all':
        feature_list = [
            "board",
            "ones",
            "turns_since",
            "liberties",
            "capture_size",
            "self_atari_size",
            "liberties_after",
            # "ladder_capture",
            # "ladder_escape",
            "sensibleness",
            "zeros"]
    else:
        feature_list = args.features.split(",")

    if args.verbose:
        print("using features", feature_list)

    converter = game_converter(feature_list)

    def _is_sgf(fname):
        return fname.strip()[-4:] == ".sgf"

    def _walk_all_sgfs(root):
        """a helper function/generator to get all SGF files in subdirectories of root
        """
        for (dirpath, dirname, files) in os.walk(root):
            for filename in files:
                if _is_sgf(filename):
                    # yield the full (relative) path to the file
                    yield os.path.join(dirpath, filename)

    def _list_sgfs(path):
        """helper function to get all SGF files in a directory (does not recurse)
        """
        files = os.listdir(path)
        return (os.path.join(path, f) for f in files if _is_sgf(f))

    # get an iterator of SGF files according to command line args
    if args.directory:
        if args.recurse:
            files = _walk_all_sgfs(args.directory)
        else:
            files = _list_sgfs(args.directory)
    else:
        files = (f.strip() for f in sys.stdin if _is_sgf(f))

    converter.sgfs_to_hdf5(files, args.outfile, bd_size=args.size, verbose=args.verbose)

Example 26

Project: yum
Source File: rstgenerator.py
View license
def generateAll(source_directory, output_directory):
    #Verify that both the source and output directories exist


    # Keep a set of file names that are packages.  This is 
    # useful so that later we will be able to figure out full
    # module names.
    packages = set()

    # Keep a list of tuples containing python module names and
    # relative paths, so that we can build the index file later
    modules = []

    # Walk the directory tree
    for dirpath, dirnames, filenames in os.walk(source_directory, topdown=True):
        
        # print dirpath
        # print dirnames
        # print filenames
        # print

        # Add the curent directory to packages if __init__.py exists
        if '__init__.py' in filenames:
            packages.add(dirpath)

        # Find the hierarchy of packages that we are currently in
        package_heirarchy = []
        #Recurse up to the root
        dirpath_i = dirpath
        while dirpath_i != '/':
            if dirpath_i in packages:
                dirpath_i, tail = os.path.split(dirpath_i)
                package_heirarchy.insert(0, tail)
            else:
                break

        # Find the relative output directory, mirroring the input
        # directory structure
        relative_output_directory = ''
        if not os.path.samefile(dirpath, source_directory):
            relative_output_directory = os.path.relpath(dirpath, source_directory)
        
        # Don't recurse into directories that are hidden, or for docs
        for directory in dirnames:
            if directory == "docs" or directory.startswith("."):
                dirnames.remove(directory)

        # Generate the rst for a file if it is a python source code file
        for file_name in filenames:
            # Skip file names that contain dashes, since they're not
            # valid module names, so we won't be able to import them
            # to generate the documentation anyway
            if '-' in file_name:
                continue

            if file_name.endswith('.py'):
                module_name = file_name.partition('.')[0]
                modules.append(os.path.join(relative_output_directory, 
                                            module_name))
                generateFile(dirpath, file_name, 
                             os.path.join(output_directory, relative_output_directory),
                             package_heirarchy, module_name)

        
    
    # Create the index.rst file
    generateIndex(modules, output_directory)

Example 27

View license
    def __init__(self, __genparams):
        '''The constructor for the GenerateListForTree class'''
        __arcpath = __genparams['arcpath']
        __delimiter = __genparams['delimiter']
        self._genfull = __genparams['genfull']
        self.__listoutput = __genparams['listoutput']
        self.__confoutput = __genparams['confoutput']
        self.__fulloutput = __genparams['fulloutput']
        self.__getallhashes  = __genparams['getallhashes']
        self.__hashtype = __genparams['hashtype']
        self.__parsingexceptions = __genparams['parsingexceptions']
        self.__confname = __genparams['confname']
        __listoffiles = ['[files]\n']
        __oneline = '{value}{delimiter} ={value} uid{delimiter}{value} gid{delimiter}{value} owner{delimiter}{value} group{delimiter}{value} mode{delimiter}{value} type{delimiter}{value} mtime{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
        if self.__getallhashes:
            if not self.__hashtype:
                __onelinewithhash = '{value}{delimiter} ={value} uid{delimiter}{value} gid{delimiter}{value} owner{delimiter}{value} group{delimiter}{value} mode{delimiter}{value} type{delimiter}{value} mtime{delimiter}{value} md5{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
            else:
                __onelinewithhash = '{value}{delimiter} ={value} uid{delimiter}{value} gid{delimiter}{value} owner{delimiter}{value} group{delimiter}{value} mode{delimiter}{value} type{delimiter}{value} mtime{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__hashtype, delimiter=__delimiter)
        else:
            __onelinewithouthash = '{value}{delimiter} ={value} uid{delimiter}{value} gid{delimiter}{value} owner{delimiter}{value} group{delimiter}{value} mode{delimiter}{value} type{delimiter}{value} mtime{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
        # we also need parameters for symbolic links
        __onelinewithtarget = '{value}{delimiter} ={value} uid{delimiter}{value} gid{delimiter}{value} owner{delimiter}{value} group{delimiter}{value} mode{delimiter}{value} type{delimiter}{value} mtime{delimiter}{value} target{delimiter}{value}\n'.format(value='{}', delimiter=__delimiter)
        
        for __dirpath, __dirnames, __filenames, in os.walk(__arcpath):
            # ignoring the uppest directory
            if os.path.relpath(__dirpath, __arcpath) != '.':
                # studying directories
                __dirinfo = os.lstat(__dirpath)
                __dirmode = oct(stat.S_IMODE(__dirinfo.st_mode)).split('o')[-1]
                # translate file type in backupchecker intern file type
                __type = self.__translate_type(__dirinfo.st_mode)
                # extract file data
                __listoffiles.append(__oneline.format(
                                        os.path.relpath(__dirpath, __arcpath),
                                        str(__dirinfo.st_size),
                                        str(__dirinfo.st_uid),
                                        str(__dirinfo.st_gid),
                                        pwd.getpwuid(__dirinfo.st_uid).pw_name,
                                        grp.getgrgid(__dirinfo.st_gid).gr_name,
                                        __dirmode,
                                        __type,
                                        str(__dirinfo.st_mtime)))
            # studying files
            for __filename in __filenames:
                __filepath = os.path.join(__dirpath, __filename)
                __filepath = self._normalize_path(__filepath)
                self.__fileinfo = os.lstat(__filepath)
                __filemode = oct(stat.S_IMODE(self.__fileinfo.st_mode)).split('o')[-1]
                __type = self.__translate_type(self.__fileinfo.st_mode)
                if __type == 'f': 
                    if self.__getallhashes:
                        if not self.__hashtype:
                            # extract hash sum of the file inside the archive
                            __hash = get_hash(open(__filepath, 'rb'), 'md5')
                        else:
                            # extract hash sum of the file inside the archive
                            __hash = get_hash(open(__filepath, 'rb'), self.__hashtype)
                        # extract file data and prepare data
                        __listoffiles.append(__onelinewithhash.format(
                                                os.path.relpath(__filepath, __arcpath),
                                                str(self.__fileinfo.st_size),
                                                str(self.__fileinfo.st_uid),
                                                str(self.__fileinfo.st_gid),
                                                pwd.getpwuid(self.__fileinfo.st_uid).pw_name,
                                                grp.getgrgid(self.__fileinfo.st_gid).gr_name,
                                                __filemode,
                                                __type,
                                                str(self.__fileinfo.st_mtime),
                                                __hash))
                    else:
                        # check if there are exceptions while parsing
                        if self.__parsingexceptions:
                            for __file in self.__parsingexceptions:
                                if fnmatch.fnmatch(os.path.relpath(__filepath, __arcpath), __file):
                                    __hash = get_hash(open(__filepath, 'rb'), self.__parsingexceptions[__file])
                                    __onelinewithhash = '{value}{delimiter} ={value} uid{delimiter}{value} gid{delimiter}{value} owner{delimiter}{value} group{delimiter}{value} mode{delimiter}{value} type{delimiter}{value} mtime{delimiter}{value} {hashtype}{delimiter}{value}\n'.format(value='{}', hashtype=self.__parsingexceptions[__file], delimiter=__delimiter)
                                    __listoffiles.append(__onelinewithhash.format(
                                                            os.path.relpath(__filepath, __arcpath),
                                                            str(self.__fileinfo.st_size),
                                                            str(self.__fileinfo.st_uid),
                                                            str(self.__fileinfo.st_gid),
                                                            pwd.getpwuid(self.__fileinfo.st_uid).pw_name,
                                                            grp.getgrgid(self.__fileinfo.st_gid).gr_name,
                                                            __filemode,
                                                            __type,
                                                            str(self.__fileinfo.st_mtime),
                                                            __hash))
                                else:
                                    # we use exceptions-file option but the file is not concerned by an exception
                                    __listoffiles.append(__onelinewithouthash.format(
                                                            os.path.relpath(__filepath, __arcpath),
                                                            str(self.__fileinfo.st_size),
                                                            str(self.__fileinfo.st_uid),
                                                            str(self.__fileinfo.st_gid),
                                                            pwd.getpwuid(self.__fileinfo.st_uid).pw_name,
                                                            grp.getgrgid(self.__fileinfo.st_gid).gr_name,
                                                            __filemode,
                                                            __type,
                                                            str(self.__fileinfo.st_mtime)))
                        else:
                            # we don't use the --exceptions-file option
                            __listoffiles.append(__onelinewithouthash.format(
                                                    os.path.relpath(__filepath, __arcpath),
                                                    str(self.__fileinfo.st_size),
                                                    str(self.__fileinfo.st_uid),
                                                    str(self.__fileinfo.st_gid),
                                                    pwd.getpwuid(self.__fileinfo.st_uid).pw_name,
                                                    grp.getgrgid(self.__fileinfo.st_gid).gr_name,
                                                    __filemode,
                                                    __type,
                                                    str(self.__fileinfo.st_mtime)))
                elif __type == 's':
                    # extract hash sum of the file inside the archive
                    # extract file data and prepare data
                    __listoffiles.append(__onelinewithtarget.format(
                                            os.path.relpath(__filepath, __arcpath),
                                            str(self.__fileinfo.st_size),
                                            str(self.__fileinfo.st_uid),
                                            str(self.__fileinfo.st_gid),
                                            pwd.getpwuid(self.__fileinfo.st_uid).pw_name,
                                            grp.getgrgid(self.__fileinfo.st_gid).gr_name,
                                            __filemode,
                                            __type,
                                            str(self.__fileinfo.st_mtime),
                                            os.readlink(__filepath)))
                else:
                    # if file is not regular file, ignoring its hash sum
                    __listoffiles.append(__oneline.format(
                                            os.path.relpath(__filepath, __arcpath),
                                            str(self.__fileinfo.st_size),
                                            str(self.__fileinfo.st_uid),
                                            str(self.__fileinfo.st_gid),
                                            pwd.getpwuid(self.__fileinfo.st_uid).pw_name,
                                            grp.getgrgid(self.__fileinfo.st_gid).gr_name,
                                            __filemode,
                                            __type,
                                            str(self.__fileinfo.st_mtime)))
                                            
        # include custom paths for output conf files
        __reparc = os.path.split(__arcpath)[-1]
        if self.__listoutput:
            # --gen-list and --output-list-dir and --configuration-name
            if self.__confname:
                __arclistpath = os.path.join(self.__listoutput, '.'.join([self.__confname, 'list']))
            else:
                # --gen-list and --output-list-dir
                __arclistpath = os.path.join(self.__listoutput, '.'.join([__reparc, 'list']))
        elif self.__fulloutput:
            if self.__confname:
                # --gen-list and --output-list-and-conf-dir and --configuration-name
                __arclistpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'list']))
            else:
                # --gen-list and --output-list-and-conf-dir
                __arclistpath = os.path.join(self.__fulloutput, '.'.join([__reparc, 'list']))
        else:
            # --gen-list only
            if self.__confname:
                __arc = os.path.dirname(__arcpath)
                __arclistpath =  os.path.join(__arc, '.'.join([self.__confname, 'list']))
            else:
                __arclistpath = ''.join([__arcpath, '.list'])
        __listconfinfo = {'arclistpath': __arclistpath,
                            'listoffiles':  __listoffiles}
        # call the method to write information in a file
        self._generate_list(__listconfinfo)
        # call the method to write the configuration file if --gen-full was required
        if self._genfull:
            if self.__confoutput:
                if self.__confname:
                    # --gen-full and --output-conf-dir and --configuration-name
                    __arcconfpath = os.path.join(self.__confoutput, '.'.join([self.__confname, 'conf']))
                else:
                    # --gen-full and --output-conf-dir
                    __arcconfpath = os.path.join(self.__confoutput, '.'.join([__reparc, 'conf']))
            elif self.__fulloutput:
                if self.__confname:
                    # --gen-full and --output-list-and-conf-dir and --configuration-name
                    __arcconfpath = os.path.join(self.__fulloutput, '.'.join([self.__confname, 'conf']))
                else:
                    # --gen-full and --output-list-and-conf-dir
                    __arcconfpath = os.path.join(self.__fulloutput, '.'.join([__reparc, 'conf']))
            else:
                if self.__confname:
                    # --gen-full and --configuration-name
                    __arc = os.path.dirname(__arcpath)
                    __arc = os.path.join(__arc, self.__confname)
                    __arcconfpath = '.'.join([__arc, 'conf'])
                else:
                    # --gen-full only
                    if self.__confname:
                        __arc = os.path.dirname(__arcpath)
                        __arcconfpath =  os.path.join(__arc, '.'.join([self.__confname, 'conf']))
                    else:
                        __arcconfpath = '.'.join([__arcpath, 'conf'])
            # generate the hash sum of the list of files
            __listhashsum = self._get_list_hash(__listconfinfo['arclistpath'])
            __arcname =  os.path.basename(__arcpath)
            if self.__confname:
                __arcname = self.__confname
            else:
                __arcname =  os.path.basename(__arcpath)
            # include custom paths for output conf files
            __confinfo = {'arcname': __arcname,
                            'arcpath': __arcpath,
                            'arcconfpath': __arcconfpath,
                            'arclistpath':  __arclistpath,
                            'arctype': 'tree',
                            'sha512': __listhashsum}
            self._generate_conf(__confinfo)

Example 28

Project: eden
Source File: mergejs.py
View license
def run (sourceDirectory, outputFilename = None, configFile = None):
    cfg = None
    if configFile:
        cfg = Config(configFile)

    allFiles = []

    ## Find all the Javascript source files
    for root, dirs, files in os.walk(sourceDirectory):
        for filename in files:
            if filename.endswith(SUFFIX_JAVASCRIPT) and not filename.startswith("."):
                filepath = os.path.join(root, filename)[len(sourceDirectory)+1:]
                filepath = filepath.replace("\\", "/")
                if cfg and cfg.include:
                    if filepath in cfg.include or filepath in cfg.forceFirst:
                        allFiles.append(filepath)
                elif (not cfg) or (not undesired(filepath, cfg.exclude)):
                    allFiles.append(filepath)

    if cfg:
        # Add files from other locations
        for line in cfg.include:
            if line.startswith(".."):
                allFiles.append(line)

    ## Header inserted at the start of each file in the output
    HEADER = "/* " + "=" * 70 + "\n    %s\n" + "   " + "=" * 70 + " */\n\n"

    files = {}

    ## Import file source code
    ## TODO: Do import when we walk the directories above?
    for filepath in allFiles:
        print "Importing: %s" % filepath
        fullpath = os.path.join(sourceDirectory, filepath).strip()
        content = open(fullpath, "U").read() # TODO: Ensure end of line @ EOF?
        files[filepath] = SourceFile(filepath, content) # TODO: Chop path?

    print

    from toposort import toposort

    complete = False
    resolution_pass = 1

    while not complete:
        complete = True

        ## Resolve the dependencies
        print "Resolution pass %s... " % resolution_pass
        resolution_pass += 1 

        for filepath, info in files.items():
            for path in info.requires:
                if not files.has_key(path):
                    complete = False
                    fullpath = os.path.join(sourceDirectory, path).strip()
                    if os.path.exists(fullpath):
                        print "Importing: %s" % path
                        content = open(fullpath, "U").read() # TODO: Ensure end of line @ EOF?
                        files[path] = SourceFile(path, content) # TODO: Chop path?
                    else:
                        raise MissingImport("File '%s' not found (required by '%s')." % (path, filepath))
        
    # create dictionary of dependencies
    dependencies = {}
    for filepath, info in files.items():
        dependencies[filepath] = info.requires

    print "Sorting..."
    order = toposort(dependencies) #[x for x in toposort(dependencies)]

    ## Move forced first and last files to the required position
    if cfg:
        print "Re-ordering files..."
        order = cfg.forceFirst + [item
                     for item in order
                     if ((item not in cfg.forceFirst) and
                         (item not in cfg.forceLast))] + cfg.forceLast
    
    print
    ## Output the files in the determined order
    result = []

    for fp in order:
        f = files[fp]
        print "Exporting: ", f.filepath
        result.append(HEADER % f.filepath)
        source = f.source
        result.append(source)
        if not source.endswith("\n"):
            result.append("\n")

    print "\nTotal files merged: %d " % len(files)

    if outputFilename:
        print "\nGenerating: %s" % (outputFilename)
        open(outputFilename, "w").write("".join(result))
    return "".join(result)

Example 29

Project: eden
Source File: mergejsmf.py
View license
def getFiles(configDict, configFile = None):
    cfg = None
    if configFile:
        cfg = Config(configFile)

    ## Build array of directories
    allDirs = []
    for k, v in configDict.iteritems():
        if not v in allDirs:
            allDirs.append(v)

    allFiles = []

    ## Find all the Javascript source files
    for sourceDirectory in allDirs:
        for root, dirs, files in os.walk(sourceDirectory):
            for filename in files:
                if filename.endswith(SUFFIX_JAVASCRIPT) and not filename.startswith("."):
                    filepath = os.path.join(root, filename)[len(sourceDirectory)+1:]
                    filepath = filepath.replace("\\", "/")
                    if cfg and cfg.include:
                        if filepath in cfg.include or filepath in cfg.forceFirst:
                            allFiles.append(filepath)
                    elif (not cfg) or (filepath not in cfg.exclude):
                        allFiles.append(filepath)

    files = {}
    order = [] # List of filepaths to output, in a dependency satisfying order 

    ## Import file source code
    ## TODO: Do import when we walk the directories above?
    for filepath in allFiles:
        #print "Importing: %s" % filepath
        if "\\" in filepath:
            filekey = filepath.replace("\\", "/").split("/")[0]
        elif "/" in filepath:
            filekey = filepath.split("/")[0]
        else:
            filekey = "."
        fullpath = os.path.join(configDict[filekey], filepath)
        content = open(fullpath, "U").read() # TODO: Ensure end of line @ EOF?
        files[filepath] = SourceFile(filepath, content) # TODO: Chop path?

    #print

    from toposortmf import toposort

    complete = False
    resolution_pass = 1

    while not complete:
        order = [] # List of filepaths to output, in a dependency satisfying order 
        nodes = []
        routes = []
        ## Resolve the dependencies
        #print "Resolution pass %s... " % resolution_pass
        resolution_pass += 1 

        for filepath, info in files.items():
            nodes.append(filepath)
            for neededFilePath in info.requires:
                routes.append((neededFilePath, filepath))

        for dependencyLevel in toposort(nodes, routes):
            for filepath in dependencyLevel:
                order.append(filepath)
                if not files.has_key(filepath):
                    #print "Importing: %s" % filepath
                    if "\\" in filepath:
                        filekey = filepath.replace("\\", "/").split("/")[0]
                    elif "/" in filepath:
                        filekey = filepath.split("/")[0]
                    else:
                        filekey = "."
                    fullpath = os.path.join(configDict[filekey], filepath)
                    content = open(fullpath, "U").read() # TODO: Ensure end of line @ EOF?
                    files[filepath] = SourceFile(filepath, content) # TODO: Chop path?

        # Double check all dependencies have been met
        complete = True
        try:
            for fp in order:
                if max([order.index(rfp) for rfp in files[fp].requires] +
                       [order.index(fp)]) != order.index(fp):
                    complete = False
        except:
            complete = False
        
        #print    


    ## Move forced first and last files to the required position
    if cfg:
        #print "Re-ordering files..."
        order = cfg.forceFirst + [item
                     for item in order
                     if ((item not in cfg.forceFirst) and
                         (item not in cfg.forceLast))] + cfg.forceLast

    return (files, order)

Example 30

Project: sigal
Source File: gallery.py
View license
    def __init__(self, settings, ncpu=None):
        self.settings = settings
        self.logger = logging.getLogger(__name__)
        self.stats = defaultdict(int)
        self.init_pool(ncpu)
        check_or_create_dir(settings['destination'])

        # Build the list of directories with images
        albums = self.albums = {}
        src_path = self.settings['source']

        ignore_dirs = settings['ignore_directories']
        ignore_files = settings['ignore_files']

        progressChars = cycle(["/", "-", "\\", "|"])
        if self.logger.getEffectiveLevel() >= logging.WARNING:
            self.progressbar_target = None
        else:
            self.progressbar_target = Devnull()

        for path, dirs, files in os.walk(src_path, followlinks=True,
                                         topdown=False):
            if self.logger.getEffectiveLevel() >= logging.WARNING:
                print("\rCollecting albums " + next(progressChars), end="")
            relpath = os.path.relpath(path, src_path)

            # Test if the directory match the ignore_dirs settings
            if ignore_dirs and any(fnmatch.fnmatch(relpath, ignore)
                                   for ignore in ignore_dirs):
                self.logger.info('Ignoring %s', relpath)
                continue

            # Remove files that match the ignore_files settings
            if ignore_files:
                files_path = {join(relpath, f) for f in files}
                for ignore in ignore_files:
                    files_path -= set(fnmatch.filter(files_path, ignore))

                self.logger.debug('Files before filtering: %r', files)
                files = [os.path.split(f)[1] for f in files_path]
                self.logger.debug('Files after filtering: %r', files)

            # Remove sub-directories that have been ignored in a previous
            # iteration (as topdown=False, sub-directories are processed before
            # their parent
            for d in dirs[:]:
                path = join(relpath, d) if relpath != '.' else d
                if path not in albums.keys():
                    dirs.remove(d)

            album = Album(relpath, settings, dirs, files, self)

            if not album.medias and not album.albums:
                self.logger.info('Skip empty album: %r', album)
            else:
                album.create_output_directories()
                albums[relpath] = album

        with progressbar(albums.values(), label="Sorting albums",
                         file=self.progressbar_target) as progress_albums:
            for album in progress_albums:
                album.sort_subdirs(settings['albums_sort_attr'])

        with progressbar(albums.values(), label="Sorting media",
                         file=self.progressbar_target) as progress_albums:
            for album in progress_albums:
                album.sort_medias(settings['medias_sort_attr'])

        self.logger.debug('Albums:\n%r', albums.values())
        signals.gallery_initialized.send(self)

Example 31

Project: subraminion
Source File: subraminion.py
View license
    def process_files(self, verbose=False, delete_duplicates=False, prompt_user=False):
        """
        """
        # http://stackoverflow.com/a/16974952/1044366
        for root, dirs, files in os.walk(self._base_path):
            for f in files:
                if self._file_type_regex and not self._file_type_regex.search(f):
                    # ignore files that do not come under the filter
                    continue
                file_path = os.path.join(root, f)
                if not os.path.isfile(file_path):
                    # ignore any non regular files.
                    continue
                if verbose:
                    print '[processing] %s' % file_path
                try:
                    file_sha1 = self._calculate_sha1(file_path)
                except IOError as e:
                    print str(e)
                    continue
                except Exception as e:
                    # print traceback
                    traceback.print_exc(file=sys.stdout)
                    sys.exit(1)  # abnormal exit
                self._sha1_file_map[file_sha1].append(file_path)

        # generate a duplicate list from the sha file map.
        self._duplicate_file_list = filter(
            lambda x: True if len(x) > 1 else False, self._sha1_file_map.values())

        if delete_duplicates and not prompt_user:
            # exterminator mode!
            self._delete_duplicate_files()
            return

        if delete_duplicates and prompt_user:
            # prompt before deleting
            for i in xrange(len(self._duplicate_file_list)):
                print '- [set %s]' % (i + 1), '-' * 80
                for j in xrange(len(self._duplicate_file_list[i])):
                    print '(%s) %s' % (j + 1, self._duplicate_file_list[i][j])
                print ''
                print 'What do we do now?'
                print '[1] Delete all duplicates.'
                print '[2] Let me pick the ones to delete.'
                exit_flag = False
                while not exit_flag:
                    choice = raw_input('Your choice: ')
                    if choice not in ('1', '2'):
                        print 'Invalid option! Try again...'
                    else:
                        if choice == '1':
                            for file_path in self._duplicate_file_list[i][1:]:
                                print '[deleting] %s' % file_path
                                os.remove(file_path)
                        elif choice == '2':
                            print 'Enter the file id (space separated for multiple)'
                            while True:
                                choice = raw_input('>>')
                                if not set(choice).issubset(set(list('0123456789 '))):
                                    # invalid input
                                    print 'Invalid option! Try again...'
                                else:
                                    # ugly hacks for user input validation :\
                                    choice_list = list(
                                        set((choice + ' ').split(' ')))
                                    # get the null str to the start
                                    choice_list.sort()
                                    choice_list = [int(x)
                                                   for x in choice_list[1:]]
                                    delete_list = list(
                                        set(range(1, len(self._duplicate_file_list[i]) + 1)) & set(choice_list))
                                    invalid_list = list(
                                        set(choice_list) - set(range(1, len(self._duplicate_file_list[i]) + 1)))
                                    for j in invalid_list:
                                        print '[ignoring] Invalid option %s.' % str(j)
                                    for j in delete_list:
                                        # as j starts with 1
                                        print '[deleting] %s' % self._duplicate_file_list[i][j - 1]
                                        os.remove(
                                            self._duplicate_file_list[i][j - 1])
                                    break
                        exit_flag = True
            return

        # just list the duplicate files.
        self._pretty_print_duplicate_file_list()

Example 32

Project: python-scripts
Source File: music-organizer.py
View license
def artist(artistDir):
    print("Organizing artist '" + artistDir + "'.")
    if not args.ignore_multiple_artists:
        artists = set()
        for dirname, dirnames, filenames in os.walk(artistDir):
            # Make sure there aren't a lot of different artists
            # in case this was called from the wrong directory.
            for filename in filenames:
                try:
                    audio = EasyID3(os.path.join(dirname, filename))
                    artist = audio['artist'][0].decode()
                    artists.add(artist)
                except:
                    pass

        if len(artists) > 2:
            while True:
                print("Warning: More than 2 artists found in '{}'.".format(
                    artistDir))
                print("This will move all songs to the '{}' directory.".format(
                    artistDir))
                print("Continue? yes/no")
                choice = raw_input().lower()
                valid = {"yes": True, "y": True, "no": False, "n": False}
                if choice in valid:
                    if valid[choice]:
                        break
                    else:
                        print("Exiting.")
                        sys.exit(-1)

    delete_dirs = []
    for dirname, dirnames, filenames in os.walk(artistDir):
        # Move all the files to the root directory.
        for filename in filenames:
            ext = os.path.splitext(filename)[1]
            if ext == ".mp3":
                fullPath = os.path.join(dirname, filename)
                print("file: " + str(fullPath))

                try:
                    audio = EasyID3(fullPath)
                    title = audio['title'][0].encode('ascii', 'ignore')
                    print("    title: " + title)
                except:
                    title = None

                if not title:
                    print("Error: title not found for '" + filename + "'")
                    sys.exit(-42)

                neatTitle = toNeat(title)
                print("    neatTitle: " + neatTitle)

                newFullPath = os.path.join(artistDir, neatTitle + ext)
                print("    newFullPath: " + newFullPath)

                if newFullPath != fullPath:
                    if os.path.isfile(newFullPath):
                        if args.delete_conflicts:
                            os.remove(fullPath)
                            print("File exists: '" + newFullPath + "'")
                            print("Deleted: '" + fullPath + "'")
                        else:
                            print("Error: File exists: '" + newFullPath + "'")
                            sys.exit(-42)
                    else:
                        os.rename(fullPath, newFullPath)
            elif ext == ".pdf":
                pass
            else:
                if not args.delete_unrecognized:
                    print("Error: Unrecognized file extension in '{}'.".format(
                        filename))
                    sys.exit(-42)

        # Delete all subdirectories.
        for subdirname in dirnames:
            delete_dirs.append(subdirname)

    for d in delete_dirs:
        shutil.rmtree(os.path.join(artistDir, d), ignore_errors=True)

Example 33

Project: iris
Source File: palette.py
View license
def _load_palette():
    """
    Load, configure and register color map palettes and initialise
    color map metadata mappings.

    """
    # Reference these module level namespace variables.
    global CMAP_BREWER, _CMAP_BY_SCHEME, _CMAP_BY_KEYWORD, _CMAP_BY_STD_NAME

    _CMAP_BY_SCHEME = {}
    _CMAP_BY_KEYWORD = {}
    _CMAP_BY_STD_NAME = {}

    filenames = []

    # Identify all .txt color map palette files.
    for root, dirs, files in os.walk(iris.config.PALETTE_PATH):
        # Prune any .svn directory from the tree walk.
        if '.svn' in dirs:
            del dirs[dirs.index('.svn')]
        # Identify any target .txt color map palette files.
        filenames.extend([os.path.join(root, filename)
                          for filename in files
                          if os.path.splitext(filename)[1] == '.txt'])

    for filename in filenames:
        # Default color map name based on the file base-name (case-SENSITIVE).
        cmap_name = os.path.splitext(os.path.basename(filename))[0]
        cmap_scheme = None
        cmap_keywords = []
        cmap_std_names = []
        cmap_type = None

        # Perform default color map interpolation for quantization
        # levels per primary color.
        interpolate_flag = True

        # Read the file header.
        with open(filename) as file_handle:
            header = filter(lambda line: re.match('^\s*#.*:\s+.*$', line),
                            file_handle.readlines())

        # Extract the file header metadata.
        for line in header:
            line = line.replace('#', '', 1).split(':')
            head = line[0].strip().lower()
            body = line[1].strip()

            if head == 'name':
                # Case-SENSITIVE.
                cmap_name = 'brewer_{}'.format(body)

            if head == 'scheme':
                # Case-insensitive.
                cmap_scheme = body.lower()

            if head == 'keyword':
                # Case-insensitive.
                keywords = [part.strip().lower() for part in body.split(',')]
                cmap_keywords.extend(keywords)

            if head == 'std_name':
                # Case-insensitive.
                std_names = [part.strip().lower() for part in body.split(',')]
                cmap_std_names.extend(std_names)

            if head == 'interpolate':
                # Case-insensitive.
                interpolate_flag = body.lower() != 'off'

            if head == 'type':
                # Case-insensitive.
                cmap_type = body.lower()

        # Integrity check for meta-data 'type' field.
        assert cmap_type is not None, \
            'Missing meta-data "type" keyword for color map file, "%s"' % \
            filename
        assert cmap_type == 'rgb', \
            'Invalid type [%s] for color map file "%s"' % (cmap_type, filename)

        # Update the color map look-up dictionaries.
        CMAP_BREWER.add(cmap_name)

        if cmap_scheme is not None:
            scheme_group = _CMAP_BY_SCHEME.setdefault(cmap_scheme, set())
            scheme_group.add(cmap_name)

        for keyword in cmap_keywords:
            keyword_group = _CMAP_BY_KEYWORD.setdefault(keyword, set())
            keyword_group.add(cmap_name)

        for std_name in cmap_std_names:
            std_name_group = _CMAP_BY_STD_NAME.setdefault(std_name, set())
            std_name_group.add(cmap_name)

        # Load palette data and create the associated color map.
        cmap_data = np.loadtxt(filename)
        # Ensure to restrict the number of RGB quantization levels to
        # prevent color map interpolation.

        if interpolate_flag:
            # Perform default color map interpolation for quantization
            # levels per primary color.
            cmap = mpl_colors.LinearSegmentedColormap.from_list(
                cmap_name, cmap_data)
        else:
            # Restrict quantization levels per primary color (turn-off
            # interpolation).
            # Typically used for Brewer color maps.
            cmap = mpl_colors.LinearSegmentedColormap.from_list(
                cmap_name, cmap_data, N=len(cmap_data))

        # Register the color map for use.
        mpl_cm.register_cmap(cmap=cmap)

Example 34

Project: glm-parser
Source File: data_pool.py
View license
    def load(self, formatPath, sparkContext=None):
        """
        For each section in the initializer, iterate through all files
        under that section directory, and load the content of each
        individual file into the class instance.

        This method should be called after section regex has been initalized
        and before any get_data method is called.
        """
        logging.debug("Loading data...")
        print("Loading data...")

        # Load format file
        print("Loading dataFormat from: " + formatPath)
        if self.hadoop == True:
            fformat = fileRead(formatPath, sparkContext=sparkContext)
        else:
            fformat = open(formatPath)

        self.field_name_list = []
        self.comment_sign = ''

        remaining_field_names = 0
        for line in fformat:
            format_line = line.strip().split()

            if remaining_field_names > 0:
                self.field_name_list.append(line.strip())
                remaining_field_names -= 1

            if format_line[0] == "field_names:":
                remaining_field_names = int(format_line[1])

            if format_line[0] == "comment_sign:":
                self.comment_sign = format_line[1]

        if self.field_name_list == []:
            raise RuntimeError("DATAPOOL [ERROR]: format file read failure")

        if self.hadoop == False:
            fformat.close()

        # Load data
        if self.hadoop == True:
            self.dataPrep.loadHadoop()
        else:
            self.dataPrep.loadLocal()

        # Add data to data_list
        # If using yarn mode, local data will not be loaded
        if self.hadoop == False:
            for dirName, subdirList, fileList in os.walk(self.dataPrep.localPath()):
                for file_name in fileList:
                    file_path = "%s/%s" % ( str(dirName), str(file_name) )
                    self.data_list += self.get_data_list(file_path)
        else:
            aRdd = sparkContext.textFile(self.dataPrep.hadoopPath()).cache()
            tmp  = aRdd.collect()
            tmpStr = ''.join(str(e)+"\n" for e in tmp)
            self.load_stringtext(textString  = tmpStr,
                                format_list  = self.field_name_list,
                                comment_sign = self.comment_sign)
        return

Example 35

Project: will
Source File: main.py
View license
    def bootstrap_plugins(self):
        puts("Bootstrapping plugins...")
        OTHER_HELP_HEADING = "Other"
        plugin_modules = {}
        plugin_modules_library = {}

        # NOTE: You can't access self.storage here, or it will deadlock when the threads try to access redis.
        with indent(2):
            parent_help_text = None
            for plugin_name, plugin_root in self.plugins_dirs.items():
                for root, dirs, files in os.walk(plugin_root, topdown=False):
                    for f in files:
                        if f[-3:] == ".py" and f != "__init__.py":
                            try:
                                module_path = os.path.join(root, f)
                                path_components = os.path.split(module_path)
                                module_name = path_components[-1][:-3]
                                full_module_name = ".".join(path_components)
                                # Need to pass along module name, path all the way through
                                combined_name = ".".join([plugin_name, module_name])

                                # Check blacklist.
                                blacklisted = False
                                for b in settings.PLUGIN_BLACKLIST:
                                    if b in combined_name:
                                        blacklisted = True

                                try:
                                    plugin_modules[full_module_name] = imp.load_source(module_name, module_path)
                                except:
                                    # If it's blacklisted, don't worry if this blows up.
                                    if blacklisted:
                                        pass
                                    else:
                                        raise

                                parent_mod = path_components[-2].split("/")[-1]
                                parent_help_text = parent_mod.title()
                                try:
                                    parent_root = os.path.join(root, "__init__.py")
                                    parent = imp.load_source(parent_mod, parent_root)
                                    parent_help_text = getattr(parent, "MODULE_DESCRIPTION", parent_help_text)
                                except:
                                    # If it's blacklisted, don't worry if this blows up.
                                    if blacklisted:
                                        pass
                                    else:
                                        raise

                                plugin_modules_library[full_module_name] = {
                                    "full_module_name": full_module_name,
                                    "file_path": module_path,
                                    "name": module_name,
                                    "parent_name": plugin_name,
                                    "parent_module_name": parent_mod,
                                    "parent_help_text": parent_help_text,
                                    "blacklisted": blacklisted,
                                }
                            except Exception, e:
                                self.startup_error("Error loading %s" % (module_path,), e)

                self.plugins = []
                for name, module in plugin_modules.items():
                    try:
                        for class_name, cls in inspect.getmembers(module, predicate=inspect.isclass):
                            try:
                                if hasattr(cls, "is_will_plugin") and cls.is_will_plugin and class_name != "WillPlugin":
                                    self.plugins.append({
                                        "name": class_name,
                                        "class": cls,
                                        "module": module,
                                        "full_module_name": name,
                                        "parent_name": plugin_modules_library[name]["parent_name"],
                                        "parent_module_name": plugin_modules_library[name]["parent_module_name"],
                                        "parent_help_text": plugin_modules_library[name]["parent_help_text"],
                                        "blacklisted": plugin_modules_library[name]["blacklisted"],
                                    })
                            except Exception, e:
                                self.startup_error("Error bootstrapping %s" % (class_name,), e)
                    except Exception, e:
                        self.startup_error("Error bootstrapping %s" % (name,), e)

            self._plugin_modules_library = plugin_modules_library

            # Sift and Sort.
            self.message_listeners = []
            self.periodic_tasks = []
            self.random_tasks = []
            self.bottle_routes = []
            self.all_listener_regexes = []
            self.help_modules = {}
            self.help_modules[OTHER_HELP_HEADING] = []
            self.some_listeners_include_me = False
            self.plugins.sort(key=operator.itemgetter("parent_module_name"))
            self.required_settings_from_plugins = {}
            last_parent_name = None
            for plugin_info in self.plugins:
                try:
                    if last_parent_name != plugin_info["parent_help_text"]:
                        friendly_name = "%(parent_help_text)s " % plugin_info
                        module_name = " %(parent_name)s" % plugin_info
                        # Justify
                        friendly_name = friendly_name.ljust(50, '-')
                        module_name = module_name.rjust(40, '-')
                        puts("")
                        puts("%s%s" % (friendly_name, module_name))

                        last_parent_name = plugin_info["parent_help_text"]
                    with indent(2):
                        plugin_name = plugin_info["name"]
                        # Just a little nicety
                        if plugin_name[-6:] == "Plugin":
                            plugin_name = plugin_name[:-6]
                        if plugin_info["blacklisted"]:
                            puts("✗ %s (blacklisted)" % plugin_name)
                        else:
                            plugin_instances = {}
                            for function_name, fn in inspect.getmembers(
                                plugin_info["class"],
                                predicate=inspect.ismethod
                            ):
                                try:
                                    # Check for required_settings
                                    with indent(2):
                                        if hasattr(fn, "will_fn_metadata"):
                                            meta = fn.will_fn_metadata
                                            if "required_settings" in meta:
                                                for s in meta["required_settings"]:
                                                    self.required_settings_from_plugins[s] = {
                                                        "plugin_name": plugin_name,
                                                        "function_name": function_name,
                                                        "setting_name": s,
                                                    }
                                            if (
                                                "listens_to_messages" in meta and
                                                meta["listens_to_messages"] and
                                                "listener_regex" in meta
                                            ):
                                                # puts("- %s" % function_name)
                                                regex = meta["listener_regex"]
                                                if not meta["case_sensitive"]:
                                                    regex = "(?i)%s" % regex
                                                help_regex = meta["listener_regex"]
                                                if meta["listens_only_to_direct_mentions"]:
                                                    help_regex = "@%s %s" % (settings.HANDLE, help_regex)
                                                self.all_listener_regexes.append(help_regex)
                                                if meta["__doc__"]:
                                                    pht = plugin_info.get("parent_help_text", None)
                                                    if pht:
                                                        if pht in self.help_modules:
                                                            self.help_modules[pht].append(meta["__doc__"])
                                                        else:
                                                            self.help_modules[pht] = [meta["__doc__"]]
                                                    else:
                                                        self.help_modules[OTHER_HELP_HEADING].append(meta["__doc__"])
                                                if meta["multiline"]:
                                                    compiled_regex = re.compile(regex, re.MULTILINE | re.DOTALL)
                                                else:
                                                    compiled_regex = re.compile(regex)

                                                if plugin_info["class"] in plugin_instances:
                                                    instance = plugin_instances[plugin_info["class"]]
                                                else:
                                                    instance = plugin_info["class"]()
                                                    plugin_instances[plugin_info["class"]] = instance

                                                self.message_listeners.append({
                                                    "function_name": function_name,
                                                    "class_name": plugin_info["name"],
                                                    "regex_pattern": meta["listener_regex"],
                                                    "regex": compiled_regex,
                                                    "fn": getattr(instance, function_name),
                                                    "args": meta["listener_args"],
                                                    "include_me": meta["listener_includes_me"],
                                                    "direct_mentions_only": meta["listens_only_to_direct_mentions"],
                                                    "admin_only": meta["listens_only_to_admin"],
                                                    "acl": meta["listeners_acl"],
                                                })
                                                if meta["listener_includes_me"]:
                                                    self.some_listeners_include_me = True
                                            elif "periodic_task" in meta and meta["periodic_task"]:
                                                # puts("- %s" % function_name)
                                                self.periodic_tasks.append((plugin_info, fn, function_name))
                                            elif "random_task" in meta and meta["random_task"]:
                                                # puts("- %s" % function_name)
                                                self.random_tasks.append((plugin_info, fn, function_name))
                                            elif "bottle_route" in meta:
                                                # puts("- %s" % function_name)
                                                self.bottle_routes.append((plugin_info["class"], function_name))
                                except Exception, e:
                                    error(plugin_name)
                                    self.startup_error(
                                        "Error bootstrapping %s.%s" % (
                                            plugin_info["class"],
                                            function_name,
                                        ), e
                                    )
                            show_valid(plugin_name)
                except Exception, e:
                    self.startup_error("Error bootstrapping %s" % (plugin_info["class"],), e)
        puts("")

Example 36

Project: luigi
Source File: hadoop.py
View license
def create_packages_archive(packages, filename):
    """
    Create a tar archive which will contain the files for the packages listed in packages.
    """
    import tarfile
    tar = tarfile.open(filename, "w")

    def add(src, dst):
        logger.debug('adding to tar: %s -> %s', src, dst)
        tar.add(src, dst)

    def add_files_for_package(sub_package_path, root_package_path, root_package_name):
        for root, dirs, files in os.walk(sub_package_path):
            if '.svn' in dirs:
                dirs.remove('.svn')
            for f in files:
                if not f.endswith(".pyc") and not f.startswith("."):
                    add(dereference(root + "/" + f), root.replace(root_package_path, root_package_name) + "/" + f)

    for package in packages:
        # Put a submodule's entire package in the archive. This is the
        # magic that usually packages everything you need without
        # having to attach packages/modules explicitly
        if not getattr(package, "__path__", None) and '.' in package.__name__:
            package = __import__(package.__name__.rpartition('.')[0], None, None, 'non_empty')

        n = package.__name__.replace(".", "/")

        if getattr(package, "__path__", None):
            # TODO: (BUG) picking only the first path does not
            # properly deal with namespaced packages in different
            # directories
            p = package.__path__[0]

            if p.endswith('.egg') and os.path.isfile(p):
                raise 'egg files not supported!!!'
                # Add the entire egg file
                # p = p[:p.find('.egg') + 4]
                # add(dereference(p), os.path.basename(p))

            else:
                # include __init__ files from parent projects
                root = []
                for parent in package.__name__.split('.')[0:-1]:
                    root.append(parent)
                    module_name = '.'.join(root)
                    directory = '/'.join(root)

                    add(dereference(__import__(module_name, None, None, 'non_empty').__path__[0] + "/__init__.py"),
                        directory + "/__init__.py")

                add_files_for_package(p, p, n)

                # include egg-info directories that are parallel:
                for egg_info_path in glob.glob(p + '*.egg-info'):
                    logger.debug(
                        'Adding package metadata to archive for "%s" found at "%s"',
                        package.__name__,
                        egg_info_path
                    )
                    add_files_for_package(egg_info_path, p, n)

        else:
            f = package.__file__
            if f.endswith("pyc"):
                f = f[:-3] + "py"
            if n.find(".") == -1:
                add(dereference(f), os.path.basename(f))
            else:
                add(dereference(f), n + ".py")
    tar.close()

Example 37

Project: buildfox
Source File: lib_util.py
View license
def glob_folders(pattern, base_path, generated, excluded_dirs):
	if not pattern.endswith("/"): # this shouldn't fail
		raise ValueError("pattern should always end with \"/\", but got \"%s\"" % pattern)

	real_folders = [base_path.rstrip("/")]
	gen_folders = [base_path.rstrip("/")]

	pattern = pattern[2:] if pattern.startswith("./") else pattern

	for folder in re_pattern_split.split(pattern):
		recursive_match = re_recursive_glob_noslash.match(folder)
		if recursive_match:
			regex_filter = recursive_match.group(1)
			re_regex_filter = re.compile("^%s.*$" % regex_filter) if regex_filter else None

			new_real_folders = []
			for real_folder in real_folders:
				new_real_folders.append(real_folder)
				for root, dirs, filenames in os.walk(real_folder, topdown = True): # TODO this is slow, optimize
					dirs[:] = [dir for dir in dirs if dir not in excluded_dirs]
					if re_regex_filter:
						dirs[:] = [dir for dir in dirs if re_regex_filter.match(dir)]
					for dir in dirs:
						result = os.path.join(root, dir).replace("\\", "/")
						new_real_folders.append(result)
			real_folders = new_real_folders

			new_gen_folders = []
			for gen_folder in gen_folders:
				prepend_dot = False
				if gen_folder.startswith("./"):
					prepend_dot = True
					gen_folder = gen_folder[2:] # strip ./

				gen_folder_len = len(gen_folder)
				for folder in generated.keys():
					if folder.startswith(gen_folder):
						root = folder[:gen_folder_len]
						sub_folders = folder[gen_folder_len:]
						sub_folders = sub_folders.lstrip("/").rstrip("/")
						# walk through directories in similar fashion with os.walk
						new_gen_folders.append("./%s" % root if prepend_dot else root)
						for subfolder in sub_folders.split("/"): 
							if subfolder in excluded_dirs:
								break
							if re_regex_filter and not re_regex_filter.match(subfolder):
								break
							root += "/%s" % subfolder
							new_gen_folders.append("./%s" % root if prepend_dot else root)
			gen_folders = list(set(new_gen_folders))
		else:
			real_folders = ["%s/%s" % (p, folder) for p in real_folders]
			gen_folders = ["%s/%s" % (p, folder) for p in gen_folders]

	return (real_folders, gen_folders)

Example 38

Project: FoBiS
Source File: utils.py
View license
def dependency_hiearchy(builder, pfiles, print_w=None, force_compile=False):
  """
  Build parsed files hierarchy.

  Paramters
  ---------
  builder : Builder object
  pfiles : list
    list of ParsedFile objects
  print_w : function
    function for printing emphized warning message
  force_compile : {False}
    flag for forcing (re-)compiling of all dependency
  """
  if print_w is None:
    print_wm = print_fake
  else:
    print_wm = print_w

  # direct dependencies list used after for building indirect (complete) dependencies list
  for parsed_file in pfiles:
    parsed_file.pfile_dep = []
    for dep in parsed_file.dependencies:
      if dep.type == "module":
        dep.file, fnum = module_is_in(pfiles=pfiles, module=dep.name)
        if fnum > -1:
          if not pfiles[fnum] in parsed_file.pfile_dep:
            parsed_file.pfile_dep.append(pfiles[fnum])
        else:
          # there is no source containg the searched module, try to find it into a precompiled mod file into eventually included paths
          for dinc in builder.dinc:
            for _, _, files in os.walk(dinc):
              for filename in files:
                if os.path.splitext(os.path.basename(filename))[0].lower() == dep.name.lower() and os.path.splitext(os.path.basename(filename))[1].lower() == '.mod':
                  fnum = 0
                  break
              if fnum == 0:
                break
            if fnum == 0:
              break
          if fnum != 0:
            print_wm("Attention: the file '" + parsed_file.name + "' depends on '" + dep.name + "' that is unreachable")
            # sys.exit(1)
      if dep.type == "include":
        dep.file, fnum = include_is_in(pfiles=pfiles, include=dep.name)
        if fnum > -1:
          if not pfiles[fnum] in parsed_file.pfile_dep:
            pfiles[fnum].program = False
            pfiles[fnum].module = False
            pfiles[fnum].nomodlib = False
            pfiles[fnum].include = True
            parsed_file.pfile_dep.append(pfiles[fnum])
            if not os.path.dirname(pfiles[fnum].name) in builder.dinc:
              builder.dinc.append(os.path.dirname(pfiles[fnum].name))
        else:
          print_wm("Attention: the file '" + parsed_file.name + "' depends on '" + dep.name + "' that is unreachable")
          # sys.exit(1)
  # indirect dependency list
  for parsed_file in pfiles:
    parsed_file.create_pfile_dep_all()
  # using the just created hiearchy for checking which files must be (re-)compiled
  for parsed_file in pfiles:
    parsed_file.check_compile(obj_dir=builder.obj_dir, force_compile=force_compile)

Example 39

Project: xbmc-addon-watchdog
Source File: inotify_c.py
View license
    def read_events(self, event_buffer_size=DEFAULT_EVENT_BUFFER_SIZE):
        """
        Reads events from inotify and yields them.
        """
        # HACK: We need to traverse the directory path
        # recursively and simulate events for newly
        # created subdirectories/files. This will handle
        # mkdir -p foobar/blah/bar; touch foobar/afile

        def _recursive_simulate(src_path):
            events = []
            for root, dirnames, filenames in os.walk(src_path):
                for dirname in dirnames:
                    try:
                        full_path = os.path.join(root, dirname)
                        wd_dir = self._add_watch(full_path, self._event_mask)
                        e = InotifyEvent(
                            wd_dir, InotifyConstants.IN_CREATE | InotifyConstants.IN_ISDIR, 0, dirname, full_path)
                        events.append(e)
                    except OSError:
                        pass
                for filename in filenames:
                    full_path = os.path.join(root, filename)
                    wd_parent_dir = self._wd_for_path[os.path.dirname(full_path)]
                    e = InotifyEvent(
                        wd_parent_dir, InotifyConstants.IN_CREATE, 0, filename, full_path)
                    events.append(e)
            return events

        event_buffer = None
        while True:
            try:
                event_buffer = os.read(self._inotify_fd, event_buffer_size)
            except OSError as e:
                if e.errno == errno.EINTR:
                    continue
            break

        with self._lock:
            event_list = []
            for wd, mask, cookie, name in Inotify._parse_event_buffer(event_buffer):
                if wd == -1:
                    continue
                wd_path = self._path_for_wd[wd]
                src_path = os.path.join(wd_path, name) if name else wd_path #avoid trailing slash
                inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)

                if inotify_event.is_moved_from:
                    self.remember_move_from_event(inotify_event)
                elif inotify_event.is_moved_to:
                    move_src_path = self.source_for_move(inotify_event)
                    if move_src_path in self._wd_for_path:
                        moved_wd = self._wd_for_path[move_src_path]
                        del self._wd_for_path[move_src_path]
                        self._wd_for_path[inotify_event.src_path] = moved_wd
                        self._path_for_wd[moved_wd] = inotify_event.src_path
                    src_path = os.path.join(wd_path, name)
                    inotify_event = InotifyEvent(wd, mask, cookie, name, src_path)

                if inotify_event.is_ignored:
                    # Clean up book-keeping for deleted watches.
                    self._remove_watch_bookkeeping(src_path)
                    continue

                event_list.append(inotify_event)

                if (self.is_recursive and inotify_event.is_directory and
                        inotify_event.is_create):

                    # TODO: When a directory from another part of the
                    # filesystem is moved into a watched directory, this
                    # will not generate events for the directory tree.
                    # We need to coalesce IN_MOVED_TO events and those
                    # IN_MOVED_TO events which don't pair up with
                    # IN_MOVED_FROM events should be marked IN_CREATE
                    # instead relative to this directory.
                    try:
                        self._add_watch(src_path, self._event_mask)
                    except OSError:
                        continue

                    event_list.extend(_recursive_simulate(src_path))

        return event_list

Example 40

Project: topera
Source File: pricillapluginmanager.py
View license
    def find_plugins(self, plugins_folder):
        """
        Find plugins in the given folder.

        The folder must contain one subfolder for each plugin category,
        inside which are the plugins.

        Each plugin is defined in a file with the ".golismero" extension.
        The code for each plugin must be in a Python script within the same
        folder as the ".golismero" file, or within any subdirectory of it.

        :param plugins_folder: Folder where to look for plugins.
        :type plugins_folder: str

        :returns: tuple(list, list) -- A list of plugins loaded, and a list of plugins that failed to load.
        """

        # Make sure the plugins folder is an absolute path
        plugins_folder = path.abspath(plugins_folder)

        # Raise an exception if the plugins folder doesn't exist or isn't a folder
        if not path.isdir(plugins_folder):
            raise ValueError("Invalid plugin folder: %s" % plugins_folder)

        # List to collect the plugins that loaded successfully
        success = list()

        # List to collect the plugins that failed to load
        failure = list()


        # The following levels belong to the plugins
        for (dirpath, dirnames, filenames) in walk(plugins_folder):

            # Look for plugin descriptor files
            for fname in filenames:
                if not fname.endswith(".topera"):
                    continue

                # Convert the plugin descriptor filename to an absolute path
                fname = path.abspath(path.join(dirpath, fname))

                # The plugin name is the relative path + filename without extension,
                # where the path separator is always "/" regardless of the current OS
                plugin_name = path.splitext(fname)[0][len(plugins_folder):]
                if plugin_name[0] == path.sep:
                    plugin_name = plugin_name[1:]
                if path.sep != "/":
                    plugin_name = plugin_name.replace(path.sep, "/")

                # If the plugin name already exists, skip it
                if plugin_name in self.__plugins:
                    failure.append(plugin_name)
                    continue

                # Parse the plugin descriptor file
                try:
                    plugin_info = PluginInfo(plugin_name, fname)

                    # Collect the plugin info
                    self.__plugins[plugin_name] = plugin_info

                    # Add the plugin name to the success list
                    success.append(plugin_name)

                # On error add the plugin name to the list of failures
                except Exception:
                    failure.append(plugin_name)

        return success, failure

Example 41

Project: Gloo
Source File: gloo.py
View license
    def load_project(self):
        """
        Loads the project into the workspace

        Does four things currently
          1. Recursively loads you csv files into data frames
             prepending the folder if not in data
          2. Runs files in the munge folder.  These are
             preprocessing scripts
          3. Imports files in lib
          4. Starts Logging

        """

        shell = get_ipython()

        with open('.gloo', 'r') as f:
            config = pickle.load(f)

        filename = lambda x: x.split('.')[0]

        #Push variables namespace
        vars_to_push = {}
        for directory, _, datafiles in os.walk('data', topdown=False):
            for datafile in datafiles:
                if directory == 'data':
                    var_name = filename(datafile)
                    read_location = os.path.join('data', datafile)
                    vars_to_push[var_name] = pandas.read_csv(read_location)
                else:
                    var_name = os.path.split(directory)[-1] + '_' \
                            + filename(datafile)
                    read_location = os.path.join(directory, datafile)
                    vars_to_push[var_name] = pandas.read_csv(read_location)
        shell.push(vars_to_push)

        #Run munge file
        mungefiles = os.listdir('munge')
        for mungefile in mungefiles:
            shell.magic('run -i munge/%s' % mungefile)

        #import lib files
        libfiles = os.listdir('lib')
        libs_to_push = {}
        os.chdir('lib')
        for libfile in libfiles:
            mod = filename(libfile)
            libs_to_push[mod] = __import__(filename(libfile))
        shell.push(libs_to_push)
        os.chdir('..')

        #Import packages
        if self.packages:
            packages = self.packages
            for package in packages:
                if isinstance(package, str):
                    shell.runcode('import %s' % package)
                elif isinstance(package, tuple):
                    package_name, alias = package
                    shell.runcode('import %s as %s' % (package_name, alias))

        if config['logging']:
            shell.magic_logstart(os.getcwd().split('/')[-1]
              + '_logging.py append')

Example 42

Project: QGIS-CKAN-Browser
Source File: util.py
View license
    def add_lyrs_from_dir(self, data_dir, layer_name, layer_url):
        try:
            file_types = (
                '*.[Ss][Hh][Pp]',
                '*.[Gg][Mm][Ll]',
                '*.[Gg][Ee][Oo][Rr][Ss][Ss]',
                '*.[Xx][Mm][Ll]',
                '*.[Cc][Ss][Vv]',
                '*.[Tt][Xx][Tt]',
                '*.[Pp][Dd][Ff]',
                '*.[Tt][Ii][Ff]',
                '*.[Tt][Ii][Ff][Ff]',
                '*.[Aa][Ss][Cc]',
                '*.[Qq][Ll][Rr]',
                '*.[Ii][Mm][Gg]',
                '*.[Jj][2][Kk]',
                '*.[Jj][Pp][2]',
                '*.[Rr][Ss][Tt]',
                '*.[Dd][Ee][Mm]',
                '*.[Ww][Mm][Tt][Ss]',
                '*.[Ww][Mm][Ss]',
                '*.[Ww][Ff][Ss]',
                '*.[Kk][Mm][Ll]',
                '*.[Xx][Ll][Ss]',
                '*.[Xx][Ll][Ss][Xx]',
                '*.[Dd][Oo][Cc]',
                '*.[Dd][Oo][Cc][Xx]',
                '*.[Jj][Pp][Gg]',
                '*.[Jj][Pp][Ee][Gg]',
                '*.[Pp][Nn][Gg]',
                '*.*[Jj][Ss][Oo][Nn]'
            )
            geo_files = []
            for file_type in file_types:
                ### python 3.5
                # glob_term = os.path.join(data_dir, '**', file_type)
                # geo_files.extend(glob.glob(glob_term))
                ### python > 2.2
                for root, dir_names, file_names in os.walk(data_dir):
                    for file_name in filter(file_names, file_type):
                        geo_files.append(os.path.join(root, file_name))

            self.msg_log(u'add lyrs: {0}'.format(data_dir))
            self.msg_log(u'add lyrs: {0}'.format('\n'.join(geo_files)))

            if len(geo_files) < 1:
                self.msg_log('len(geo_files)<1')
#                 return False, u'Keine anzeigbaren Daten gefunden in\n{0}.\n\n\n     ===----!!!TODO!!!----===\n\nBenutzer anbieten Verzeichnis zu öffnen'.format(dir)
                return False, {"message": "unknown fileytpe", "dir_path": data_dir}
            for geo_file in geo_files:
                if os.path.basename(geo_file).lower().endswith('.shp.xml'):
                    self.msg_log(u'skipping {0}'.format(geo_file))
                    continue
                self.msg_log(geo_file)
                full_path = os.path.join(data_dir, geo_file)
                full_layer_name = layer_name + ' - ' + os.path.basename(geo_file)
                low_case = os.path.basename(geo_file).lower()
                lyr = None
                
                if low_case.endswith('json'): 
                    self.msg_log(u'Open JSON')
                    if False is self.__is_geojson(full_path):
                        if self.__open_with_system(full_path) > 0:
                            if QMessageBox.Yes == self.dlg_yes_no(self.tr(u'py_dlg_base_open_manager').format(layer_url)):
                                self.open_in_manager(data_dir)
                        continue
                
                if( 
                        low_case.endswith('.txt') or 
                        low_case.endswith('.pdf') or
                        low_case.endswith('.xls') or
                        low_case.endswith('.xlsx') or
                        low_case.endswith('.doc') or
                        low_case.endswith('.docx') or
                        low_case.endswith('.jpg') or
                        low_case.endswith('.jpeg') or
                        low_case.endswith('.png')
                    ):
                    if self.__open_with_system(full_path) > 0:
                        if QMessageBox.Yes == self.dlg_yes_no(self.tr(u'py_dlg_base_open_manager').format(layer_url)):
                            self.open_in_manager(data_dir)
                    continue
                elif low_case.endswith('.qlr'):
                    lyr = self.__add_layer_definition_file(
                        full_path,
                        QgsProject.instance().layerTreeRoot()
                    )
                elif low_case.endswith('.wmts') or low_case.endswith('.wms'): # for now, we assume it's a WMTS
                    self.msg_log(u'Open WM(T)S')
                    self._open_wmts(layer_name, layer_url)
                    continue
                elif low_case.endswith('.wfs'): # for now, we assume it's a WMTS
                    self.msg_log(u'Open WFS')
                    self._open_wfs(layer_name, layer_url)
                    continue
                elif low_case.endswith('.csv'):
#                     lyr = self.__add_csv_table(full_path, full_layer_name)
                    self.msg_log(u'Open CSV')
                    self._open_csv(full_path)
                    continue
                elif( 
                        low_case.endswith('.asc') or 
                        low_case.endswith('.tif') or 
                        low_case.endswith('.tiff') or 
                        low_case.endswith('.img') or 
                        low_case.endswith('.jp2') or 
                        low_case.endswith('.j2k') or 
                        low_case.endswith('.rst') or 
                        low_case.endswith('.dem')
                    ):
                    lyr = self.__add_raster_layer(full_path, full_layer_name)
                else:
                    lyr = self.__add_vector_layer(full_path, full_layer_name)
                if lyr is not None:
                    if not lyr.isValid():
                        self.msg_log(u'not valid: {0}'.format(full_path))
                        if QMessageBox.Yes == self.dlg_yes_no(self.tr(u'py_dlg_base_open_manager').format(layer_url)):
                            self.open_in_manager(data_dir)
                        continue
                    QgsMapLayerRegistry.instance().addMapLayer(lyr)
                else:
                    self.msg_log(u'could not add layer: {0}'.format(full_path))
            return True, None
        except AttributeError as ae:
            return False, ae.message
        except TypeError as te:
            return False, te.message
        except:
            return False, sys.exc_info()[0]

Example 43

Project: QGIS-CKAN-Browser
Source File: util.py
View license
    def add_lyrs_from_dir(self, data_dir, layer_name, layer_url):
        try:
            file_types = (
                '*.[Ss][Hh][Pp]',
                '*.[Gg][Mm][Ll]',
                '*.[Gg][Ee][Oo][Rr][Ss][Ss]',
                '*.[Xx][Mm][Ll]',
                '*.[Cc][Ss][Vv]',
                '*.[Tt][Xx][Tt]',
                '*.[Pp][Dd][Ff]',
                '*.[Tt][Ii][Ff]',
                '*.[Tt][Ii][Ff][Ff]',
                '*.[Aa][Ss][Cc]',
                '*.[Qq][Ll][Rr]',
                '*.[Ii][Mm][Gg]',
                '*.[Jj][2][Kk]',
                '*.[Jj][Pp][2]',
                '*.[Rr][Ss][Tt]',
                '*.[Dd][Ee][Mm]',
                '*.[Ww][Mm][Tt][Ss]',
                '*.[Ww][Mm][Ss]',
                '*.[Ww][Ff][Ss]',
                '*.[Kk][Mm][Ll]',
                '*.[Xx][Ll][Ss]',
                '*.[Xx][Ll][Ss][Xx]',
                '*.[Dd][Oo][Cc]',
                '*.[Dd][Oo][Cc][Xx]',
                '*.[Jj][Pp][Gg]',
                '*.[Jj][Pp][Ee][Gg]',
                '*.[Pp][Nn][Gg]',
                '*.*[Jj][Ss][Oo][Nn]'
            )
            geo_files = []
            for file_type in file_types:
                ### python 3.5
                # glob_term = os.path.join(data_dir, '**', file_type)
                # geo_files.extend(glob.glob(glob_term))
                ### python > 2.2
                for root, dir_names, file_names in os.walk(data_dir):
                    for file_name in filter(file_names, file_type):
                        geo_files.append(os.path.join(root, file_name))

            self.msg_log(u'add lyrs: {0}'.format(data_dir))
            self.msg_log(u'add lyrs: {0}'.format('\n'.join(geo_files)))

            if len(geo_files) < 1:
                self.msg_log('len(geo_files)<1')
#                 return False, u'Keine anzeigbaren Daten gefunden in\n{0}.\n\n\n     ===----!!!TODO!!!----===\n\nBenutzer anbieten Verzeichnis zu öffnen'.format(dir)
                return False, {"message": "unknown fileytpe", "dir_path": data_dir}
            for geo_file in geo_files:
                if os.path.basename(geo_file).lower().endswith('.shp.xml'):
                    self.msg_log(u'skipping {0}'.format(geo_file))
                    continue
                self.msg_log(geo_file)
                full_path = os.path.join(data_dir, geo_file)
                full_layer_name = layer_name + ' - ' + os.path.basename(geo_file)
                low_case = os.path.basename(geo_file).lower()
                lyr = None
                
                if low_case.endswith('json'): 
                    self.msg_log(u'Open JSON')
                    if False is self.__is_geojson(full_path):
                        if self.__open_with_system(full_path) > 0:
                            if QMessageBox.Yes == self.dlg_yes_no(self.tr(u'py_dlg_base_open_manager').format(layer_url)):
                                self.open_in_manager(data_dir)
                        continue
                
                if( 
                        low_case.endswith('.txt') or 
                        low_case.endswith('.pdf') or
                        low_case.endswith('.xls') or
                        low_case.endswith('.xlsx') or
                        low_case.endswith('.doc') or
                        low_case.endswith('.docx') or
                        low_case.endswith('.jpg') or
                        low_case.endswith('.jpeg') or
                        low_case.endswith('.png')
                    ):
                    if self.__open_with_system(full_path) > 0:
                        if QMessageBox.Yes == self.dlg_yes_no(self.tr(u'py_dlg_base_open_manager').format(layer_url)):
                            self.open_in_manager(data_dir)
                    continue
                elif low_case.endswith('.qlr'):
                    lyr = self.__add_layer_definition_file(
                        full_path,
                        QgsProject.instance().layerTreeRoot()
                    )
                elif low_case.endswith('.wmts') or low_case.endswith('.wms'): # for now, we assume it's a WMTS
                    self.msg_log(u'Open WM(T)S')
                    self._open_wmts(layer_name, layer_url)
                    continue
                elif low_case.endswith('.wfs'): # for now, we assume it's a WMTS
                    self.msg_log(u'Open WFS')
                    self._open_wfs(layer_name, layer_url)
                    continue
                elif low_case.endswith('.csv'):
#                     lyr = self.__add_csv_table(full_path, full_layer_name)
                    self.msg_log(u'Open CSV')
                    self._open_csv(full_path)
                    continue
                elif( 
                        low_case.endswith('.asc') or 
                        low_case.endswith('.tif') or 
                        low_case.endswith('.tiff') or 
                        low_case.endswith('.img') or 
                        low_case.endswith('.jp2') or 
                        low_case.endswith('.j2k') or 
                        low_case.endswith('.rst') or 
                        low_case.endswith('.dem')
                    ):
                    lyr = self.__add_raster_layer(full_path, full_layer_name)
                else:
                    lyr = self.__add_vector_layer(full_path, full_layer_name)
                if lyr is not None:
                    if not lyr.isValid():
                        self.msg_log(u'not valid: {0}'.format(full_path))
                        if QMessageBox.Yes == self.dlg_yes_no(self.tr(u'py_dlg_base_open_manager').format(layer_url)):
                            self.open_in_manager(data_dir)
                        continue
                    QgsMapLayerRegistry.instance().addMapLayer(lyr)
                else:
                    self.msg_log(u'could not add layer: {0}'.format(full_path))
            return True, None
        except AttributeError as ae:
            return False, ae.message
        except TypeError as te:
            return False, te.message
        except:
            return False, sys.exc_info()[0]

Example 44

Project: Veil-Evasion
Source File: controller.py
View license
    def OutputMenu(self, payload, code, showTitle=True, interactive=True, args=None):
        """
        Write a chunk of payload code to a specified ouput file base.
        Also outputs a handler script if required from the options.

        code = the source code to write
        OutputBaseChoice = "payload" or user specified string

        Returns the full name the source was written to.
        """

        OutputBaseChoice = ""
        overwrite = False

        # if we have arguments passed, extract out the values we want
        if args:
            OutputBaseChoice = args.o
            overwrite = args.overwrite

        # if we get .exe or ELF (with no base) code back, output to the compiled folder, otherwise write to the source folder
        if payload.extension == "exe" or payload.extension == "war":
            outputFolder = settings.PAYLOAD_COMPILED_PATH
        # Check for ELF binary
        elif hasattr(payload, 'type') and payload.type == "ELF":
            outputFolder = settings.PAYLOAD_COMPILED_PATH
        else:
            outputFolder = settings.PAYLOAD_SOURCE_PATH

        # only show get input if we're doing the interactive menu
        if interactive:
            if showTitle:
                if settings.TERMINAL_CLEAR != "false": messages.title()

            # Get the base install name for the payloads (i.e. OutputBaseChoice.py/OutputBaseChoice.exe)
            OutputBaseChoice = raw_input("\n [>] Please enter the base name for output files (default is 'payload'): ")

            # ensure we get a base name and not a full path
            while OutputBaseChoice != "" and "/" in OutputBaseChoice:
                print helpers.color(" [!] Please provide a base name, not a path, for the output base", warning=True)
                OutputBaseChoice = raw_input("\n [>] Please enter the base name for output files (default is 'payload'): ")

        # for invalid output base choices that are passed by arguments
        else:
            if "/" in OutputBaseChoice:
                print helpers.color(" [!] Please provide a base name, not a path, for the output base", warning=True)
                print helpers.color(" [!] Defaulting to 'payload' for output base...", warning=True)
                OutputBaseChoice = "payload"

        if OutputBaseChoice == "": OutputBaseChoice = "payload"

        # if we are overwriting, this is the base choice used
        FinalBaseChoice = OutputBaseChoice

        # if we're not overwriting output files, walk the existing and increment
        if not overwrite:
            # walk the output path and grab all the file bases, disregarding extensions
            fileBases = []
            for (dirpath, dirnames, filenames) in os.walk(outputFolder):
                fileBases.extend(list(set([x.split(".")[0] for x in filenames if x.split(".")[0] != ''])))
                break

            # as long as the file exists, increment a counter to add to the filename
            # i.e. "payload3.py", to make sure we don't overwrite anything
            FinalBaseChoice = OutputBaseChoice
            x = 1
            while FinalBaseChoice in fileBases:
                FinalBaseChoice = OutputBaseChoice + str(x)
                x += 1

        # set the output name to /outout/source/BASENAME.EXT unless it is an ELF then no extension
        if hasattr(payload, 'type') and payload.type == "ELF":
            OutputFileName = outputFolder + FinalBaseChoice + payload.extension
        else:
            OutputFileName = outputFolder + FinalBaseChoice + "." + payload.extension

        OutputFile = open(OutputFileName, 'w')
        OutputFile.write(code)
        OutputFile.close()

        # start building the information string for the generated payload
        # extract the payload class name from the instantiated object, then chop off the load folder prefix
        payloadname = "/".join(str(str(payload.__class__)[str(payload.__class__).find("payloads"):]).split(".")[0].split("/")[1:])
        message = "\n Language:\t\t"+helpers.color(payload.language)+"\n Payload:\t\t"+payloadname
        handler = ""

        if hasattr(payload, 'shellcode'):
            # check if msfvenom was used or something custom, print appropriately
            if payload.shellcode.customshellcode != "":
                message += "\n Shellcode:\t\tcustom"
            else:
                message += "\n Shellcode:\t\t" + payload.shellcode.msfvenompayload

                # if the shellcode wasn't custom, build out a handler script
                handler = "use exploit/multi/handler\n"
                handler += "set PAYLOAD " + payload.shellcode.msfvenompayload + "\n"

                # extract LHOST if it's there
                p = re.compile('LHOST=(.*?) ')
                parts = p.findall(payload.shellcode.msfvenomCommand)
                if len(parts) > 0:
                    handler += "set LHOST " + parts[0] + "\n"
                else:
                    # try to extract this local IP
                    handler += "set LHOST " + helpers.LHOST() + "\n"

                # extract LPORT if it's there
                p = re.compile('LPORT=(.*?) ')
                parts = p.findall(payload.shellcode.msfvenomCommand)
                if len(parts) > 0:
                    handler += "set LPORT " + parts[0] + "\n"

                # Removed autoscript smart migrate due to users on forum saying that migrate itself caused detection
                # in an otherwise undetectable (at the time) payload
                handler += "set ExitOnSession false\n"
                handler += "exploit -j\n"

            # print out any msfvenom options we used in shellcode generation if specified
            if len(payload.shellcode.options) > 0:
                message += "\n Options:\t\t"
                parts = ""
                for option in payload.shellcode.options:
                    parts += ' ' + option + ' '
                message += parts.strip()

            # reset the internal shellcode state the options don't persist
            payload.shellcode.Reset()

        # if required options were specified, output them
        if hasattr(payload, 'required_options'):
            t = ""
            # sort the dictionary by key before we output, so it looks nice
            for key in sorted(payload.required_options.iterkeys()):
                t += " " + key + "=" + payload.required_options[key][0] + " "
            message += "\n" + helpers.formatLong("Required Options:", t.strip(), frontTab=False, spacing=24)

            # check if any options specify that we should build a handler out
            keys = payload.required_options.keys()

            # assuming if LHOST is set, we need a handler script
            if "LHOST" in keys or "RHOST" in keys:

                handler = "use exploit/multi/handler\n"
                # do our best to determine the payload type

                architecture = ""
                if hasattr(payload, "architecture") and payload.architecture == "64":
                    architecture = "x64/"


                # handle options from the backdoor factory
                if "payload" in keys:
                    p = payload.required_options["payload"][0]

                    if "rev_tcp" in p:
                        handler += "set PAYLOAD windows/%smeterpreter/reverse_tcp\n" % architecture
                    elif "bind_tcp" in p:
                        handler += "set PAYLOAD windows/%smeterpreter/bind_tcp\n" % architecture
                    elif "https" in p:
                        handler += "set PAYLOAD windows/%smeterpreter/reverse_https\n" % architecture
                    elif "shell" in  p:
                        handler += "set PAYLOAD windows/%sshell_reverse_tcp\n" % architecture
                    else: pass

                # if not BDF, try to extract the handler type from the payload name
                else:
                    # extract the payload class name from the instantiated object, then chop off the load folder prefix
                    payloadname = "/".join(str(str(payload.__class__)[str(payload.__class__).find("payloads"):]).split(".")[0].split("/")[1:])

                    # pure rev_tcp stager
                    if "rev_tcp" in payloadname.lower():
                        handler += "set PAYLOAD windows/%smeterpreter/reverse_tcp\n" % architecture
                    # pure bind_tcp stager
                    elif "bind_tcp" in payloadname.lower():
                        handler += "set PAYLOAD windows/%smeterpreter/bind_tcp\n" % architecture
                    # pure rev_https stager
                    elif "https" in payloadname.lower():
                        handler += "set PAYLOAD windows/%smeterpreter/reverse_https\n" % architecture
                    # pure rev_http stager
                    elif "http" in payloadname.lower():
                        handler += "set PAYLOAD windows/%smeterpreter/reverse_http\n" % architecture
                    else: pass

                # grab the LHOST value
                if "LHOST" in keys:
                    handler += "set LHOST " + payload.required_options["LHOST"][0] + "\n"
                if "RHOST" in keys:
                    handler += "set RHOST " + payload.required_options["RHOST"][0] + "\n"

                # grab the LPORT value if it was set
                if "LPORT" in keys:
                    handler += "set LPORT " + payload.required_options["LPORT"][0] + "\n"

                # grab the LURI value if it was set. ignore the / as that is the default
                if "LURI" in keys and payload.required_options["LURI"][0] != "/":
                    handler += "set LURI " + payload.required_options["LURI"][0] + "\n"

                handler += "set ExitOnSession false\n"
                handler += "exploit -j\n"

        message += "\n Payload File:\t\t"+OutputFileName + "\n"

        # if we're generating the handler script, write it out
        try:
            if settings.GENERATE_HANDLER_SCRIPT.lower() == "true":
                if handler != "":
                    handlerFileName = settings.HANDLER_PATH + FinalBaseChoice + "_handler.rc"
                    handlerFile = open(handlerFileName, 'w')
                    handlerFile.write(handler)
                    handlerFile.close()
                    message += " Handler File:\t\t"+handlerFileName + "\n"
        except:
            # is that option fails, it probably means that the /etc/veil/settings.py file hasn't been updated
            print helpers.color("\n [!] Internal error #1. Please run %s manually\n" % (os.path.abspath("./config/update.py")), warning=True)

        # print out notes if set
        if hasattr(payload, 'notes'):
            #message += " Notes:\t\t\t" + payload.notes
            message += helpers.formatLong("Notes:", payload.notes, frontTab=False, spacing=24)

        # check if compile_to_exe is in the required options, if so,
        # call supportfiles.supportingFiles() to compile appropriately
        if hasattr(self.payload, 'required_options') and self.payload.language.lower() != "powershell":
            if "COMPILE_TO_EXE" in self.payload.required_options:
                value = self.payload.required_options['COMPILE_TO_EXE'][0].lower()[0]

                if value == "y" or value==True:

                    # check if the --pwnstaller flag was passed
                    if args and args.pwnstaller:
                        supportfiles.supportingFiles(self.payload, OutputFileName, {'method':'pwnstaller'})
                    else:
                        # if interactive, allow the user to choose the method
                        if interactive:
                            supportfiles.supportingFiles(self.payload, OutputFileName, {})
                        # otherwise specify the default, pyinstaller
                        else:
                            supportfiles.supportingFiles(self.payload, OutputFileName, {'method':'pyinstaller'})

                    # if we're compiling, set the returned file name to the output .exe
                    # so we can return this for external calls to the framework
                    OutputFileName = settings.PAYLOAD_COMPILED_PATH + FinalBaseChoice + ".exe"

        # This block of code is going to be used to SHA1 hash our compiled payloads to potentially submit the
        # hash with VTNotify to detect if it's been flagged
        try:
            CompiledHashFile = settings.HASH_LIST
            HashFile = open(CompiledHashFile, 'a')
            OutputFile = open(OutputFileName, 'rb')
            Sha1Hasher = hashlib.sha1()
            Sha1Hasher.update(OutputFile.read())
            SHA1Hash = Sha1Hasher.hexdigest()
            OutputFile.close()
            HashFile.write(SHA1Hash + ":" + FinalBaseChoice + "\n")
            HashFile.close()

            # print the full message containing generation notes
            print message

            # print the end message
            messages.endmsg()
        except:
            # if that option fails, it probably means that the /etc/veil/settings.py file hasn't been updated
            print helpers.color("\n [!] Internal error #2. Unable to generate output. Please run %s manually\n" % (os.path.abspath("./config/update.py")), warning=True)

        if interactive:
            raw_input(" [>] Press any key to return to the main menu.")
            print ""
            self.MainMenu(showMessage=True)

        return OutputFileName

Example 45

Project: dota2_nohats
Source File: vpk.py
View license
def create_vpk(prefix, pack_dir):
    filelist = []
    for (p, ds, fs) in walk(pack_dir):
        rel_p = canonical_file(relpath(p, pack_dir))
        for f in fs:
            filelist.append((rel_p, canonical_file(f)))

    file_types = OrderedDict()
    crc_index = {}
    i = 0
    max_o = 2**20 * 100
    archive_file = "{}_{:03}.vpk".format(prefix, i)
    s = open(archive_file, "wb")
    try:
        for p, f in filelist:
            o = s.tell()
            if o > max_o:
                s.close()
                i += 1
                archive_file = "{}_{:03}.vpk".format(prefix, i)
                s = open(archive_file, "wb")
                o = 0

            with open(join(pack_dir, p, f), "rb") as t:
                d = t.read()
            size = len(d)
            crc = crc32(d)
            name, extension = f.rsplit(".", 1)
            if (crc, size) in crc_index:
                # TODO: actually check equality in case of CRC+size collisions!
                our_i, our_o = crc_index[(crc, size)]
            else:
                crc_index[(crc, size)] = (i, o)
                s.write(d)
                our_i = i
                our_o = o
            file_types.setdefault(extension, OrderedDict()).setdefault(p, []).append((name, our_i, our_o, size, crc))
    finally:
        s.close()

    types = []
    for t, ds in file_types.items():
        dirs = []
        for d, fs in ds.items():
            files = []
            for (name, i, o, size, crc) in fs:
                files.append({
                    "filename": name,
                    "crc": crc,
                    "preloadsize": 0,
                    "archive_index": i,
                    "archive_offset": o,
                    "archive_size": size,
                    "preload_data": b"",
                    })
            dirs.append({
                "path": d,
                "file": files,
                })
        types.append({
            "type": t,
            "directory": dirs,
            })
    v = VPK()
    v.data = {
        "version": 1,
        "index_size": 0,
        "index": types,
        }
    with open("{}_dir.vpk".format(prefix), "wb") as s:
        v.pack(s)

Example 46

Project: viper
Source File: commands.py
View license
    def cmd_store(self, *args):
        parser = argparse.ArgumentParser(prog='store', description="Store the opened file to the local repository")
        parser.add_argument('-d', '--delete', action='store_true', help="Delete the original file")
        parser.add_argument('-f', '--folder', type=str, nargs='+', help="Specify a folder to import")
        parser.add_argument('-s', '--file-size', type=int, help="Specify a maximum file size")
        parser.add_argument('-y', '--file-type', type=str, help="Specify a file type pattern")
        parser.add_argument('-n', '--file-name', type=str, help="Specify a file name pattern")
        parser.add_argument('-t', '--tags', type=str, nargs='+', help="Specify a list of comma-separated tags")

        try:
            args = parser.parse_args(args)
        except:
            return

        if args.folder is not None:
            # Allows to have spaces in the path.
            args.folder = " ".join(args.folder)

        if args.tags is not None:
            # Remove the spaces in the list of tags
            args.tags = "".join(args.tags)

        def add_file(obj, tags=None):
            if get_sample_path(obj.sha256):
                self.log('warning', "Skip, file \"{0}\" appears to be already stored".format(obj.name))
                return False

            if __sessions__.is_attached_misp(quiet=True):
                if tags is not None:
                    tags += ',misp:{}'.format(__sessions__.current.misp_event.event_id)
                else:
                    tags = 'misp:{}'.format(__sessions__.current.misp_event.event_id)

            # Try to store file object into database.
            status = self.db.add(obj=obj, tags=tags)
            if status:
                # If succeeds, store also in the local repository.
                # If something fails in the database (for example unicode strings)
                # we don't want to have the binary lying in the repository with no
                # associated database record.
                new_path = store_sample(obj)
                self.log("success", "Stored file \"{0}\" to {1}".format(obj.name, new_path))

            else:
                return False

            # Delete the file if requested to do so.
            if args.delete:
                try:
                    os.unlink(obj.path)
                except Exception as e:
                    self.log('warning', "Failed deleting file: {0}".format(e))

            return True

        # If the user specified the --folder flag, we walk recursively and try
        # to add all contained files to the local repository.
        # This is note going to open a new session.
        # TODO: perhaps disable or make recursion optional?
        if args.folder is not None:
            # Check if the specified folder is valid.
            if os.path.isdir(args.folder):
                # Walk through the folder and subfolders.
                for dir_name, dir_names, file_names in walk(args.folder):
                    # Add each collected file.
                    for file_name in file_names:
                        file_path = os.path.join(dir_name, file_name)

                        if not os.path.exists(file_path):
                            continue
                        # Check if file is not zero.
                        if not os.path.getsize(file_path) > 0:
                            continue

                        # Check if the file name matches the provided pattern.
                        if args.file_name:
                            if not fnmatch.fnmatch(file_name, args.file_name):
                                # self.log('warning', "Skip, file \"{0}\" doesn't match the file name pattern".format(file_path))
                                continue

                        # Check if the file type matches the provided pattern.
                        if args.file_type:
                            if args.file_type not in File(file_path).type:
                                # self.log('warning', "Skip, file \"{0}\" doesn't match the file type".format(file_path))
                                continue

                        # Check if file exceeds maximum size limit.
                        if args.file_size:
                            # Obtain file size.
                            if os.path.getsize(file_path) > args.file_size:
                                self.log('warning', "Skip, file \"{0}\" is too big".format(file_path))
                                continue

                        file_obj = File(file_path)

                        # Add file.
                        add_file(file_obj, args.tags)
                        if add_file and cfg.autorun.enabled:
                            autorun_module(file_obj.sha256)
                            # Close the open session to keep the session table clean
                            __sessions__.close()

            else:
                self.log('error', "You specified an invalid folder: {0}".format(args.folder))
        # Otherwise we try to store the currently opened file, if there is any.
        else:
            if __sessions__.is_set():
                if __sessions__.current.file.size == 0:
                    self.log('warning', "Skip, file \"{0}\" appears to be empty".format(__sessions__.current.file.name))
                    return False

                # Add file.
                if add_file(__sessions__.current.file, args.tags):
                    # Open session to the new file.
                    self.cmd_open(*[__sessions__.current.file.sha256])
                    if cfg.autorun.enabled:
                        autorun_module(__sessions__.current.file.sha256)
            else:
                self.log('error', "No open session")

Example 47

Project: vispy
Source File: _runners.py
View license
def _examples(fnames_str):
    """Run examples and make sure they work.

    Parameters
    ----------
    fnames_str : str
        Can be a space-separated list of paths to test, or an empty string to
        auto-detect and run all examples.
    """
    import_dir, dev = _get_import_dir()
    reason = None
    if not dev:
        reason = 'Cannot test examples unless in vispy git directory'
    else:
        with use_log_level('warning', print_msg=False):
            good, backend = has_application(capable=('multi_window',))
        if not good:
            reason = 'Must have suitable app backend'
    if reason is not None:
        msg = 'Skipping example test: %s' % reason
        print(msg)
        raise SkipTest(msg)

    # if we're given individual file paths as a string in fnames_str,
    # then just use them as the fnames
    # otherwise, use the full example paths that have been
    # passed to us
    if fnames_str:
        fnames = fnames_str.split(' ')

    else:
        fnames = [op.join(d[0], fname)
                  for d in os.walk(op.join(import_dir, '..', 'examples'))
                  for fname in d[2] if fname.endswith('.py')]

    fnames = sorted(fnames, key=lambda x: x.lower())
    print(_line_sep + '\nRunning %s examples using %s backend'
          % (len(fnames), backend))
    op.join('tutorial', 'app', 'shared_context.py'),  # non-standard

    fails = []
    n_ran = n_skipped = 0
    t0 = time()
    for fname in fnames:
        n_ran += 1
        root_name = op.split(fname)
        root_name = op.join(op.split(op.split(root_name[0])[0])[1],
                            op.split(root_name[0])[1], root_name[1])
        good = True
        with open(fname, 'r') as fid:
            for _ in range(10):  # just check the first 10 lines
                line = fid.readline()
                if line == '':
                    break
                elif line.startswith('# vispy: ') and 'testskip' in line:
                    good = False
                    break
        if not good:
            n_ran -= 1
            n_skipped += 1
            continue
        sys.stdout.flush()
        cwd = op.dirname(fname)
        cmd = [sys.executable, '-c', _script.format(op.split(fname)[1][:-3])]
        sys.stdout.flush()
        stdout, stderr, retcode = run_subprocess(cmd, return_code=True,
                                                 cwd=cwd, env=os.environ)
        if retcode or len(stderr.strip()) > 0:
            # Skipping due to missing dependency is okay
            if "ImportError: " in stderr:
                print('S', end='')
            else:
                ext = '\n' + _line_sep + '\n'
                fails.append('%sExample %s failed (%s):%s%s%s'
                             % (ext, root_name, retcode, ext, stderr, ext))
                print(fails[-1])
        else:
            print('.', end='')
        sys.stdout.flush()
    print('')
    t = (': %s failed, %s succeeded, %s skipped in %s seconds'
         % (len(fails), n_ran - len(fails), n_skipped, round(time()-t0)))
    if len(fails) > 0:
        raise RuntimeError('Failed%s' % t)
    print('Success%s' % t)

Example 48

Project: pretty-json
Source File: mergejs.py
View license
def run (sourceDirectory, outputFilename = None, configFile = None):
    cfg = None
    if configFile:
        cfg = Config(configFile)

    allFiles = []

    ## Find all the Javascript source files
    for root, dirs, files in os.walk(sourceDirectory):
        for filename in files:
            if filename.endswith(SUFFIX_JAVASCRIPT) and not filename.startswith("."):
                filepath = os.path.join(root, filename)[len(sourceDirectory)+1:]
                filepath = filepath.replace("\\", "/")
                if cfg and cfg.include:
                    if filepath in cfg.include or filepath in cfg.forceFirst:
                        allFiles.append(filepath)
                elif (not cfg) or (filepath not in cfg.exclude):
                    allFiles.append(filepath)

    ## Header inserted at the start of each file in the output
    HEADER = "/* " + "=" * 70 + "\n    %s\n" + "   " + "=" * 70 + " */\n\n"

    files = {}

    order = [] # List of filepaths to output, in a dependency satisfying order 

    ## Import file source code
    ## TODO: Do import when we walk the directories above?
    for filepath in allFiles:
        print "Importing: %s" % filepath
        fullpath = os.path.join(sourceDirectory, filepath).strip()
        content = open(fullpath, "U").read() # TODO: Ensure end of line @ EOF?
        files[filepath] = SourceFile(filepath, content) # TODO: Chop path?

    print

    from toposort import toposort

    complete = False
    resolution_pass = 1

    while not complete:
        order = [] # List of filepaths to output, in a dependency satisfying order 
        nodes = []
        routes = []
        ## Resolve the dependencies
        print "Resolution pass %s... " % resolution_pass
        resolution_pass += 1 

        for filepath, info in files.items():
            nodes.append(filepath)
            for neededFilePath in info.requires:
                routes.append((neededFilePath, filepath))

        for dependencyLevel in toposort(nodes, routes):
            for filepath in dependencyLevel:
                order.append(filepath)
                if not files.has_key(filepath):
                    print "Importing: %s" % filepath
                    fullpath = os.path.join(sourceDirectory, filepath).strip()
                    content = open(fullpath, "U").read() # TODO: Ensure end of line @ EOF?
                    files[filepath] = SourceFile(filepath, content) # TODO: Chop path?
        


        # Double check all dependencies have been met
        complete = True
        try:
            for fp in order:
                if max([order.index(rfp) for rfp in files[fp].requires] +
                       [order.index(fp)]) != order.index(fp):
                    complete = False
        except:
            complete = False
        
        print    


    ## Move forced first and last files to the required position
    if cfg:
        print "Re-ordering files..."
        order = cfg.forceFirst + [item
                     for item in order
                     if ((item not in cfg.forceFirst) and
                         (item not in cfg.forceLast))] + cfg.forceLast
    
    print
    ## Output the files in the determined order
    result = []

    for fp in order:
        f = files[fp]
        print "Exporting: ", f.filepath
        result.append(HEADER % f.filepath)
        source = f.source
        result.append(source)
        if not source.endswith("\n"):
            result.append("\n")

    print "\nTotal files merged: %d " % len(files)

    if outputFilename:
        print "\nGenerating: %s" % (outputFilename)
        open(outputFilename, "w").write("".join(result))
    return "".join(result)

Example 49

Project: ocspbuilder
Source File: api_docs.py
View license
def run():
    """
    Looks through the docs/ dir and parses each markdown document, looking for
    sections to update from Python docstrings. Looks for section headers in
    the format:

     - ### `ClassName()` class
     - ##### `.method_name()` method
     - ##### `.attribute_name` attribute
     - ### `function_name()` function

    The markdown content following these section headers up until the next
    section header will be replaced by new markdown generated from the Python
    docstrings of the associated source files.

    By default maps docs/{name}.md to {modulename}/{name}.py. Allows for
    custom mapping via the MD_SOURCE_MAP variable.
    """

    print('Updating API docs...')

    md_files = []
    for root, _, filenames in os.walk(docs_dir):
        for filename in filenames:
            if not filename.endswith('.md'):
                continue
            md_files.append(os.path.join(root, filename))

    parser = CommonMark.Parser()

    for md_file in md_files:
        md_file_relative = md_file[len(project_dir) + 1:]
        if md_file_relative in MD_SOURCE_MAP:
            py_files = MD_SOURCE_MAP[md_file_relative]
            py_paths = [os.path.join(project_dir, py_file) for py_file in py_files]
        else:
            py_files = [os.path.basename(md_file).replace('.md', '.py')]
            py_paths = [os.path.join(project_dir, module_name, py_files[0])]

            if not os.path.exists(py_paths[0]):
                continue

        with open(md_file, 'rb') as f:
            markdown = f.read().decode('utf-8')

        original_markdown = markdown
        md_lines = list(markdown.splitlines())
        md_ast = parser.parse(markdown)

        last_class = []
        last = {}
        sections = OrderedDict()
        find_sections(md_ast, sections, last, last_class, markdown.count("\n") + 1)

        md_chunks = {}

        for index, py_file in enumerate(py_files):
            py_path = py_paths[index]

            with open(os.path.join(py_path), 'rb') as f:
                code = f.read().decode('utf-8')
                module_ast = ast.parse(code, filename=py_file)
                code_lines = list(code.splitlines())

            for node in ast.iter_child_nodes(module_ast):
                walk_ast(node, code_lines, sections, md_chunks)

        added_lines = 0

        def _replace_md(key, sections, md_chunk, md_lines, added_lines):
            start, end = sections[key]
            start -= 1
            start += added_lines
            end += added_lines
            new_lines = md_chunk.split('\n')
            added_lines += len(new_lines) - (end - start)

            # Ensure a newline above each class header
            if start > 0 and md_lines[start][0:4] == '### ' and md_lines[start - 1][0:1] == '>':
                added_lines += 1
                new_lines.insert(0, '')

            md_lines[start:end] = new_lines
            return added_lines

        for key in sections:
            if key not in md_chunks:
                raise ValueError('No documentation found for %s' % key[1])
            added_lines = _replace_md(key, sections, md_chunks[key], md_lines, added_lines)

        markdown = '\n'.join(md_lines).strip() + '\n'

        if original_markdown != markdown:
            with open(md_file, 'wb') as f:
                f.write(markdown.encode('utf-8'))

Example 50

Project: oscrypto
Source File: api_docs.py
View license
def run():
    """
    Looks through the docs/ dir and parses each markdown document, looking for
    sections to update from Python docstrings. Looks for section headers in
    the format:

     - ### `ClassName()` class
     - ##### `.method_name()` method
     - ##### `.attribute_name` attribute
     - ### `function_name()` function

    The markdown content following these section headers up until the next
    section header will be replaced by new markdown generated from the Python
    docstrings of the associated source files.

    By default maps docs/{name}.md to {modulename}/{name}.py. Allows for
    custom mapping via the MD_SOURCE_MAP variable.
    """

    print('Updating API docs...')

    md_files = []
    for root, _, filenames in os.walk(docs_dir):
        for filename in filenames:
            if not filename.endswith('.md'):
                continue
            md_files.append(os.path.join(root, filename))

    parser = CommonMark.Parser()

    for md_file in md_files:
        md_file_relative = md_file[len(project_dir) + 1:]
        if md_file_relative in MD_SOURCE_MAP:
            py_files = MD_SOURCE_MAP[md_file_relative]
            py_paths = [os.path.join(project_dir, py_file) for py_file in py_files]
        else:
            py_files = [os.path.basename(md_file).replace('.md', '.py')]
            py_paths = [os.path.join(project_dir, module_name, py_files[0])]

            if not os.path.exists(py_paths[0]):
                continue

        with open(md_file, 'rb') as f:
            markdown = f.read().decode('utf-8')

        original_markdown = markdown
        md_lines = list(markdown.splitlines())
        md_ast = parser.parse(markdown)

        last_class = []
        last = {}
        sections = OrderedDict()
        find_sections(md_ast, sections, last, last_class, markdown.count("\n") + 1)

        md_chunks = {}

        for index, py_file in enumerate(py_files):
            py_path = py_paths[index]

            with open(os.path.join(py_path), 'rb') as f:
                code = f.read().decode('utf-8')
                module_ast = ast.parse(code, filename=py_file)
                code_lines = list(code.splitlines())

            for node in ast.iter_child_nodes(module_ast):
                walk_ast(node, code_lines, sections, md_chunks)

        added_lines = 0

        def _replace_md(key, sections, md_chunk, md_lines, added_lines):
            start, end = sections[key]
            start -= 1
            start += added_lines
            end += added_lines
            new_lines = md_chunk.split('\n')
            added_lines += len(new_lines) - (end - start)

            # Ensure a newline above each class header
            if start > 0 and md_lines[start][0:4] == '### ' and md_lines[start - 1][0:1] == '>':
                added_lines += 1
                new_lines.insert(0, '')

            md_lines[start:end] = new_lines
            return added_lines

        for key in sections:
            if key not in md_chunks:
                raise ValueError('No documentation found for %s' % key[1])
            added_lines = _replace_md(key, sections, md_chunks[key], md_lines, added_lines)

        markdown = '\n'.join(md_lines).strip() + '\n'

        if original_markdown != markdown:
            with open(md_file, 'wb') as f:
                f.write(markdown.encode('utf-8'))