re.match

Here are the examples of the python api re.match taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: pgmapcss
Source File: main.py
View license
def main():
    print('pgmapcss version %s' % pgmapcss.version.VERSION)
    args = parser.parse_args()

    style_id = args.style_id

    m = re.match('(.*)\.mapcss$', style_id)
    if m:
        style_id = m.group(1)

    file_name = style_id + '.mapcss'

    parameters = { }
    if args.parameters is not None:
        parameters = {
            p[0:p.find('=')]: p[p.find('=')+1:]
            for p in args.parameters
        }

    if args.lang:
        lang = args.lang
    elif 'lang' in parameters:
        pass
    else:
        lang = os.environ.get('LANG')
        if lang:
            m = re.match('(.*)_', lang)
            if m:
                lang = m.group(1)
        else:
            # default: english
            lang = 'en'

    stat = pgmapcss.compiler.stat._stat({
        'id': style_id,
        'config': {},
        'base_style': args.base_style,
        'icons_dir': style_id + '.icons',
        'global_data': None,
        'mode': args.mode,
        'args': args,
        'lang': lang,
        'parameters': parameters,
    })

    if args.config:
        for v in args.config:
            v = v.split("=")
            if len(v) > 1:
                stat['config'][v[0]] = v[1]
            else:
                stat['config'][v[0]] = True

    conn = pgmapcss.db.connect(args, stat)

    stat['database'] = conn.database

    if not 'unit.srs' in stat['config']:
        stat['config']['unit.srs'] = 900913
    if not 'srs' in stat['config']:
        if stat['mode'] == 'database-function':
            stat['config']['srs'] = 900913
        else:
            stat['config']['srs'] = 4326

    if stat['config'].get('offline', False) in (False, 'false', 'no') and args.database_update in ('init', 're-init'):
        print('* Re-initializing database')
        pgmapcss.db.db_init(conn, stat)

    if stat['config'].get('offline', False) not in (False, 'false', 'no'):
        print('* Using offline mode. Attention! Some functionality might be missing.')

    else:
        db_version = pgmapcss.db.db_version()
        if db_version == None:
            print('* DB functions not installed; installing')
            pgmapcss.db.db_init(conn, stat)
        else:
            db_check = pgmapcss.db.db_version_check()
            if db_check == 1 and args.database_update == 'auto':
                print('* Current DB version: {version} -> updating DB functions'.format(**db_version))
                pgmapcss.db.db_update(conn)

            elif db_check == 2:
                print('* Current DB version: {version}'.format(**db_version))
                print('pgmapcss version too new. Database needs to be re-initialized. Please re-run pgmapcss with parameter "-r init". All Mapnik styles need to be re-compiled afterwards.')
                sys.exit(1)

            elif args.database_update == 'update':
                pgmapcss.db.db_update(conn)

            else:
                print('* Current DB version: {version}'.format(**db_version))

    if args.eval_tests is not False:
        if len(args.eval_tests):
            pgmapcss.eval.functions(stat).test_all(args.eval_tests)
        else:
            pgmapcss.eval.functions(stat).test_all()

        print('* All tests completed successfully.')

    try:
        os.mkdir(stat['icons_dir'])
    except OSError:
        pass

    eval_functions = pgmapcss.eval.functions(stat).list()

    content = open(file_name).read()

# check if file is XML -> extract MapCSS code
    tree = None
    if re.match('<\?xml', content):
        import xml.dom.minidom as dom
        tree = dom.parse(file_name)
        mapcss = tree.getElementsByTagName("style")
        if mapcss.length != 1:
            print("Require exactly one <style type='text/mapcss'> node")
            sys.exit(1)

        mapcss = mapcss.item(0)
        content = mapcss.firstChild.nodeValue

    try:
        pgmapcss.parser.parse_file(stat, filename=file_name, content=content, base_style=args.base_style, defaults=args.defaults)
    except pgmapcss.parser.ParseError as e:
        print(e)
        sys.exit(1)

    debug = open(style_id + '.output', 'w')

    pp = pprint.PrettyPrinter()

    debug.write("***** Structure of parsed MapCSS style *****\n")
    debug.write(pp.pformat(stat) + '\n')

    pgmapcss.renderer.init(stat)
    pgmapcss.icons.init(stat)
    pgmapcss.symbols.init(stat)

    try:
        style = pgmapcss.compiler.compile_style(stat)
    except pgmapcss.compiler.CompileError as e:
        print(e)
        sys.exit(1)

    #pp.pprint(style)
    for i in style:
        debug.write("\n***** " + i + " *****\n" + style[i])

    if stat['mode'] == 'database-function':
        pgmapcss.db.install(style_id, style, conn)
        pgmapcss.renderer.process_renderer(style_id, args, stat, conn)
    elif stat['mode'] == 'standalone':
        open(style_id + '.py', 'w').write(style['function_match'])
        os.chmod(style_id + '.py', 0o755)
        print('Created executable {}.py'.format(style_id))

    pgmapcss.icons.process_icons(style_id, args, stat, conn)
    pgmapcss.symbols.process_symbols(style_id, args, stat, conn)

    debug.close()

    if 'unresolvable_properties' in stat:
        print('WARNING: Not all values for the following properties could be guessed (e.g. as they are the result of an eval-expression, and therefore some features in the resulting image(s) may be missing: ' + ', '.join(stat['unresolvable_properties']))

    # copy result xml to original dom
    if tree:
        result_tree = dom.parse(style_id + '.mapnik')
        current = result_tree.getElementsByTagName("Map").item(0).firstChild

        while current:
            if re.search('[^\s]', current.toxml()):
                if not re.match('<!\-\-', current.toxml()):
                    copy = dom.parseString(current.toxml())
                    mapcss.parentNode.insertBefore(copy.firstChild, mapcss)
            current = current.nextSibling

        mapcss.parentNode.removeChild(mapcss)
        open(style_id + '.mapnik', 'w').write(tree.toxml())

    print('Debug output wrote to ' + style_id + '.output')

Example 2

Project: tp-libvirt
Source File: controller_functional.py
View license
def run(test, params, env):
    """
    Test for basic controller device function.

    1) Define the VM with specified controller device and check result meets
       expectation.
    2) Start the guest and check if start result meets expectation
    3) Test the function of started controller device
    4) Shutdown the VM and clean up environment
    """

    def setup_os_xml():
        """
        Prepare os part of VM XML according to params.
        """
        osxml = vm_xml.os
        orig_machine = osxml.machine
        if '-' in orig_machine:
            suffix = orig_machine.split('-')[-1]
            new_machine = '-'.join(('pc', os_machine, suffix))
        else:
            if os_machine == 'i440fx':
                new_machine = 'pc'
            else:
                new_machine = os_machine
        osxml.machine = new_machine
        vm_xml.os = osxml

    def setup_controller_xml():
        """
        Prepare controller devices of VM XML according to params.
        """
        if cntlr_type is None:
            return

        ctrl = Controller(type_name=cntlr_type)

        if model is not None:
            ctrl.model = model
        if pcihole is not None:
            ctrl.pcihole64 = pcihole
        if vectors is not None:
            ctrl.vectors = vectors
        if index is not None:
            ctrl.index = index
        if addr_str is not None:
            match = re.match(r"(?P<bus>[0-9]*):(?P<slot>[0-9]*).(?P<function>[0-9])", addr_str)
            if match:
                addr_dict = match.groupdict()
                addr_dict['bus'] = hex(int(addr_dict['bus']))
                addr_dict['slot'] = hex(int(addr_dict['slot']))
                addr_dict['function'] = hex(int(addr_dict['function']))
                addr_dict['domain'] = '0x0000'
                ctrl.address = ctrl.new_controller_address(attrs=addr_dict)

        logging.debug("Controller XML is:%s", ctrl)
        vm_xml.add_device(ctrl)

        if usb_cntlr_model is not None:
            ctrl = Controller(type_name='usb')
            ctrl.model = usb_cntlr_model
            if usb_cntlr_addr is not None:
                match = re.match(r"(?P<bus>[0-9]*):(?P<slot>[0-9]*).(?P<function>[0-9])", usb_cntlr_addr)
                if match:
                    addr_dict = match.groupdict()
                    addr_dict['bus'] = hex(int(addr_dict['bus']))
                    addr_dict['slot'] = hex(int(addr_dict['slot']))
                    addr_dict['function'] = hex(int(addr_dict['function']))
                    addr_dict['domain'] = '0x0000'
                    ctrl.address = ctrl.new_controller_address(attrs=addr_dict)
            vm_xml.add_device(ctrl)

    def define_and_check():
        """
        Predict the error message when defining and try to define the guest
        with testing XML.
        """
        fail_patts = []
        known_models = {
            'pci': ['pci-root', 'pcie-root', 'pci-bridge'],
            'virtio-serial': [],
            'usb': ['ehci', 'ich9-ehci1'],
        }

        if cntlr_type == 'pci' and model is None:
            fail_patts.append(r"Invalid PCI controller model")
        if model is not None and model not in known_models[cntlr_type]:
            fail_patts.append(r"Unknown model type")
        if os_machine == 'q35' and model in ['pci-root', 'pci-bridge']:
            fail_patts.append(r"Device requires a PCI Express slot")
        if os_machine == 'i440fx' and model == 'pcie-root':
            fail_patts.append(r"Device requires a standard PCI slot")
        # isdigit will return false on negative number, which just meet the
        # requirement of this test.
        if index is not None and not index.isdigit():
            fail_patts.append(r"Cannot parse controller index")

        vm_xml.undefine()
        res = vm_xml.virsh.define(vm_xml.xml)
        libvirt.check_result(res, expected_fails=fail_patts)
        return not res.exit_status

    def start_and_check():
        """
        Predict the error message when starting and try to start the guest.
        """
        fail_patts = []
        if model == 'pci-bridge' and (index is None or int(index) == 0):
            fail_patts.append(r"PCI bridge index should be > 0")
        res = virsh.start(vm_name)
        libvirt.check_result(res, expected_fails=fail_patts)
        return not res.exit_status

    def get_controller_addr(cntlr_type=None, model=None, index=None):
        """
        Get the address of testing controller from VM XML as a string with
        format "bus:slot.function".
        """
        cur_vm_xml = VMXML.new_from_dumpxml(vm_name)
        addr = None
        for elem in cur_vm_xml.xmltreefile.findall('/devices/controller'):
            if (
                    (cntlr_type is None or elem.get('type') == cntlr_type) and
                    (model is None or elem.get('model') == model) and
                    (index is None or elem.get('index') == index)):
                addr_elem = elem.find('./address')
                if addr_elem is not None:
                    addr = Address.new_from_element(addr_elem).attrs

        if addr is not None:
            bus = int(addr['bus'], 0)
            slot = int(addr['slot'], 0)
            func = int(addr['function'], 0)
            addr_str = '%02d:%02d.%1d' % (bus, slot, func)
            logging.debug("String for address element %s is %s", addr, addr_str)
            return addr_str

    def check_controller_addr():
        """
        Check test controller address against expectation.
        """
        addr_str = get_controller_addr(cntlr_type, model, index)

        if model in ['pci-root', 'pcie-root']:
            if addr_str is None:
                return
            else:
                raise error.TestFail('Expect controller do not have address, '
                                     'but got "%s"' % addr_str)

        exp_addr_patt = r'00:[0-9]{2}.[0-9]'
        if model in ['ehci']:
            exp_addr_patt = r'0[1-9]:[0-9]{2}.[0-9]'
        if addr_str is not None:
            exp_addr_patt = addr_str

        if not re.match(exp_addr_patt, addr_str):
            raise error.TestFail('Expect get controller address "%s", '
                                 'but got "%s"' % (exp_addr_patt, addr_str))

    def check_qemu_cmdline():
        """
        Check domain qemu command line against expectation.
        """
        cmdline = open('/proc/%s/cmdline' % vm.get_pid()).read()
        logging.debug('Qemu command line: %s', cmdline)
        options = cmdline.split('\x00')

        # Get pcihole options from qemu command line
        pcihole_opt = ''
        for idx, opt in enumerate(options):
            if 'pci-hole64-size' in opt:
                pcihole_opt = opt

        # Get expected pcihole options from params
        exp_pcihole_opt = ''
        if cntlr_type == 'pci' and model in ['pci-root', 'pcie-root'] and pcihole is not None:
            if 'i440fx' in os_machine:
                exp_pcihole_opt = 'i440FX-pcihost'
            elif 'q35' in os_machine:
                exp_pcihole_opt = 'q35-pcihost'
            exp_pcihole_opt += '.pci-hole64-size=%sK' % pcihole

        # Check options against expectation
        if pcihole_opt != exp_pcihole_opt:
            raise error.TestFail('Expect get qemu command serial option "%s", '
                                 'but got "%s"' % (exp_pcihole_opt, pcihole_opt))

    def check_msi():
        """
        Check MSI state against expectation.
        """
        addr_str = get_controller_addr(cntlr_type='virtio-serial')

        if addr_str is None:
            raise error.TestError("Can't find target controller in XML")

        session = vm.wait_for_login()
        status, output = session.cmd_status_output('lspci -vvv -s %s' % addr_str)
        logging.debug("lspci output is: %s", output)

        if (vectors is not None and int(vectors) == 0):
            if 'MSI' in output:
                raise error.TestFail('Expect MSI disable with zero vectors,'
                                     ' but got %s' % output)
        if (vectors is None or int(vectors) != 0):
            if 'MSI' not in output:
                raise error.TestFail('Expect MSI enable with non-zero vectors,'
                                     ' but got %s' % output)

    os_machine = params.get('os_machine', 'i440fx')
    cntlr_type = params.get('controller_type', None)
    model = params.get('controller_model', None)
    index = params.get('controller_index', None)
    vectors = params.get('controller_vectors', None)
    pcihole = params.get('controller_pcihole64', None)
    addr_str = params.get('controller_address', None)
    usb_cntlr_model = params.get('usb_controller_model', None)
    usb_cntlr_addr = params.get('usb_controller_address', None)
    vm_name = params.get("main_vm", "avocado-vt-vm1")

    vm = env.get_vm(vm_name)
    vm_xml = VMXML.new_from_inactive_dumpxml(vm_name)
    vm_xml_backup = vm_xml.copy()
    try:
        vm_xml.remove_all_device_by_type('controller')
        remove_all_addresses(vm_xml)
        remove_usb_devices(vm_xml)
        setup_controller_xml()
        setup_os_xml()
        logging.debug("Test VM XML is %s" % vm_xml)

        if not define_and_check():
            logging.debug("Can't define the VM, exiting.")
            return

        check_controller_addr()

        try:
            if not start_and_check():
                logging.debug("Can't start the VM, exiting.")
                return
        except virt_vm.VMStartError, detail:
            raise error.TestFail(detail)

        check_qemu_cmdline()

        if cntlr_type == 'virtio-serial':
            check_msi()
    finally:
        vm_xml_backup.sync()

Example 3

Project: pyomo
Source File: CPLEX.py
View license
    def process_soln_file(self,results):

        # the only suffixes that we extract from CPLEX are
        # constraint duals, constraint slacks, and variable
        # reduced-costs. scan through the solver suffix list
        # and throw an exception if the user has specified
        # any others.
        extract_duals = False
        extract_slacks = False
        extract_reduced_costs = False
        extract_rc = False
        extract_lrc = False
        extract_urc = False
        for suffix in self._suffixes:
            flag=False
            if re.match(suffix,"dual"):
                extract_duals = True
                flag=True
            if re.match(suffix,"slack"):
                extract_slacks = True
                flag=True
            if re.match(suffix,"rc"):
                extract_reduced_costs = True
                extract_rc = True
                flag=True
            if re.match(suffix,"lrc"):
                extract_reduced_costs = True
                extract_lrc = True
                flag=True
            if re.match(suffix,"urc"):
                extract_reduced_costs = True
                extract_urc = True
                flag=True
            if not flag:
                raise RuntimeError("***The CPLEX solver plugin cannot extract solution suffix="+suffix)

        # check for existence of the solution file
        # not sure why we just return - would think that we
        # would want to indicate some sort of error
        if not os.path.exists(self._soln_file):
            return

        range_duals = {}
        range_slacks = {}
        soln = Solution()
        soln.objective['__default_objective__'] = {'Value':None}

        # caching for efficiency
        soln_variables = soln.variable
        soln_constraints = soln.constraint

        INPUT = open(self._soln_file, "r")
        results.problem.number_of_objectives=1
        time_limit_exceeded = False
        mip_problem=False
        for line in INPUT:
            line = line.strip()
            line = line.lstrip('<?/')
            line = line.rstrip('/>?')
            tokens=line.split(' ')

            if tokens[0] == "variable":
                variable_name = None
                variable_value = None
                variable_reduced_cost = None
                variable_status = None
                for i in xrange(1,len(tokens)):
                    field_name =  tokens[i].split('=')[0]
                    field_value = tokens[i].split('=')[1].lstrip("\"").rstrip("\"")
                    if field_name == "name":
                        variable_name = field_value
                    elif field_name == "value":
                        variable_value = field_value
                    elif (extract_reduced_costs is True) and (field_name == "reducedCost"):
                        variable_reduced_cost = field_value
                    elif (extract_reduced_costs is True) and (field_name == "status"):
                        variable_status = field_value

                # skip the "constant-one" variable, used to capture/retain objective offsets in the CPLEX LP format.
                if variable_name != "ONE_VAR_CONSTANT":
                    variable = soln_variables[variable_name] = {"Value" : float(variable_value)}
                    if (variable_reduced_cost is not None) and (extract_reduced_costs is True):
                        try:
                            if extract_rc is True:
                                variable["Rc"] = float(variable_reduced_cost)
                            if variable_status is not None:
                                if extract_lrc is True:
                                    if variable_status == "LL":
                                        variable["Lrc"] = float(variable_reduced_cost)
                                    else:
                                        variable["Lrc"] = 0.0
                                if extract_urc is True:
                                    if variable_status == "UL":
                                        variable["Urc"] = float(variable_reduced_cost)
                                    else:
                                        variable["Urc"] = 0.0
                        except:
                            raise ValueError("Unexpected reduced-cost value="+str(variable_reduced_cost)+" encountered for variable="+variable_name)
            elif (tokens[0] == "constraint") and ((extract_duals is True) or (extract_slacks is True)):
                is_range = False
                rlabel = None
                rkey = None
                for i in xrange(1,len(tokens)):
                    field_name =  tokens[i].split('=')[0]
                    field_value = tokens[i].split('=')[1].lstrip("\"").rstrip("\"")
                    if field_name == "name":
                        if field_value.startswith('c_'):
                            constraint = soln_constraints[field_value] = {}
                        elif field_value.startswith('r_l_'):
                            is_range = True
                            rlabel = field_value[4:]
                            rkey = 0
                        elif field_value.startswith('r_u_'):
                            is_range = True
                            rlabel = field_value[4:]
                            rkey = 1
                    elif (extract_duals is True) and (field_name == "dual"): # for LPs
                        if is_range is False:
                            constraint["Dual"] = float(field_value)
                        else:
                            range_duals.setdefault(rlabel,[0,0])[rkey] = float(field_value)
                    elif (extract_slacks is True) and (field_name == "slack"): # for MIPs
                        if is_range is False:
                            constraint["Slack"] = float(field_value)
                        else:
                            range_slacks.setdefault(rlabel,[0,0])[rkey] = float(field_value)
            elif tokens[0].startswith("problemName"):
                filename = (tokens[0].split('=')[1].strip()).lstrip("\"").rstrip("\"")
                results.problem.name = os.path.basename(filename)
                if '.' in results.problem.name:
                    results.problem.name = results.problem.name.split('.')[0]
                tINPUT=open(filename,"r")
                for tline in tINPUT:
                    tline = tline.strip()
                    if tline == "":
                        continue
                    tokens = re.split('[\t ]+',tline)
                    if tokens[0][0] in ['\\', '*']:
                        continue
                    elif tokens[0] == "NAME":
                        results.problem.name = tokens[1]
                    else:
                        sense = tokens[0].lower()
                        if sense in ['max','maximize']:
                            results.problem.sense = ProblemSense.maximize
                        if sense in ['min','minimize']:
                            results.problem.sense = ProblemSense.minimize
                    break
                tINPUT.close()

            elif tokens[0].startswith("objectiveValue"):
                objective_value = (tokens[0].split('=')[1].strip()).lstrip("\"").rstrip("\"")
                soln.objective['__default_objective__']['Value'] = float(objective_value)
            elif tokens[0].startswith("solutionStatusValue"):
               pieces = tokens[0].split("=")
               solution_status = eval(pieces[1])
               # solution status = 1 => optimal
               # solution status = 3 => infeasible
               if soln.status == SolutionStatus.unknown:
                  if solution_status == 1:
                    soln.status = SolutionStatus.optimal
                  elif solution_status == 3:
                    soln.status = SolutionStatus.infeasible
                    soln.gap = None
                  else:
                      # we are flagging anything with a solution status >= 4 as an error, to possibly
                      # be over-ridden as we learn more about the status (e.g., due to time limit exceeded).
                      soln.status = SolutionStatus.error
                      soln.gap = None
            elif tokens[0].startswith("solutionStatusString"):
                solution_status = ((" ".join(tokens).split('=')[1]).strip()).lstrip("\"").rstrip("\"")
                if solution_status in ["optimal", "integer optimal solution", "integer optimal, tolerance"]:
                    soln.status = SolutionStatus.optimal
                    soln.gap = 0.0
                    results.problem.lower_bound = soln.objective['__default_objective__']['Value']
                    results.problem.upper_bound = soln.objective['__default_objective__']['Value']
                    if "integer" in solution_status:
                        mip_problem=True
                elif solution_status in ["infeasible"]:
                    soln.status = SolutionStatus.infeasible
                    soln.gap = None
                elif solution_status in ["time limit exceeded"]:
                    # we need to know if the solution is primal feasible, and if it is, set the solution status accordingly.
                    # for now, just set the flag so we can trigger the logic when we see the primalFeasible keyword.
                    time_limit_exceeded = True
            elif tokens[0].startswith("MIPNodes"):
                if mip_problem:
                    n = eval(eval((" ".join(tokens).split('=')[1]).strip()).lstrip("\"").rstrip("\""))
                    results.solver.statistics.branch_and_bound.number_of_created_subproblems=n
                    results.solver.statistics.branch_and_bound.number_of_bounded_subproblems=n
            elif tokens[0].startswith("primalFeasible") and (time_limit_exceeded is True):
                primal_feasible = int(((" ".join(tokens).split('=')[1]).strip()).lstrip("\"").rstrip("\""))
                if primal_feasible == 1:
                    soln.status = SolutionStatus.feasible
                    if (results.problem.sense == ProblemSense.minimize):
                        results.problem.upper_bound = soln.objective['__default_objective__']['Value']
                    else:
                        results.problem.lower_bound = soln.objective['__default_objective__']['Value']
                else:
                    soln.status = SolutionStatus.infeasible


        if self._best_bound is not None:
            if results.problem.sense == ProblemSense.minimize:
                results.problem.lower_bound = self._best_bound
            else:
                results.problem.upper_bound = self._best_bound
        if self._gap is not None:
            soln.gap = self._gap

        # For the range constraints, supply only the dual with the largest
        # magnitude (at least one should always be numerically zero)
        for key,(ld,ud) in iteritems(range_duals):
            if abs(ld) > abs(ud):
                soln_constraints['r_l_'+key] = {"Dual" : ld}
            else:
                soln_constraints['r_l_'+key] = {"Dual" : ud}                # Use the same key
        # slacks
        for key,(ls,us) in iteritems(range_slacks):
            if abs(ls) > abs(us):
                soln_constraints.setdefault('r_l_'+key,{})["Slack"] = ls
            else:
                soln_constraints.setdefault('r_l_'+key,{})["Slack"] = us    # Use the same key

        if not results.solver.status is SolverStatus.error:
            if results.solver.termination_condition in [TerminationCondition.unknown,
                                                        #TerminationCondition.maxIterations,
                                                        #TerminationCondition.minFunctionValue,
                                                        #TerminationCondition.minStepLength,
                                                        TerminationCondition.globallyOptimal,
                                                        TerminationCondition.locallyOptimal,
                                                        TerminationCondition.optimal,
                                                        #TerminationCondition.maxEvaluations,
                                                        TerminationCondition.other]:
                results.solution.insert(soln)
            elif (results.solver.termination_condition is \
                  TerminationCondition.maxTimeLimit) and \
                  (soln.status is not SolutionStatus.infeasible):
                results.solution.insert(soln)

        INPUT.close()

Example 4

Project: pyomo
Source File: gurobi_direct.py
View license
    def _postsolve(self):

        # the only suffixes that we extract from GUROBI are
        # constraint duals, constraint slacks, and variable
        # reduced-costs. scan through the solver suffix list
        # and throw an exception if the user has specified
        # any others.
        extract_duals = False
        extract_slacks = False
        extract_reduced_costs = False
        for suffix in self._suffixes:
            flag=False
            if re.match(suffix,"dual"):
                extract_duals = True
                flag=True
            if re.match(suffix,"slack"):
                extract_slacks = True
                flag=True
            if re.match(suffix,"rc"):
                extract_reduced_costs = True
                flag=True
            if not flag:
                raise RuntimeError(
                    "***The gurobi_direct solver plugin "
                    "cannot extract solution suffix="+suffix)

        gprob = self._gurobi_instance

        if (gprob.getAttr(GRB.Attr.IsMIP)):
            extract_reduced_costs = False
            extract_duals = False

        pvars = gprob.getVars()
        cons = gprob.getConstrs()
        qcons = []
        if _GUROBI_VERSION_MAJOR >= 5:
            qcons = gprob.getQConstrs()

        results = SolverResults()
        soln = Solution()
        problem = results.problem
        solver  = results.solver

        # cache the variable and constraint dictionaries -
        # otherwise, each invocation will include a lookup in a
        # MapContainer, which is extremely expensive.
        soln_variables = soln.variable
        soln_constraints = soln.constraint

        solver.name = "Gurobi %s.%s%s" % gurobi.version()
        solver.wallclock_time = gprob.Runtime

        if gprob.Status == 1: # problem is loaded, but no solution
            solver.termination_message = "Model is loaded, but no solution information is available."
            solver.termination_condition = TerminationCondition.other
        if gprob.Status == 2: # optimal
            solver.termination_message = "Model was solved to optimality (subject to tolerances)."
            solver.termination_condition = TerminationCondition.optimal
        elif gprob.Status == 3: # infeasible
            solver.termination_message = "Model was proven to be infeasible."
            solver.termination_condition = TerminationCondition.infeasible
        elif gprob.Status == 4: # infeasible or unbounded
            solver.termination_message = "Model was proven to be either infeasible or unbounded."
            solver.termination_condition = TerminationCondition.infeasible # picking one of the pre-specified Pyomo termination conditions - we don't have either-or.
        elif gprob.Status == 5: # unbounded
            solver.termination_message = "Model was proven to be unbounded."
            solver.termination_condition = TerminationCondition.unbounded
        elif gprob.Status == 6: # cutoff
            solver.termination_message = "Optimal objective for model was proven to be worse than the value specified in the Cutoff parameter."
            solver.termination_condition = TerminationCondition.minFunctionValue
        elif gprob.Status == 7: # iteration limit
            solver.termination_message = "Optimization terminated because the total number of simplex or barrier iterations exceeded specified limits."
            solver.termination_condition = TerminationCondition.maxIterations
        elif gprob.Status == 8: # node limit
            solver.termination_message = "Optimization terminated because the total number of branch-and-cut nodes exceeded specified limits."
            solver.termination_condition = TerminationCondition.maxEvaluations
        elif gprob.Status == 9: # time limit
            solver.termination_message = "Optimization terminated because the total time expended exceeded specified limits."
            solver.termination_condition = TerminationCondition.maxTimeLimit
        elif gprob.Status == 10: # solution limit
            solver.termination_message = "Optimization terminated because the number of solutions found reached specified limits."
            solver.termination_condition = TerminationCondition.other
        elif gprob.Status == 11: # interrupted
            solver.termination_message = "Optimization was terminated by the user."
            solver.termination_condition = TerminationCondition.other
        elif gprob.Status == 12: # numeric issues
            solver.termination_message = "Optimization was terminated due to unrecoverable numerical difficulties."
            solver.termination_condition = TerminationCondition.other
        elif gprob.Status == 13: # suboptimal
            solver.termination_message = "Optimization was unable to satisfy optimality tolerances; returned solution is sub-optimal."
            solver.termination_condition = TerminationCondition.other
        else: # unknown
            solver.termination_message = "Unknown termination condition received following optimization."
            solver.termination_condition = TerminationCondition.other

        problem.name = gprob.ModelName
        if gprob.ModelSense > 0: # positive numbers indicate minimization
            # depending on whether the problem is a MIP or an LP, and whether
            # or not the solution is feasible, one or two of the attributes
            # below may not exist, yielding an exception as a result of the query.
            try:
                problem.upper_bound = gprob.ObjVal
                problem.lower_bound = gprob.ObjBound
            except:
                pass
        else:
            try:
                problem.upper_bound = gprob.ObjBound
                problem.lower_bound = gprob.ObjVal
            except:
                pass

        problem.number_of_constraints          = len(cons)+len(qcons)+gprob.NumSOS
        problem.number_of_nonzeros             = gprob.NumNZs
        problem.number_of_variables            = gprob.NumVars
        problem.number_of_binary_variables     = gprob.NumBinVars
        problem.number_of_integer_variables    = gprob.NumIntVars
        problem.number_of_continuous_variables = gprob.NumVars \
                                                - gprob.NumIntVars \
                                                - gprob.NumBinVars
        problem.number_of_objectives = 1
        problem.number_of_solutions = gprob.SolCount

        problem.sense = ProblemSense.minimize
        if problem.sense == GRB_MAX:
            problem.sense = ProblemSense.maximize

        soln.status = self._gurobi_get_solution_status()
        soln.gap = None # until proven otherwise

        # if a solve was stopped by a limit, we still need to check to
        # see if there is a solution available - this may not always
        # be the case, both in LP and MIP contexts.
        if (soln.status == SolutionStatus.optimal) or \
           ((soln.status == SolutionStatus.stoppedByLimit) and \
            (gprob.SolCount > 0)):

            obj_val = gprob.ObjVal

            if (problem.number_of_binary_variables + \
                problem.number_of_integer_variables) > 0:
                obj_bound = gprob.ObjBound

            if problem.sense == ProblemSense.minimize:
                if (problem.number_of_binary_variables + \
                    problem.number_of_integer_variables) == 0:
                    problem.upper_bound = obj_val
                else:
                    problem.lower_bound = obj_bound
                    problem.upper_bound = obj_val
            else:

                if (problem.number_of_binary_variables + \
                    problem.number_of_integer_variables) == 0:
                    problem.lower_bound = obj_val
                else:
                    problem.upper_bound = obj_bound
                    problem.lower_bound = obj_val

            soln.objective[self._objective_label] = \
                {'Value': obj_val}
            if (problem.number_of_binary_variables + \
                problem.number_of_integer_variables) == 0:
                soln.gap = None
            else:
                soln.gap = math.fabs(obj_val - obj_bound)

            # Those variables not added by gurobi due to range constraints
            for var in itertools.islice(pvars, self._last_native_var_idx + 1):
                soln_variables[ var.VarName ] = {"Value" : var.X}

            if extract_reduced_costs:
                for var in itertools.islice(pvars, self._last_native_var_idx + 1):
                    soln_variables[ var.VarName ]["Rc"] = var.Rc

            if extract_duals or extract_slacks:
                for con in cons:
                    soln_constraints[con.ConstrName] = {}
                for con in qcons:
                    soln_constraints[con.QCName] = {}

            if extract_duals:
                for con in cons:
                    # Pi attributes in Gurobi are the constraint duals
                    soln_constraints[ con.ConstrName ]["Dual"] = con.Pi
                for con in qcons:
                    # QCPI attributes in Gurobi are the constraint duals
                    soln_constraints[ con.QCName ]["Dual"] = con.QCPi

            if extract_slacks:
                for con in cons:
                    soln_constraints[ con.ConstrName ]["Slack"] = con.Slack
                for con in qcons:
                    soln_constraints[ con.QCName ]["Slack"] = con.QCSlack
                # The above loops may include range constraints but will
                # always report a slack of zero since gurobi transforms
                # range constraints by adding a slack variable in the following way
                # L <= f(x) <= U
                # becomes
                # 0 <= U-f(x) <= U-L
                # becomes
                # U-f(x) == s
                # 0 <= s <= U-L
                # Therefore we need to check the value of the
                # associated slack variable with its upper bound to
                # compute the original constraint slacks. To conform
                # with the other problem writers we return the slack
                # value that is largest in magnitude (L-f(x) or
                # U-f(x))
                for con,var_idx in self._range_con_var_pairs:
                    var = pvars[var_idx]
                    # U-f(x)
                    Us_ = var.X
                    # f(x)-L
                    Ls_ = var.UB-var.X
                    if Us_ > Ls_:
                        soln_constraints[ con.ConstrName ]["Slack"] = Us_
                    else:
                        soln_constraints[ con.ConstrName ]["Slack"] = -Ls_

            byObject = self._symbol_map.byObject
            referenced_varnames = \
                set(byObject[varid]
                    for varid in self._referenced_variable_ids)
            names_to_delete = \
                set(soln_variables.keys()) - referenced_varnames
            for varname in names_to_delete:
                del soln_variables[varname]

        results.solution.insert(soln)

        self.results = results
        # Done with the model object; free up some memory.
        self._last_native_var_idx = -1
        self._range_con_var_pairs = []

        # finally, clean any temporary files registered with the temp file
        # manager, created populated *directly* by this plugin.
        pyutilib.services.TempfileManager.pop(remove=not self._keepfiles)

        # let the base class deal with returning results.
        return OptSolver._postsolve(self)

Example 5

Project: pyomo
Source File: GUROBI_RUN.py
View license
def gurobi_run(model_file, warmstart_file, soln_file, mipgap, options, suffixes):

    # figure out what suffixes we need to extract.
    extract_duals = False
    extract_slacks = False
    extract_reduced_costs = False
    for suffix in suffixes:
        flag=False
        if re.match(suffix,"dual"):
            extract_duals = True
            flag=True
        if re.match(suffix,"slack"):
            extract_slacks = True
            flag=True
        if re.match(suffix,"rc"):
            extract_reduced_costs = True
            flag=True
        if not flag:
            print("***The GUROBI solver plugin cannot extract solution suffix="+suffix)
            return

    # Load the lp model
    model = read(model_file)

    # if the use wants to extract duals or reduced costs and the
    # model has quadratic constraints then we need to set the 
    # QCPDual param to 1 (which apparently makes the solve more
    # expensive in the quadratic case). If we do not set this param
    # and and we attempt to access these suffixes in the solution
    # printing the module will crash (when we have a QCP)
    if GUROBI_VERSION[0] >= 5:
        if (extract_reduced_costs is True) or (extract_duals is True):
            model.setParam(GRB.Param.QCPDual,1)

    if model is None:
        print("***The GUROBI solver plugin failed to load the input LP file="+soln_file)
        return

    if warmstart_file is not None:
        model.read(warmstart_file)

    # set the mipgap if specified.
    if mipgap is not None:
        model.setParam("MIPGap", mipgap)

    # set all other solver parameters, if specified.
    # GUROBI doesn't throw an exception if an unknown
    # key is specified, so you have to stare at the
    # output to see if it was accepted.
    for key, value in options.iteritems():
        model.setParam(key, value)
        
    if 'relax_integrality' in options:
        for v in model.getVars():
            if v.vType != GRB.CONTINUOUS:
                v.vType = GRB.CONTINUOUS
        model.update()

    # optimize the model
    model.optimize()

    solver_status = model.getAttr(GRB.Attr.Status)
    return_code = 0
    if (solver_status == GRB.LOADED):
        status = 'aborted'
        message = 'Model is loaded, but no solution information is availale.'
        term_cond = 'error'
    elif (solver_status == GRB.OPTIMAL):
        status = 'ok'
        message = 'Model was solved to optimality (subject to tolerances), and an optimal solution is available.'
        term_cond = 'optimal'
    elif (solver_status == GRB.INFEASIBLE):
        status = 'warning'
        message = 'Model was proven to be infeasible.'
        term_cond = 'infeasible'
    elif (solver_status == GRB.INF_OR_UNBD):
        status = 'warning'
        message = 'Problem proven to be infeasible or unbounded.'
        term_cond = 'infeasible' # Pyomo doesn't have an analog to "infeasible or unbounded", which is a weird concept anyway.
    elif (solver_status == GRB.UNBOUNDED):
        status = 'warning'
        message = 'Model was proven to be unbounded.'
        term_cond = 'unbounded'
    elif (solver_status == GRB.CUTOFF):
        status = 'aborted'
        message = 'Optimal objective for model was proven to be worse than the value specified in the Cutoff  parameter. No solution information is available.'
        term_cond = 'minFunctionValue'
    elif (solver_status == GRB.ITERATION_LIMIT):
        status = 'aborted'
        message = 'Optimization terminated because the total number of simplex iterations performed exceeded the value specified in the IterationLimit parameter.'
        term_cond = 'maxIterations'
    elif (solver_status == GRB.NODE_LIMIT):
        status = 'aborted'
        message = 'Optimization terminated because the total number of branch-and-cut nodes explored exceeded the value specified in the NodeLimit parameter.'
        term_cond = 'maxEvaluations'
    elif (solver_status == GRB.TIME_LIMIT):
        status = 'aborted'
        message = 'Optimization terminated because the time expended exceeded the value specified in the TimeLimit parameter.'
        term_cond = 'maxTimeLimit'
    elif (solver_status == GRB.SOLUTION_LIMIT):
        status = 'aborted'
        message = 'Optimization terminated because the number of solutions found reached the value specified in the SolutionLimit parameter.'
        term_cond = 'stoppedByLimit'
    elif (solver_status == GRB.INTERRUPTED):
        status = 'aborted'
        message = 'Optimization was terminated by the user.'
        term_cond = 'error'
    elif (solver_status == GRB.NUMERIC):
        status = 'error'
        message = 'Optimization was terminated due to unrecoverable numerical difficulties.'
        term_cond = 'error'
    elif (solver_status == GRB.SUBOPTIMAL):
        status = 'warning'
        message = 'Unable to satisfy optimality tolerances; a sub-optimal solution is available.'
        term_cond = 'other'
    else:
        print(solver_status)
        status = 'error'
        message = 'Unknown return code from GUROBI model.getAttr(GRB.Attr.Status) call'
        term_cond = 'error'

    try:
        obj_value = model.getAttr(GRB.Attr.ObjVal)
    except:
        obj_value = None

    # write the solution file
    solnfile = open(soln_file, "w+")

    # write the information required by results.problem
    solnfile.write("section:problem\n")
    name = model.getAttr(GRB.Attr.ModelName)
    solnfile.write("name: "+name+'\n')

    sense = model.getAttr(GRB.Attr.ModelSense)

    # TODO: find out about bounds and fix this with error checking
    # this line fails for some reason so set the value to unknown
    try:
        bound = model.getAttr(GRB.Attr.ObjBound)
    except Exception:
        if term_cond == 'optimal':
            bound = obj_value
        else:
            bound = None

    if (sense < 0):
        solnfile.write("sense:maximize\n")
        if bound is None:
            solnfile.write("upper_bound: %f\n" % float('infinity'))
        else:
            solnfile.write("upper_bound: %s\n" % str(bound))
    else:
        solnfile.write("sense:minimize\n")
        if bound is None:
            solnfile.write("lower_bound: %f\n" % float('-infinity'))
        else:
            solnfile.write("lower_bound: %s\n" % str(bound))

    # TODO: Get the number of objective functions from GUROBI
    n_objs = 1
    solnfile.write("number_of_objectives: %d\n" % n_objs)

    cons = model.getConstrs()
    qcons = []
    if GUROBI_VERSION[0] >= 5:
        qcons = model.getQConstrs()
    solnfile.write("number_of_constraints: %d\n" % (len(cons)+len(qcons)+model.NumSOS,))

    vars = model.getVars()
    solnfile.write("number_of_variables: %d\n" % len(vars))

    n_binvars = model.getAttr(GRB.Attr.NumBinVars)
    solnfile.write("number_of_binary_variables: %d\n" % n_binvars)

    n_intvars = model.getAttr(GRB.Attr.NumIntVars)
    solnfile.write("number_of_integer_variables: %d\n" % n_intvars)

    solnfile.write("number_of_continuous_variables: %d\n" % (len(vars)-n_intvars,))

    solnfile.write("number_of_nonzeros: %d\n" % model.getAttr(GRB.Attr.NumNZs))

    # write out the information required by results.solver
    solnfile.write("section:solver\n")

    solnfile.write('status: %s\n' % status)
    solnfile.write('return_code: %s\n' % str(return_code))
    solnfile.write('message: %s\n' % message)
    solnfile.write('user_time: %s\n' % str(model.getAttr(GRB.Attr.Runtime)))
    solnfile.write('system_time: %s\n' % str(0.0))
    solnfile.write('termination_condition: %s\n' % term_cond)
    solnfile.write('termination_message: %s\n' % message)

    is_discrete = False
    if (model.getAttr(GRB.Attr.IsMIP)):
        is_discrete = True

    if (term_cond == 'optimal') or (model.getAttr(GRB.Attr.SolCount) >= 1):
        solnfile.write('section:solution\n')
        solnfile.write('status:optimal\n')
        solnfile.write('message: %s\n' % message)
        solnfile.write('objective: %s\n' % str(obj_value))
        solnfile.write('gap: 0.0\n')

        for var in vars:
            solnfile.write('var: %s : %s\n' % (str(var.getAttr(GRB.Attr.VarName)), str(var.getAttr(GRB.Attr.X))))

        if (is_discrete is False) and (extract_reduced_costs is True):
            for var in vars:
                solnfile.write('varrc: %s : %s\n' % (str(var.getAttr(GRB.Attr.VarName)), str(var.getAttr(GRB.Attr.RC))))

        if (is_discrete is False) and (extract_duals is True):
            for con in cons:
               # Pi attributes in Gurobi are the constraint duals
                solnfile.write("constraintdual: %s : %s\n" % (str(con.getAttr(GRB.Attr.ConstrName)), str(con.getAttr(GRB.Attr.Pi))))
            for con in qcons:
                # QCPI attributes in Gurobi are the constraint duals
                solnfile.write("constraintdual: %s : %s\n" % (str(con.getAttr(GRB.Attr.QCName)), str(con.getAttr(GRB.Attr.QCPi))))

        if (extract_slacks is True):
            for con in cons:
                solnfile.write("constraintslack: %s : %s\n" % (con.getAttr(GRB.Attr.ConstrName), str(con.getAttr(GRB.Attr.Slack))))
            for con in qcons:
                solnfile.write("constraintslack: %s : %s\n" % (con.getAttr(GRB.Attr.QCName), str(con.getAttr(GRB.Attr.QCSlack))))

    solnfile.close()

Example 6

Project: RMG-Py
Source File: adjlist.py
View license
def fromAdjacencyList(adjlist, group=False, saturateH=False):
    """
    Convert a string adjacency list `adjlist` into a set of :class:`Atom` and
    :class:`Bond` objects.
    """
    atoms = []
    atomdict = {}
    bonds = {}
    multiplicity = None
    
    adjlist = adjlist.strip()
    lines = adjlist.splitlines()
    if adjlist == '' or len(lines) == 0:
        raise InvalidAdjacencyListError('Empty adjacency list.')

    # Detect old-style adjacency lists by looking at the last line's syntax
    lastLine = lines[-1].strip()
    while not lastLine:  # Remove any empty lines from the end
        lines.pop()
        lastLine = lines[-1].strip()
    if re_IntermediateAdjList.match(lastLine):
        logging.debug("adjacency list:\n{1}\nline '{0}' looks like an intermediate style adjacency list".format(lastLine, adjlist))
        return fromOldAdjacencyList(adjlist, group=group, saturateH=saturateH)
    if re_OldAdjList.match(lastLine):
        logging.debug("Adjacency list:\n{1}\nline '{0}' looks like an old style adjacency list".format(lastLine, adjlist))
        if not group:
            logging.debug("Will assume implicit H atoms")
        return fromOldAdjacencyList(adjlist, group=group, saturateH=(not group))

    # Interpret the first line if it contains a label
    if len(lines[0].split()) == 1:
        label = lines.pop(0)
        if len(lines) == 0:
            raise InvalidAdjacencyListError('No atoms specified in adjacency list.')
        
    # Interpret the second line if it contains a multiplicity
    if lines[0].split()[0] == 'multiplicity':
        line = lines.pop(0)
        if group:
            match = re.match('\s*multiplicity\s+\[\s*(\d(?:,\s*\d)*)\s*\]\s*$', line)
            if not match:
                rematch = re.match('\s*multiplicity\s+x\s*$', line)
                assert rematch, "Invalid multiplicity line '{0}'. Should be a list like 'multiplicity [1,2,3]' or a wildcard 'multiplicity x'".format(line)
            else:
            # should match "multiplicity [1]" or " multiplicity   [ 1, 2, 3 ]" or " multiplicity [1,2,3]"
            # and whatever's inside the [] (excluding leading and trailing spaces) should be captured as group 1.
            # If a wildcard is desired, this line can be omitted or replaced with 'multiplicity x'
            # Multiplicities must be only one digit (i.e. less than 10)
            # The (?:,\s*\d)* matches patters like ", 2" 0 or more times, but doesn't capture them (because of the leading ?:)            
                multiplicities = match.group(1).split(',')
                multiplicity = [int(i) for i in multiplicities]
        else:
            match = re.match('\s*multiplicity\s+\d+\s*$', line)
            assert match, "Invalid multiplicity line '{0}'. Should be an integer like 'multiplicity 2'".format(line)
            multiplicity = int(line.split()[1])
        if len(lines) == 0:
            raise InvalidAdjacencyListError('No atoms specified in adjacency list: \n{0}'.format(adjlist))
    
    mistake1 = re.compile('\{[^}]*\s+[^}]*\}')
    # Iterate over the remaining lines, generating Atom or GroupAtom objects
    for line in lines:

        # Sometimes people put spaces after commas, which messes up the
        # parse-by-whitespace. Examples include '[Cd, Ct]'.
        if mistake1.search(line):
            raise InvalidAdjacencyListError(
                "{1} Shouldn't have spaces inside braces:\n{0}".format(mistake1.search(line).group(), adjlist)
                )

        # Sometimes commas are used to delimit bonds in the bond list,
        # so replace them just in case
        line = line.replace('},{', '} {')
        
        data = line.split()

        # Skip if blank line
        if len(data) == 0: continue

        # First item is index for atom
        # Sometimes these have a trailing period (as if in a numbered list),
        # so remove it just in case
        aid = int(data[0].strip('.'))

        # If second item starts with '*', then atom is labeled
        label = ''; index = 1
        if data[1][0] == '*':
            label = data[1]
            index += 1

        # Next is the element or functional group element
        # A list can be specified with the {,} syntax
        atomType = data[index]
        if atomType[0] == '[':
            if not group:
                raise InvalidAdjacencyListError("Error on:\n{0}\nA molecule should not assign more than one atomtype per atom.".format(adjlist))
            atomType = atomType[1:-1].split(',')
        else:
            atomType = [atomType]
        index += 1
        
        # Next the number of unpaired electrons
        unpairedElectrons = []
        uState = data[index]
        if uState[0] == 'u':
            if uState[1] == '[':
                uState = uState[2:-1].split(',')
            else:
                uState = [uState[1]]
            for u in uState:
                if u == '0':
                    unpairedElectrons.append(0)
                elif u == '1':
                    unpairedElectrons.append(1)
                elif u == '2':
                    unpairedElectrons.append(2)
                elif u == '3':
                    unpairedElectrons.append(3)
                elif u == '4':
                    unpairedElectrons.append(4)
                elif u == 'x':
                    if not group:
                        raise InvalidAdjacencyListError("Error on:\n{0}\nA molecule should not assign a wildcard to number of unpaired electrons.".format(adjlist))
                else:
                    raise InvalidAdjacencyListError('Number of unpaired electrons not recognized on\n{0}.'.format(adjlist))
            index += 1
        else:
            raise InvalidAdjacencyListError('Number of unpaired electrons not defined on\n{0}.'.format(adjlist))
        
        # Next the number of lone electron pairs (if provided)
        lonePairs = []
        if len(data) > index:
            lpState = data[index]
            if lpState[0] == 'p':
                if lpState[1] == '[':
                    lpState = lpState[2:-1].split(',')
                else:
                    lpState = [lpState[1]]
                for l in lpState:
                    if l == '0':
                        lonePairs.append(0)
                    elif l == '1':
                        lonePairs.append(1)
                    elif l == '2':
                        lonePairs.append(2)
                    elif l == '3':
                        lonePairs.append(3)
                    elif l == '4':
                        lonePairs.append(4)
                    elif l == 'x':
                        if not group:
                            raise InvalidAdjacencyListError("Error in adjacency list:\n{0}\nA molecule should not have a wildcard assigned to number of lone pairs.".format(adjlist))
                    else:
                        raise InvalidAdjacencyListError('Error in adjacency list:\n{0}\nNumber of lone electron pairs not recognized.'.format(adjlist))
                index += 1
            else:
                if not group:
                    lonePairs.append(0)
        else:
            if not group:
                lonePairs.append(0)
            
        # Next the number of partial charges (if provided)
        partialCharges = []
        if len(data) > index:
            eState = data[index]
            if eState[0] == 'c':
                if eState[1] == '[':
                    eState = eState[2:-1].split(',')
                else:
                    eState = [eState[1:]]
                for e in eState:
                    if e == '0':
                        partialCharges.append(0)
                    elif e == '+1':
                        partialCharges.append(1)
                    elif e == '+2':
                        partialCharges.append(2)
                    elif e == '+3':
                        partialCharges.append(3)
                    elif e == '+4':
                        partialCharges.append(4)
                    elif e == '-1':
                        partialCharges.append(-1)
                    elif e == '-2':
                        partialCharges.append(-2)
                    elif e == '-3':
                        partialCharges.append(-3)
                    elif e == '-4':
                        partialCharges.append(-4)
                    elif e == 'x':
                        if not group:
                            raise InvalidAdjacencyListError("Error on adjacency list:\n{0}\nA molecule should not have a wildcard assigned to number of charges.".format(adjlist))
                    else:
                        raise InvalidAdjacencyListError('Error on adjacency list:\n{0}\nNumber of partial charges not recognized.'.format(adjlist))
                index += 1
            else:
                if not group:
                    partialCharges.append(0)
        else:
            if not group:
                partialCharges.append(0)
        

        # Next the isotope (if provided)
        isotope = -1
        if len(data) > index:
            iState = data[index]
            if iState[0] == 'i':
                isotope = int(iState[1:])
                index += 1


        # Create a new atom based on the above information
        if group:
            atom = GroupAtom(atomType, unpairedElectrons, partialCharges, label, lonePairs)
        else:
            atom = Atom(atomType[0], unpairedElectrons[0], partialCharges[0], label, lonePairs[0])
            if isotope != -1:
                atom.element = getElement(atom.number, isotope)

        # Add the atom to the list
        atoms.append(atom)
        atomdict[aid] = atom
        
        # Process list of bonds
        bonds[aid] = {}
        for datum in data[index:]:

            # Sometimes commas are used to delimit bonds in the bond list,
            # so strip them just in case
            datum = datum.strip(',')
            
            aid2, comma, order = datum[1:-1].partition(',')
            aid2 = int(aid2)
            if aid == aid2:
                raise InvalidAdjacencyListError('Error in adjacency list:\n{1}\nAttempted to create a bond between atom {0:d} and itself.'.format(aid, adjlist))
            
            if order[0] == '[':
                order = order[1:-1].split(',')
            else:
                order = [order]

            bonds[aid][aid2] = order

    # Check consistency using bonddict
    for atom1 in bonds:
        for atom2 in bonds[atom1]:
            if atom2 not in bonds:
                raise InvalidAdjacencyListError('Error in adjacency list:\n{1}\nAtom {0:d} not in bond dictionary.'.format(atom2, adjlist))
            elif atom1 not in bonds[atom2]:
                raise InvalidAdjacencyListError('Error in adjacency list:\n{2}\nFound bond between {0:d} and {1:d}, but not the reverse.'.format(atom1, atom2, adjlist))
            elif bonds[atom1][atom2] != bonds[atom2][atom1]:
                raise InvalidAdjacencyListError('Error in adjacency list:\n{4}\nFound bonds between {0:d} and {1:d}, but of different orders "{2}" and "{3}".'.format(atom1, atom2, bonds[atom1][atom2], bonds[atom2][atom1], adjlist))

    # Convert bonddict to use Atom[group] and Bond[group] objects
    atomkeys = atomdict.keys()
    atomkeys.sort()
    for aid1 in atomkeys:
        atomkeys2 = bonds[aid1].keys()
        atomkeys2.sort()
        for aid2 in atomkeys2:
            if aid1 < aid2:
                atom1 = atomdict[aid1]
                atom2 = atomdict[aid2]
                order = bonds[aid1][aid2]
                if group:
                    bond = GroupBond(atom1, atom2, order)
                elif len(order) == 1:
                    bond = Bond(atom1, atom2, order[0])
                else:
                    raise InvalidAdjacencyListError('Error in adjacency list:\n{0}\nMultiple bond orders specified for an atom in a Molecule.'.format(adjlist))
                atom1.edges[atom2] = bond
                atom2.edges[atom1] = bond
    
    if saturateH:
        # Add explicit hydrogen atoms to complete structure if desired
        if not group:
            Saturator.saturate(atoms)

    
    # Consistency checks
    if not group:
        # Molecule consistency check
        # Electron and valency consistency check for each atom
        for atom in atoms: ConsistencyChecker.check_partial_charge(atom)

        nRad = sum([atom.radicalElectrons for atom in atoms])
        absolute_spin_per_electron = 1/2.
        if multiplicity == None: multiplicity = 2* (nRad * absolute_spin_per_electron) + 1
            
        ConsistencyChecker.check_multiplicity(nRad, multiplicity)
        for atom in atoms: ConsistencyChecker.check_hund_rule(atom, multiplicity)
        return atoms, multiplicity
    else:
        # Currently no group consistency check
        return atoms, multiplicity

Example 7

Project: rockstor-core
Source File: disk.py
View license
    @staticmethod
    @transaction.atomic
    def _update_disk_state():
        """
        A db atomic method to update the database of attached disks / drives.
        Works only on device serial numbers for drive identification.
        Calls scan_disks to establish the current connected drives info.
        Initially removes duplicate by serial number db entries to deal
        with legacy db states and obfuscates all previous device names as they
        are transient. The drive database is then updated with the attached
        disks info and previously known drives no longer found attached are
        marked as offline. All offline drives have their SMART availability and
        activation status removed and all attached drives have their SMART
        availability assessed and activated if available.
        :return: serialized models of attached and missing disks via serial num
        """
        # Acquire a list (namedtupil collection) of attached drives > min size
        disks = scan_disks(settings.MIN_DISK_SIZE)
        serial_numbers_seen = []
        # Make sane our db entries in view of what we know we have attached.
        # Device serial number is only known external unique entry, scan_disks
        # make this so in the case of empty or repeat entries by providing
        # fake serial numbers which are in turn flagged via WebUI as unreliable.
        # 1) scrub all device names with unique but nonsense uuid4
        # 1) mark all offline disks as such via db flag
        # 2) mark all offline disks smart available and enabled flags as False
        # logger.info('update_disk_state() Called')
        for do in Disk.objects.all():
            # Replace all device names with a unique placeholder on each scan
            # N.B. do not optimize by re-using uuid index as this could lead
            # to a non refreshed webui acting upon an entry that is different
            # from that shown to the user.
            do.name = 'detached-' + str(uuid.uuid4()).replace('-', '')
            # Delete duplicate or fake by serial number db disk entries.
            # It makes no sense to save fake serial number drives between scans
            # as on each scan the serial number is re-generated (fake) anyway.
            # Serial numbers beginning with 'fake-serial-' are from scan_disks.
            if (do.serial in serial_numbers_seen) or (
                    re.match('fake-serial-', do.serial) is not None):
                logger.info('Deleting duplicate or fake (by serial) Disk db '
                            'entry. Serial = %s' % do.serial)
                do.delete()  # django >=1.9 returns a dict of deleted items.
                # Continue onto next db disk object as nothing more to process.
                continue
            # first encounter of this serial in the db so stash it for reference
            serial_numbers_seen.append(deepcopy(do.serial))
            # Look for devices (by serial number) that are in the db but not in
            # our disk scan, ie offline / missing.
            if (do.serial not in [d.serial for d in disks]):
                # update the db entry as offline
                do.offline = True
                # disable S.M.A.R.T available and enabled flags.
                do.smart_available = do.smart_enabled = False
            do.save()  # make sure all updates are flushed to db
        # Our db now has no device name info as all dev names are place holders.
        # Iterate over attached drives to update the db's knowledge of them.
        # Kernel dev names are unique so safe to overwrite our db unique name.
        for d in disks:
            # start with an empty disk object
            dob = None
            # Convert our transient but just scanned so current sda type name
            # to a more useful by-id type name as found in /dev/disk/by-id
            byid_disk_name, is_byid = get_dev_byid_name(d.name, True)
            # If the db has an entry with this disk's serial number then
            # use this db entry and update the device name from our recent scan.
            if (Disk.objects.filter(serial=d.serial).exists()):
                dob = Disk.objects.get(serial=d.serial)
                #dob.name = d.name
                dob.name = byid_disk_name
            else:
                # We have an assumed new disk entry as no serial match in db.
                # Build a new entry for this disk.
                #dob = Disk(name=d.name, serial=d.serial)
                # N.B. we may want to force a fake-serial here if is_byid False,
                # that way we flag as unusable disk as no by-id type name found.
                # It may already have been set though as the only by-id
                # failures so far are virtio disks with no serial so scan_disks
                # will have already given it a fake serial in d.serial.
                dob = Disk(name=byid_disk_name, serial=d.serial)
            # Update the db disk object (existing or new) with our scanned info
            dob.size = d.size
            dob.parted = d.parted
            dob.offline = False  # as we are iterating over attached devices
            dob.model = d.model
            dob.transport = d.transport
            dob.vendor = d.vendor
            dob.btrfs_uuid = d.btrfs_uuid
            # If attached disk has an fs and it isn't btrfs
            if (d.fstype is not None and d.fstype != 'btrfs'):
                dob.btrfs_uuid = None
                dob.parted = True  # overload use of parted as non btrfs flag.
                # N.B. this overload use may become redundant with the addition
                # of the Disk.role field.
            # Update the role field with scan_disks findings, currently only
            # mdraid membership type based on fstype info. In the case of
            # these raid member indicators from scan_disks() we have the
            # current truth provided so update the db role status accordingly.
            # N.B. this if else could be expanded to accommodate other
            # roles based on the fs found
            if d.fstype == 'isw_raid_member' or d.fstype == 'linux_raid_member':
                # We have an indicator of mdraid membership so update existing
                # role info if any.
                # N.B. We have a minor legacy issue in that prior to using json
                # format for the db role field we stored one of 2 strings.
                # if these 2 strings are found then ignore them as we then
                # overwrite with our current finding and in the new json format.
                # I.e. non None could also be a legacy entry so follow overwrite
                # path when legacy entry found by treating as a None entry.
                # TODO: When we reset migrations the following need only check
                # TODO: "dob.role is not None"
                if dob.role is not None and dob.role != 'isw_raid_member' \
                        and dob.role != 'linux_raid_member':
                    # get our known roles into a dictionary
                    known_roles = json.loads(dob.role)
                    # create or update an mdraid dictionary entry
                    known_roles['mdraid'] = str(d.fstype)
                    # return updated dict to json format and store in db object
                    dob.role = json.dumps(known_roles)
                else:  # We have a dob.role = None so just insert our new role.
                    # Also applies to legacy pre json role entries.
                    dob.role = '{"mdraid": "' + d.fstype + '"}'  # json string
            else:  # We know this disk is not an mdraid raid member.
                # No identified role from scan_disks() fstype value (mdraid
                # only for now )so we preserve any prior known roles not
                # exposed by scan_disks but remove the mdraid role if found.
                # TODO: When we reset migrations the following need only check
                # TODO: "dob.role is not None"
                if dob.role is not None and dob.role != 'isw_raid_member' \
                        and dob.role != 'linux_raid_member':
                    # remove mdraid role if found but preserve prior roles
                    # which should now only be in json format
                    known_roles = json.loads(dob.role)
                    if 'mdraid' in known_roles:
                        if len(known_roles) > 1:
                            # mdraid is not the only entry so we have to pull
                            # out only mdraid from dict and convert back to json
                            del known_roles['mdraid']
                            dob.role = json.dumps(known_roles)
                        else:
                            # mdraid was the only entry so we need not bother
                            # with dict edit and json conversion only to end up
                            # with an empty json {} so revert to default 'None'.
                            dob.role = None
                else:  # Empty or legacy role entry.
                    # We have either None or a legacy mdraid role when this disk
                    # is no longer an mdraid member. We can now assert None.
                    dob.role = None
            # If our existing Pool db knows of this disk's pool via it's label:
            if (Pool.objects.filter(name=d.label).exists()):
                # update the disk db object's pool field accordingly.
                dob.pool = Pool.objects.get(name=d.label)

                #this is for backwards compatibility. root pools created
                #before the pool.role migration need this. It can safely be
                #removed a few versions after 3.8-11 or when we reset migrations.
                if (d.root is True):
                    dob.pool.role = 'root'
                    dob.pool.save()
            else:  # this disk is not known to exist in any pool via it's label
                dob.pool = None
            # If no pool has yet been found with this disk's label in and
            # the attached disk is our root disk (flagged by scan_disks)
            if (dob.pool is None and d.root is True):
                # setup our special root disk db entry in Pool
                # TODO: dynamically retrieve raid level.
                p = Pool(name=d.label, raid='single', role='root')
                p.disk_set.add(dob)
                p.save()
                # update disk db object to reflect special root pool status
                dob.pool = p
                dob.save()
                p.size = pool_usage(mount_root(p))[0]
                enable_quota(p)
                p.uuid = btrfs_uuid(dob.name)
                p.save()
            # save our updated db disk object
            dob.save()
        # Update online db entries with S.M.A.R.T availability and status.
        for do in Disk.objects.all():
            # find all the not offline db entries
            if (not do.offline):
                # We have an attached disk db entry.
                # Since our Disk.name model now uses by-id type names we can
                # do cheap matches to the beginnings of these names to find
                # virtio, md, or sdcard devices which are assumed to have no
                # SMART capability.
                # We also disable devices smart support when they have a
                # fake serial number as ascribed by scan_disks as any SMART
                # data collected is then less likely to be wrongly associated
                # with the next device that takes this temporary drive's name.
                # Also note that with no serial number some device types will
                # not have a by-id type name expected by the smart subsystem.
                # This has only been observed in no serial virtio devices.
                if (re.match('fake-serial-', do.serial) is not None) or \
                        (re.match('virtio-|md-|mmc-|nvme-', do.name) is not None):
                    # Virtio disks (named virtio-*), md devices (named md-*),
                    # and an sdcard reader that provides devs named mmc-* have
                    # no smart capability so avoid cluttering logs with
                    # exceptions on probing these with smart.available.
                    # nvme not yet supported by CentOS 7 smartmontools:
                    # https://www.smartmontools.org/ticket/657
                    # Thanks to @snafu in rockstor forum post 1567 for this.
                    do.smart_available = do.smart_enabled = False
                    continue
                # try to establish smart availability and status and update db
                try:
                    # for non ata/sata drives
                    do.smart_available, do.smart_enabled = smart.available(
                        do.name, do.smart_options)
                except Exception, e:
                    logger.exception(e)
                    do.smart_available = do.smart_enabled = False
            do.save()
        ds = DiskInfoSerializer(Disk.objects.all().order_by('name'), many=True)
        return Response(ds.data)

Example 8

Project: rpmlint
Source File: SpecCheck.py
View license
    def check_spec(self, pkg, spec_file):
        self._spec_file = spec_file
        spec_only = isinstance(pkg, Pkg.FakePkg)
        patches = {}
        applied_patches = []
        applied_patches_ifarch = []
        patches_auto_applied = False
        source_dir = False
        buildroot = False
        configure_linenum = None
        configure_cmdline = ""
        mklibname = False
        is_lib_pkg = False
        if_depth = 0
        ifarch_depth = -1
        current_section = 'package'
        buildroot_clean = {'clean': False, 'install': False}
        depscript_override = False
        depgen_disabled = False
        patch_fuzz_override = False
        indent_spaces = 0
        indent_tabs = 0
        section = {}
        # None == main package
        current_package = None
        package_noarch = {}

        is_utf8 = False
        if self._spec_file and use_utf8:
            if Pkg.is_utf8(self._spec_file):
                is_utf8 = True
            else:
                printError(pkg, "non-utf8-spec-file",
                           self._spec_name or self._spec_file)

        # gather info from spec lines

        pkg.current_linenum = 0

        nbsp = UNICODE_NBSP if is_utf8 else chr(0xA0)
        do_unicode = is_utf8 and sys.version_info[0] <= 2

        for line in Pkg.readlines(spec_file):

            pkg.current_linenum += 1

            if do_unicode:
                line = unicode(line, "utf-8", "replace")  # noqa false positive

            char = line.find(nbsp)
            if char != -1:
                printWarning(pkg, "non-break-space", "line %s, char %d" %
                             (pkg.current_linenum, char))

            section_marker = False
            for sec, regex in section_regexs.items():
                res = regex.search(line)
                if res:
                    current_section = sec
                    section_marker = True
                    section[sec] = section.get(sec, 0) + 1
                    if sec in ('package', 'files'):
                        rest = filelist_regex.sub('', line[res.end() - 1:])
                        res = pkgname_regex.search(rest)
                        if res:
                            current_package = res.group(1)
                        else:
                            current_package = None
                    break

            if section_marker:

                if not is_lib_pkg and lib_package_regex.search(line):
                    is_lib_pkg = True

                continue

            if current_section in ('prep', 'build') and \
                    contains_buildroot(line):
                printWarning(pkg, 'rpm-buildroot-usage', '%' + current_section,
                             line[:-1].strip())

            if make_check_regex.search(line) and current_section not in \
                    ('check', 'changelog', 'package', 'description'):
                printWarning(pkg, 'make-check-outside-check-section',
                             line[:-1])

            if current_section in buildroot_clean and \
                    not buildroot_clean[current_section] and \
                    contains_buildroot(line) and rm_regex.search(line):
                buildroot_clean[current_section] = True

            if ifarch_regex.search(line):
                if_depth = if_depth + 1
                ifarch_depth = if_depth

            if if_regex.search(line):
                if_depth = if_depth + 1

            if setup_regex.match(line):
                if not setup_q_regex.search(line):
                    # Don't warn if there's a -T without -a or -b
                    if setup_t_regex.search(line):
                        if setup_ab_regex.search(line):
                            printWarning(pkg, 'setup-not-quiet')
                    else:
                        printWarning(pkg, 'setup-not-quiet')
                if current_section != 'prep':
                    printWarning(pkg, 'setup-not-in-prep')
            elif autopatch_regex.search(line):
                patches_auto_applied = True
                if current_section != 'prep':
                    printWarning(pkg, '%autopatch-not-in-prep')
            else:
                res = autosetup_regex.search(line)
                if res:
                    if not autosetup_n_regex.search(res.group(1)):
                        patches_auto_applied = True
                    if current_section != 'prep':
                        printWarning(pkg, '%autosetup-not-in-prep')

            if endif_regex.search(line):
                if ifarch_depth == if_depth:
                    ifarch_depth = -1
                if_depth = if_depth - 1

            res = applied_patch_regex.search(line)
            if res:
                pnum = res.group(1) or 0
                for tmp in applied_patch_p_regex.findall(line) or [pnum]:
                    pnum = int(tmp)
                    applied_patches.append(pnum)
                    if ifarch_depth > 0:
                        applied_patches_ifarch.append(pnum)
            else:
                res = applied_patch_pipe_regex.search(line)
                if res:
                    pnum = int(res.group(1))
                    applied_patches.append(pnum)
                    if ifarch_depth > 0:
                        applied_patches_ifarch.append(pnum)
            if not res and not source_dir:
                res = source_dir_regex.search(line)
                if res:
                    source_dir = True
                    printError(pkg, "use-of-RPM_SOURCE_DIR")

            if configure_linenum:
                if configure_cmdline[-1] == "\\":
                    configure_cmdline = configure_cmdline[:-1] + line.strip()
                else:
                    res = configure_libdir_spec_regex.search(configure_cmdline)
                    if not res:
                        # Hack to get the correct (start of ./configure) line
                        # number displayed:
                        real_linenum = pkg.current_linenum
                        pkg.current_linenum = configure_linenum
                        printWarning(pkg, "configure-without-libdir-spec")
                        pkg.current_linenum = real_linenum
                    elif res.group(1):
                        res = re.match(hardcoded_library_paths, res.group(1))
                        if res:
                            printError(pkg, "hardcoded-library-path",
                                       res.group(1), "in configure options")
                    configure_linenum = None

            hashPos = line.find("#")

            if current_section != 'changelog':
                cfgPos = line.find('./configure')
                if cfgPos != -1 and (hashPos == -1 or hashPos > cfgPos):
                    # store line where it started
                    configure_linenum = pkg.current_linenum
                    configure_cmdline = line.strip()

            res = hardcoded_library_path_regex.search(line)
            if current_section != 'changelog' and res and not \
                    (biarch_package_regex.match(pkg.name) or
                     hardcoded_lib_path_exceptions_regex.search(
                         res.group(1).lstrip())):
                printError(pkg, "hardcoded-library-path", "in",
                           res.group(1).lstrip())

            if '%mklibname' in line:
                mklibname = True

            if current_section == 'package':

                # Would be cleaner to get sources and patches from the
                # specfile parsed in Python (see below), but we want to
                # catch %ifarch'd etc ones as well, and also catch these when
                # the specfile is not parseable.

                res = patch_regex.search(line)
                if res:
                    pnum = int(res.group(1) or 0)
                    patches[pnum] = res.group(2)

                res = obsolete_tags_regex.search(line)
                if res:
                    printWarning(pkg, "obsolete-tag", res.group(1))

                res = buildroot_regex.search(line)
                if res:
                    buildroot = True
                    if res.group(1).startswith('/'):
                        printWarning(pkg, 'hardcoded-path-in-buildroot-tag',
                                     res.group(1))

                res = buildarch_regex.search(line)
                if res:
                    if res.group(1) != "noarch":
                        printError(pkg,
                                   'buildarch-instead-of-exclusivearch-tag',
                                   res.group(1))
                    else:
                        package_noarch[current_package] = True

                res = packager_regex.search(line)
                if res:
                    printWarning(pkg, 'hardcoded-packager-tag', res.group(1))

                res = prefix_regex.search(line)
                if res:
                    if not res.group(1).startswith('%'):
                        printWarning(pkg, 'hardcoded-prefix-tag', res.group(1))

                res = prereq_regex.search(line)
                if res:
                    printError(pkg, 'prereq-use', res.group(2))

                res = buildprereq_regex.search(line)
                if res:
                    printError(pkg, 'buildprereq-use', res.group(1))

                if scriptlet_requires_regex.search(line):
                    printError(pkg, 'broken-syntax-in-scriptlet-requires',
                               line.strip())

                res = requires_regex.search(line)
                if res:
                    reqs = Pkg.parse_deps(res.group(1))
                    for req in unversioned(reqs):
                        if compop_regex.search(req):
                            printWarning(pkg,
                                         'comparison-operator-in-deptoken',
                                         req)

                res = provides_regex.search(line)
                if res:
                    provs = Pkg.parse_deps(res.group(1))
                    for prov in unversioned(provs):
                        printWarning(pkg, 'unversioned-explicit-provides',
                                     prov)
                        if compop_regex.search(prov):
                            printWarning(pkg,
                                         'comparison-operator-in-deptoken',
                                         prov)

                res = obsoletes_regex.search(line)
                if res:
                    obses = Pkg.parse_deps(res.group(1))
                    for obs in unversioned(obses):
                        printWarning(pkg, 'unversioned-explicit-obsoletes',
                                     obs)
                        if compop_regex.search(obs):
                            printWarning(pkg,
                                         'comparison-operator-in-deptoken',
                                         obs)

                res = conflicts_regex.search(line)
                if res:
                    confs = Pkg.parse_deps(res.group(1))
                    for conf in unversioned(confs):
                        if compop_regex.search(conf):
                            printWarning(pkg,
                                         'comparison-operator-in-deptoken',
                                         conf)

            if current_section == 'changelog':
                for match in AbstractCheck.macro_regex.findall(line):
                    res = re.match('%+', match)
                    if len(res.group(0)) % 2:
                        printWarning(pkg, 'macro-in-%changelog', match)
            else:
                if not depscript_override:
                    depscript_override = \
                        depscript_override_regex.search(line) is not None
                if not depgen_disabled:
                    depgen_disabled = \
                        depgen_disable_regex.search(line) is not None
                if not patch_fuzz_override:
                    patch_fuzz_override = \
                        patch_fuzz_override_regex.search(line) is not None

            if current_section == 'files':
                # TODO: check scriptlets for these too?
                if package_noarch.get(current_package) or \
                        (current_package not in package_noarch and
                         package_noarch.get(None)):
                    res = libdir_regex.search(line)
                    if res:
                        pkgname = current_package
                        if pkgname is None:
                            pkgname = '(main package)'
                        printWarning(pkg, 'libdir-macro-in-noarch-package',
                                     pkgname, line.rstrip())

            if not indent_tabs and '\t' in line:
                indent_tabs = pkg.current_linenum
            if not indent_spaces and indent_spaces_regex.search(line):
                indent_spaces = pkg.current_linenum

            # Check if egrep or fgrep is used
            if current_section not in \
                    ('package', 'changelog', 'description', 'files'):
                greps = deprecated_grep_regex.findall(line)
                if greps:
                    printWarning(pkg, "deprecated-grep", greps)

            # If not checking spec file only, we're checking one inside a
            # SRPM -> skip this check to avoid duplicate warnings (#167)
            if spec_only and VALID_GROUPS and \
               line.lower().startswith("group:"):
                group = line[6:].strip()
                if group not in VALID_GROUPS:
                    printWarning(pkg, 'non-standard-group', group)

            # Test if there are macros in comments
            if hashPos != -1 and \
                    (hashPos == 0 or line[hashPos - 1] in (" ", "\t")):
                for match in AbstractCheck.macro_regex.findall(
                        line[hashPos + 1:]):
                    res = re.match('%+', match)
                    if len(res.group(0)) % 2:
                        printWarning(pkg, 'macro-in-comment', match)

        # Last line read is not useful after this point
        pkg.current_linenum = None

        for sect in (x for x in buildroot_clean if not buildroot_clean[x]):
            printWarning(pkg, 'no-cleaning-of-buildroot', '%' + sect)

        if not buildroot:
            printWarning(pkg, 'no-buildroot-tag')

        for sec in ('prep', 'build', 'install', 'clean'):
            if not section.get(sec):
                printWarning(pkg, 'no-%%%s-section' % sec)
        for sec in ('changelog',):
            # prep, build, install, clean, check prevented by rpmbuild 4.4
            if section.get(sec, 0) > 1:
                printWarning(pkg, 'more-than-one-%%%s-section' % sec)

        if is_lib_pkg and not mklibname:
            printError(pkg, 'lib-package-without-%mklibname')

        if depscript_override and not depgen_disabled:
            printWarning(pkg, 'depscript-without-disabling-depgen')

        if patch_fuzz_override:
            printWarning(pkg, 'patch-fuzz-is-changed')

        if indent_spaces and indent_tabs:
            pkg.current_linenum = max(indent_spaces, indent_tabs)
            printWarning(pkg, 'mixed-use-of-spaces-and-tabs',
                         '(spaces: line %d, tab: line %d)' %
                         (indent_spaces, indent_tabs))
            pkg.current_linenum = None

        # process gathered info
        if not patches_auto_applied:
            for pnum, pfile in patches.items():
                if pnum in applied_patches_ifarch:
                    printWarning(pkg, "%ifarch-applied-patch",
                                 "Patch%d:" % pnum, pfile)
                if pnum not in applied_patches:
                    printWarning(pkg, "patch-not-applied",
                                 "Patch%d:" % pnum, pfile)

        # Rest of the checks require a real spec file
        if not self._spec_file:
            return

        # We'd like to parse the specfile only once using python bindings,
        # but it seems errors from rpmlib get logged to stderr and we can't
        # capture and print them nicely, so we do it once each way :P

        out = Pkg.getstatusoutput(('env', 'LC_ALL=C', 'rpm', '-q',
                                   '--qf=', '--specfile', self._spec_file))
        parse_error = False
        for line in out[1].splitlines():
            # No such file or dir hack: https://bugzilla.redhat.com/487855
            if 'No such file or directory' not in line:
                parse_error = True
                printError(pkg, 'specfile-error', line)

        if not parse_error:
            # grab sources and patches from parsed spec object to get
            # them with macros expanded for URL checking

            spec_obj = None
            try:
                ts = rpm.TransactionSet()
                spec_obj = ts.parseSpec(self._spec_file)
            except:
                # errors logged above already
                pass
            if spec_obj:
                try:
                    # rpm < 4.8.0
                    sources = spec_obj.sources()
                except TypeError:
                    # rpm >= 4.8.0
                    sources = spec_obj.sources
                for src in sources:
                    (url, num, flags) = src
                    (scheme, netloc) = urlparse(url)[0:2]
                    if flags & 1:  # rpmspec.h, rpm.org ticket #123
                        srctype = "Source"
                    else:
                        srctype = "Patch"
                    tag = '%s%s' % (srctype, num)
                    if scheme and netloc:
                        info = self.check_url(pkg, tag, url)
                        if not info or not hasattr(pkg, 'files'):
                            continue
                        clen = info.get("Content-Length")
                        if clen is not None:
                            clen = int(clen)
                        cmd5 = info.get("Content-MD5")
                        if cmd5 is not None:
                            cmd5 = cmd5.lower()
                        if clen is not None or cmd5 is not None:
                            # Not using path from urlparse results to match how
                            # rpm itself parses the basename.
                            pkgfile = pkg.files().get(url.split("/")[-1])
                            if pkgfile:
                                if clen is not None and pkgfile.size != clen:
                                    printWarning(pkg, 'file-size-mismatch',
                                                 '%s = %s, %s = %s' %
                                                 (pkgfile.name, pkgfile.size,
                                                  url, clen))
                                # pkgfile.md5 could be some other digest than
                                # MD5, treat as MD5 only if it's 32 chars long
                                if cmd5 and len(pkgfile.md5) == 32 \
                                        and pkgfile.md5 != cmd5:
                                    printWarning(pkg, 'file-md5-mismatch',
                                                 '%s = %s, %s = %s' %
                                                 (pkgfile.name, pkgfile.md5,
                                                  url, cmd5))
                    elif srctype == "Source" and tarball_regex.search(url):
                        printWarning(pkg, 'invalid-url', '%s:' % tag, url)

Example 9

Project: NIPAP
Source File: text-import.py
View license
    def parse_line(self, line):
        """ Parse one line
        """

        try:
            # text params, ie params from the text file
            tp = self.split_columns(line)
        except CommentLine:
            # just ignore comments
            return

        if tp['prefix_type'] == 'reservation':  # reservations / aggregates
            print "Reservation:", tp['prefix'], tp['description']
            p = Prefix()
            p.schema = self.schema
            p.prefix = tp['prefix']
            p.type = 'reservation'
            p.description = tp['description']
            p.monitor = True
            p.alarm_priority = 'low'
            p.authoritative_source = 'nw'
            p.save({})
            return

        elif tp['node'] == '.' and tp['description'] == '.':
            # ignore prefixes without description or node set
            return

        elif tp['prefix_length'] == 32:   # loopback
            # if it's a loopback, the covering prefix will be a reservation and we can just insert an assignment.
            # if this insert fails, it means the parent prefix is an assignment and we instead insert a host
            try:
                p = Prefix()
                p.schema = self.schema
                p.prefix = tp['prefix']
                # loopbacks are always of type 'assignment'
                p.type = 'assignment'
                p.node = tp['node']
                p.description = tp['description']
                p.monitor = True
                p.alarm_priority = tp['alarm_priority']
                p.authoritative_source = 'nw'
                p.save({})
                print "Loopback:", tp['prefix']
                return
            except:
                p = Prefix()
                p.schema = self.schema
                p.prefix = tp['prefix']
                # loopbacks are always of type 'assignment'
                p.type = 'host'
                p.node = tp['node']
                p.description = tp['description']
                p.monitor = True
                p.alarm_priority = tp['alarm_priority']
                p.authoritative_source = 'nw'
                p.save({})
                print "Host:", tp['prefix']
                return

        elif tp['prefix_length'] == 30 or tp['prefix_length'] == 31:   # link network
            octets = tp['address'].split('.')
            prefix_node1 = None
            prefix_node2 = None
            if tp['prefix_length'] == 30:
                prefix_node1 = '.'.join(octets[:3] + [str( int(octets[3]) + 1 )] ) + '/32'
                prefix_node2 = '.'.join(octets[:3] + [str( int(octets[3]) + 2 )] ) + '/32'
            else:
                prefix_node1 = '.'.join(octets) + '/32'
                prefix_node2 = '.'.join(octets[:3] + [str( int(octets[3]) + 1 )] ) + '/32'

            #m = re.match('(ETHER_KAP|ETHER_PORT|IP-KAP|IP-PORT|IP-SIPNET|IP-SNIX|IPSUR|L2L|RED-IPPORT|SNIX|SWIP|[email protected]|T2V-DIGTV|T2V-SUR)[0-9]{4,}', tp['order_id'])
            m = re.match('.*[0-9]{6}$', tp['order_id'])
            if m is not None or tp['type'] == 'CUSTOMER':
                print "Customer link", tp['prefix'], ':', tp['description']
                p = Prefix()
                p.schema = self.schema
                p.prefix = tp['prefix']
                p.type = 'assignment'
                p.description = tp['description']
                p.alarm_priority = tp['alarm_priority']
                p.authoritative_source = 'nw'
                if tp['order_id'] != '.':
                    p.order_id = tp['order_id']
                p.save({})

                # insert node1 and node2
                p1 = Prefix()
                p1.schema = self.schema
                p1.prefix = prefix_node1
                p1.type = 'host'
                p1.description = 'Some PE router'
                p1.authoritative_source = 'nw'
                p1.save({})

                p2 = Prefix()
                p2.schema = self.schema
                p2.prefix = prefix_node2
                p2.type = 'host'
                p2.node = tp['node']
                p2.description = 'CPE'
                p2.authoritative_source = 'nw'
                p2.save({})

                return


            m = re.match(r'([^\s]+)\s*<->\s*([^\s]+)', tp['description'])
            if m is not None:
                node1 = m.group(1)
                node2 = m.group(2)
                print "Link network: ", tp['prefix'], "  ", node1, "<->", node2

                p = Prefix()
                p.schema = self.schema
                p.prefix = tp['prefix']
                p.type = 'assignment'
                p.description = node1 + ' <-> ' + node2
                p.monitor = True
                p.alarm_priority = tp['alarm_priority']
                p.authoritative_source = 'nw'
                p.save({})

                # insert node1 and node2
                p1 = Prefix()
                p1.schema = self.schema
                p1.prefix = prefix_node1
                p1.type = 'host'
                p1.node = node1
                p1.description = node1
                p1.authoritative_source = 'nw'
                p1.save({})

                p2 = Prefix()
                p2.schema = self.schema
                p2.prefix = prefix_node2
                p2.type = 'host'
                p2.node = node2
                p2.description = node2
                p2.authoritative_source = 'nw'
                p2.save({})

                return

            m = re.match('(DN)[0-9]{4,}', tp['order_id'])
            if m is not None:
                print "Internal order link network", tp['prefix'], ':', tp['description']
                p = Prefix()
                p.schema = self.schema
                p.prefix = tp['prefix']
                p.type = 'assignment'
                p.description = tp['description']
                p.monitor = True
                p.alarm_priority = tp['alarm_priority']
                p.authoritative_source = 'nw'
                p.save({})
                return

            print "Other link network", tp['prefix'], ':', tp['description']
            p = Prefix()
            p.schema = self.schema
            p.prefix = tp['prefix']
            p.type = 'assignment'
            p.description = tp['description']
            p.monitor = True
            p.alarm_priority = tp['alarm_priority']
            p.authoritative_source = 'nw'
            p.save({})
            return

        else:
            try:
                p = Prefix()
                p.schema = self.schema
                p.prefix = tp['prefix']
                p.type = 'assignment'
                p.description = tp['description']
                p.monitor = True
                p.alarm_priority = 'low'
                p.authoritative_source = 'nw'
                p.save({})
                print "Other:", tp['prefix']
            except NipapValueError, e:
                print tp['prefix'], ':', e
                sys.exit(1)

            return

Example 10

Project: timestring
Source File: Date.py
View license
    def __init__(self, date, offset=None, start_of_week=None, tz=None, verbose=False):
        if isinstance(date, Date):
            self.date = copy(date.date)
            return

        # The original request
        self._original = date
        if tz:
            tz = pytz.timezone(str(tz))

        if date == 'infinity':
            self.date = 'infinity'

        elif date == 'now':
            self.date = datetime.now()

        elif type(date) in (str, unicode) and re.match(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+-\d{2}", date):
            self.date = datetime.strptime(date[:-3], "%Y-%m-%d %H:%M:%S.%f") - timedelta(hours=int(date[-3:]))

        else:
            # Determinal starting date.
            if type(date) in (str, unicode):
                """The date is a string and needs to be converted into a <dict> for processesing
                """
                _date = date.lower()
                res = TIMESTRING_RE.search(_date.strip())
                if res:
                    date = res.groupdict()
                    if verbose:
                        print("Matches:\n", ''.join(["\t%s: %s\n" % (k, v) for k, v in date.items() if v]))
                else:
                    raise TimestringInvalid('Invalid date string >> %s' % date)

                date = dict((k, v if type(v) is str else v) for k, v in date.items() if v)
                #print(_date, dict(map(lambda a: (a, date.get(a)), filter(lambda a: date.get(a), date))))

            if isinstance(date, dict):
                # Initial date.
                new_date = datetime(*time.localtime()[:3])
                if tz and tz.zone != "UTC":
                    #
                    # The purpose here is to adjust what day it is based on the timezeone
                    #
                    ts = datetime.now()
                    # Daylight savings === second Sunday in March and reverts to standard time on the first Sunday in November
                    # Monday is 0 and Sunday is 6.
                    # 14 days - dst_start.weekday()
                    dst_start = datetime(ts.year, 3, 1, 2, 0, 0) + timedelta(13 - datetime(ts.year, 3, 1).weekday())
                    dst_end = datetime(ts.year, 11, 1, 2, 0, 0) + timedelta(6 - datetime(ts.year, 11, 1).weekday())

                    ts = ts + tz.utcoffset(new_date, is_dst=(dst_start < ts < dst_end))
                    new_date = datetime(ts.year, ts.month, ts.day)

                if date.get('unixtime'):
                    new_date = datetime.fromtimestamp(int(date.get('unixtime')))

                # !number of (days|...) (ago)?
                elif date.get('num') and (date.get('delta') or date.get('delta_2')):
                    if date.get('num', '').find('couple') > -1:
                        i = 2 * int(1 if date.get('ago', True) or date.get('ref') == 'last' else -1)
                    else:
                        i = int(text2num(date.get('num', 'one'))) * int(1 if date.get('ago') or (date.get('ref', '') or '') == 'last' else -1)

                    delta = (date.get('delta') or date.get('delta_2')).lower()
                    if delta.startswith('y'):
                        try:
                            new_date = new_date.replace(year=(new_date.year - i))
                        # day is out of range for month
                        except ValueError:
                            new_date = new_date - timedelta(days=(365*i))
                    elif delta.startswith('month'):
                        try:
                            new_date = new_date.replace(month=(new_date.month - i))
                        # day is out of range for month
                        except ValueError:
                            new_date = new_date - timedelta(days=(30*i))

                    elif delta.startswith('q'):
                        '''
                        This section is not working...
                        Most likely need a generator that will take me to the right quater.
                        '''
                        q1, q2, q3, q4 = datetime(new_date.year, 1, 1), datetime(new_date.year, 4, 1), datetime(new_date.year, 7, 1), datetime(new_date.year, 10, 1)
                        if q1 <= new_date < q2:
                            # We are in Q1
                            if i == -1:
                                new_date = datetime(new_date.year-1, 10, 1)
                            else:
                                new_date = q2
                        elif q2 <= new_date < q3:
                            # We are in Q2
                            pass
                        elif q3 <= new_date < q4:
                            # We are in Q3
                            pass
                        else:
                            # We are in Q4
                            pass
                        new_date = new_date - timedelta(days=(91*i))

                    elif delta.startswith('w'):
                        new_date = new_date - timedelta(days=(i * 7))

                    else:
                        new_date = new_date - timedelta(**{('days' if delta.startswith('d') else 'hours' if delta.startswith('h') else 'minutes' if delta.startswith('m') else 'seconds'): i})

                # !dow
                if [date.get(key) for key in ('day', 'day_2', 'day_3') if date.get(key)]:
                    dow = max([date.get(key) for key in ('day', 'day_2', 'day_3') if date.get(key)])
                    iso = dict(monday=1, tuesday=2, wednesday=3, thursday=4, friday=5, saturday=6, sunday=7, mon=1, tue=2, tues=2, wed=3, wedn=3, thu=4, thur=4, fri=5, sat=6, sun=7).get(dow)
                    if iso:
                        # determin which direction
                        if date.get('ref') not in ('this', 'next'):
                            days = iso - new_date.isoweekday() - (7 if iso >= new_date.isoweekday() else 0)
                        else:
                            days = iso - new_date.isoweekday() + (7 if iso < new_date.isoweekday() else 0)

                        new_date = new_date + timedelta(days=days)

                    elif dow == 'yesterday':
                        new_date = new_date - timedelta(days=1)
                    elif dow == 'tomorrow':
                        new_date = new_date + timedelta(days=1)

                # !year
                year = [int(CLEAN_NUMBER.sub('', date[key])) for key in ('year', 'year_2', 'year_3', 'year_4', 'year_5', 'year_6') if date.get(key)]
                if year:
                    year = max(year)
                    if len(str(year)) != 4:
                        year += 2000 if year <= 40 else 1900
                    new_date = new_date.replace(year=year)

                # !month
                month = [date.get(key) for key in ('month', 'month_1', 'month_2', 'month_3', 'month_4') if date.get(key)]
                if month:
                    new_date = new_date.replace(day=1)
                    new_date = new_date.replace(month=int(max(month)) if re.match('^\d+$', max(month)) else dict(january=1, february=2, march=3, april=4, june=6, july=7, august=8, september=9, october=10, november=11, december=12, jan=1, feb=2, mar=3, apr=4, may=5, jun=6, jul=7, aug=8, sep=9, sept=9, oct=10, nov=11, dec=12).get(max(month),  new_date.month))

                # !day
                day = [date.get(key) for key in ('date', 'date_2', 'date_3') if date.get(key)]
                if day:
                    new_date = new_date.replace(day=int(max(day)))

                # !daytime
                if date.get('daytime'):
                    if date['daytime'].find('this time') >= 1:
                        new_date = new_date.replace(hour=datetime(*time.localtime()[:5]).hour,
                                                    minute=datetime(*time.localtime()[:5]).minute)
                    else:
                        new_date = new_date.replace(hour=dict(morning=9, noon=12, afternoon=15, evening=18, night=21, nighttime=21, midnight=24).get(date.get('daytime'), 12))
                    # No offset because the hour was set.
                    offset = False

                # !hour
                hour = [date.get(key) for key in ('hour', 'hour_2', 'hour_3') if date.get(key)]
                if hour:
                    new_date = new_date.replace(hour=int(max(hour)))
                    am = [date.get(key) for key in ('am', 'am_1') if date.get(key)]
                    if am and max(am) in ('p', 'pm'):
                        h = int(max(hour))
                        if h < 12:
                            new_date = new_date.replace(hour=h+12)
                    # No offset because the hour was set.
                    offset = False

                    #minute
                    minute = [date.get(key) for key in ('minute', 'minute_2') if date.get(key)]
                    if minute:
                        new_date = new_date.replace(minute=int(max(minute)))

                    #second
                    seconds = date.get('seconds', 0)
                    if seconds:
                        new_date = new_date.replace(second=int(seconds))

                self.date = new_date

            elif type(date) in (int, long, float) and re.match('^\d{10}$', str(date)):
                self.date = datetime.fromtimestamp(int(date))

            elif isinstance(date, datetime):
                self.date = date

            elif date is None:
                self.date = datetime.now()

            else:
                # Set to the current date Y, M, D, H0, M0, S0
                self.date = datetime(*time.localtime()[:3])

            if tz:
                self.date = self.date.replace(tzinfo=tz)

            # end if type(date) is types.DictType: and self.date.hour == 0:
            if offset and isinstance(offset, dict):
                self.date = self.date.replace(**offset)

Example 11

Project: pyxdg
Source File: DesktopEntry.py
View license
    def checkKey(self, key, value, group):
        # standard keys     
        if key == "Type":
            if value == "ServiceType" or value == "Service" or value == "FSDevice":
                self.warnings.append("Type=%s is a KDE extension" % key)
            elif value == "MimeType":
                self.warnings.append("Type=MimeType is deprecated")
            elif not (value == "Application" or value == "Link" or value == "Directory"):
                self.errors.append("Value of key 'Type' must be Application, Link or Directory, but is '%s'" % value)

            if self.fileExtension == ".directory" and not value == "Directory":
                self.warnings.append("File extension is .directory, but Type is '%s'" % value)
            elif self.fileExtension == ".desktop" and value == "Directory":
                self.warnings.append("Files with Type=Directory should have the extension .directory")

            if value == "Application":
                if "Exec" not in self.content[group]:
                    self.warnings.append("Type=Application needs 'Exec' key")
            if value == "Link":
                if "URL" not in self.content[group]:
                    self.warnings.append("Type=Link needs 'URL' key")

        elif key == "Version":
            self.checkValue(key, value)

        elif re.match("^Name"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif re.match("^GenericName"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif key == "NoDisplay":
            self.checkValue(key, value, type="boolean")

        elif re.match("^Comment"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif re.match("^Icon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)

        elif key == "Hidden":
            self.checkValue(key, value, type="boolean")

        elif key == "OnlyShowIn":
            self.checkValue(key, value, list=True)
            self.checkOnlyShowIn(value)

        elif key == "NotShowIn":
            self.checkValue(key, value, list=True)
            self.checkOnlyShowIn(value)

        elif key == "TryExec":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Exec":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Path":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Terminal":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "Application")
        
        elif key == "Actions":
            self.checkValue(key, value, list=True)
            self.checkType(key, "Application")

        elif key == "MimeType":
            self.checkValue(key, value, list=True)
            self.checkType(key, "Application")

        elif key == "Categories":
            self.checkValue(key, value)
            self.checkType(key, "Application")
            self.checkCategories(value)
        
        elif re.match("^Keywords"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value, type="localestring", list=True)
            self.checkType(key, "Application")

        elif key == "StartupNotify":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "Application")

        elif key == "StartupWMClass":
            self.checkType(key, "Application")

        elif key == "URL":
            self.checkValue(key, value)
            self.checkType(key, "URL")

        # kde extensions
        elif key == "ServiceTypes":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "DocPath":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "InitialPreference":
            self.checkValue(key, value, type="numeric")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "Dev":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "FSType":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "MountPoint":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "ReadOnly":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif re.match("^UnmountIcon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        # deprecated keys
        elif key == "Encoding":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif re.match("^MiniIcon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "TerminalOptions":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "DefaultApp":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "Protocols":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "Extensions":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "BinaryPattern":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "MapNotify":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif re.match("^SwallowTitle"+xdg.Locale.regex+"$", key):
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "SwallowExec":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "FilePattern":
            self.checkValue(key, value, type="regex", list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "SortOrder":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        # "X-" extensions
        elif re.match("^X-[a-zA-Z0-9-]+", key):
            pass

        else:
            self.errors.append("Invalid key: %s" % key)

Example 12

Project: ColorConvert
Source File: colorconvert.py
View license
    def run(self, edit):
        """Default function that runs when colorconvert is called.

        Attributes:
            self: The Regionset object.
            edit: The Edit object.

        """

        sels = self.view.sel()

        # Global values for the hex parts of rgb.
        global _hr
        global _hg
        global _hb

        # Global values for rgb channels.
        global _r
        global _g
        global _b

        # Global values for the hsl channels.
        global _h
        global _s
        global _l

        # Global values for the alpha channel.
        global _alpha
        global _alpha_enabled

        for sel in sels:

            # Get the selection and its length.
            str = self.view.substr(sel)
            str_len = len(str)

            # Define the regular expressions to test hex/rgb(a).
            reg_hex = '^[\#]?([\dabcdefABCDEF]{3}){1,2}$'
            reg_rgb = ('^rgb[a]?\((\s*\d+\s*),(\s*\d+\s*),(\s*\d+\s*),'
                       '?(\s*(0?.?\d)+\s*)?\)$')
            reg_hsl = ('^hsl[a]?\((\s*\d+.?\d?\s*),(\s*\d+.?\d?\%\s*),(\s*\d+.?\d?\%\s*),'
                       '?(\s*\d+.?\d?\s*)?\)$')

            hex_match = re.match(reg_hex, str)
            rgb_match = re.match(reg_rgb, str)
            hsl_match = re.match(reg_hsl, str)


            # If a hexadecimal number is found, convert it to an rgb value.
            if hex_match is not None:

                # Expand the selection if we're dealing with hex, but the
                # hashtag is not yet part of the selection.
                if (str_len == 3) or (str_len == 6):
                    start = sel.begin() - 1
                    end = sel.end()

                    # Update the selection.
                    self.view.sel().add(sublime.Region(start, end))
                    sel = sublime.Region(start, end)

                # If the hashtag is already part of the selection, make sure it
                # gets stripped.
                else:
                    str = str[1:]

                if (len(str) == 3):
                    r = str[0:1] * 2
                    g = str[1:2] * 2
                    b = str[2:3] * 2

                else:
                    r = str[0:2]
                    g = str[2:4]
                    b = str[4:6]

                # Store the hex channel values globally.
                if _hr is None:
                    _hr = r
                    _hg = g
                    _hb = b

                # If global rgb values are present, use those for speed and
                # accuracy. Otherwise convert the values.
                if _r is not None:

                    rgba_vals = [_r, _g, _b, _alpha]

                else:

                    rgba_vals = self.hexToRgb(_hr, _hg, _hb)

                # Write the output to the selection.
                if _alpha_enabled is True:

                    output = 'rgba(%d, %d, %d, %s)' % (rgba_vals[0],
                                                       rgba_vals[1],
                                                       rgba_vals[2],
                                                       rgba_vals[3])
                else:

                    output = 'rgba(%d, %d, %d)' % (rgba_vals[0], rgba_vals[1],
                                                   rgba_vals[2])

                self.view.replace(edit, sel, output)


            # If an rgb(a) value is found, convert it to an hsla value.
            elif rgb_match is not None:

                # Get all the color channels.
                r = int(rgb_match.group(1), 10)
                g = int(rgb_match.group(2), 10)
                b = int(rgb_match.group(3), 10)

                # Store the rgb channel values globally.
                if _r is None:
                    _r = r
                    _g = g
                    _b = b

                # If applicable, also get the alpha channel.
                if (rgb_match.group(4) is not None):
                    a = float(rgb_match.group(4))

                else:
                    a = _alpha

                # If global hsl values are present, use those for speed and
                # accuracy. Otherwise convert the values.
                if _h is not None:

                    hsla_vals = [_h, _s, _l, _alpha]

                else:

                    hsla_vals = self.rgbToHsl(r, g, b, a)

                # Write the output to the selection.
                if _alpha_enabled is True:

                    output = 'hsla(%.1f, %.1f%%, %.1f%%, %s)' % (hsla_vals[0],
                                                                 hsla_vals[1],
                                                                 hsla_vals[2],
                                                                 hsla_vals[3])

                else:

                    output = 'hsla(%.1f, %.1f%%, %.1f%%)' % (hsla_vals[0],
                                                             hsla_vals[1],
                                                             hsla_vals[2])

                self.view.replace(edit, sel, output)


            # If an hsl(a) value is found, convert it to a hex value.
            elif hsl_match is not None:

                # Get all the color channels.
                h = float(hsl_match.group(1).strip('%'))
                s = float(hsl_match.group(2).strip('%'))
                l = float(hsl_match.group(3).strip('%'))

                # Store the hsl channel values globally.
                if _h is None:
                    _h = h
                    _s = s
                    _l = l

                # If applicable, also get the alpha channel.
                if (hsl_match.group(4) is not None):
                    a = float(hsl_match.group(4))

                else:
                    a = _alpha

                # If global hex or rgb values are present, use those for speed
                # and accuracy. Otherwise convert the values.
                if _hr is not None:

                    hex_vals = [_hr, _hg, _hb, _alpha]

                else:

                    if _r is not None:

                        rgba_vals = [_r, _g, _b, _alpha]

                    else:

                        rgba_vals = self.hslToRgb(h, s, l, a)


                    hex_vals = self.rgbToHex(rgba_vals[0], rgba_vals[1],
                                             rgba_vals[2], rgba_vals[3])
                
                # Write the output to the selection.
                output = '#%s%s%s' % (hex_vals[0], hex_vals[1], hex_vals[2])
                self.view.replace(edit, sel, output)

Example 13

Project: youtube-dl-GUI
Source File: generic.py
View license
    def _real_extract(self, url):
        if url.startswith('//'):
            return {
                '_type': 'url',
                'url': self.http_scheme() + url,
            }

        parsed_url = compat_urlparse.urlparse(url)
        if not parsed_url.scheme:
            default_search = self._downloader.params.get('default_search')
            if default_search is None:
                default_search = 'fixup_error'

            if default_search in ('auto', 'auto_warning', 'fixup_error'):
                if '/' in url:
                    self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
                    return self.url_result('http://' + url)
                elif default_search != 'fixup_error':
                    if default_search == 'auto_warning':
                        if re.match(r'^(?:url|URL)$', url):
                            raise ExtractorError(
                                'Invalid URL:  %r . Call youtube-dl like this:  youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc"  ' % url,
                                expected=True)
                        else:
                            self._downloader.report_warning(
                                'Falling back to youtube search for  %s . Set --default-search "auto" to suppress this warning.' % url)
                    return self.url_result('ytsearch:' + url)

            if default_search in ('error', 'fixup_error'):
                raise ExtractorError(
                    '%r is not a valid URL. '
                    'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
                    % (url, url), expected=True)
            else:
                if ':' not in default_search:
                    default_search += ':'
                return self.url_result(default_search + url)

        url, smuggled_data = unsmuggle_url(url)
        force_videoid = None
        is_intentional = smuggled_data and smuggled_data.get('to_generic')
        if smuggled_data and 'force_videoid' in smuggled_data:
            force_videoid = smuggled_data['force_videoid']
            video_id = force_videoid
        else:
            video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]

        self.to_screen('%s: Requesting header' % video_id)

        head_req = HEADRequest(url)
        head_response = self._request_webpage(
            head_req, video_id,
            note=False, errnote='Could not send HEAD request to %s' % url,
            fatal=False)

        if head_response is not False:
            # Check for redirect
            new_url = head_response.geturl()
            if url != new_url:
                self.report_following_redirect(new_url)
                if force_videoid:
                    new_url = smuggle_url(
                        new_url, {'force_videoid': force_videoid})
                return self.url_result(new_url)

        full_response = None
        if head_response is False:
            full_response = self._request_webpage(url, video_id)
            head_response = full_response

        # Check for direct link to a video
        content_type = head_response.headers.get('Content-Type', '')
        m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
        if m:
            upload_date = unified_strdate(
                head_response.headers.get('Last-Modified'))
            return {
                'id': video_id,
                'title': os.path.splitext(url_basename(url))[0],
                'direct': True,
                'formats': [{
                    'format_id': m.group('format_id'),
                    'url': url,
                    'vcodec': 'none' if m.group('type') == 'audio' else None
                }],
                'upload_date': upload_date,
            }

        if not self._downloader.params.get('test', False) and not is_intentional:
            self._downloader.report_warning('Falling back on generic information extractor.')

        if not full_response:
            full_response = self._request_webpage(url, video_id)

        # Maybe it's a direct link to a video?
        # Be careful not to download the whole thing!
        first_bytes = full_response.read(512)
        if not re.match(r'^\s*<', first_bytes.decode('utf-8', 'replace')):
            self._downloader.report_warning(
                'URL could be a direct video link, returning it as such.')
            upload_date = unified_strdate(
                head_response.headers.get('Last-Modified'))
            return {
                'id': video_id,
                'title': os.path.splitext(url_basename(url))[0],
                'direct': True,
                'url': url,
                'upload_date': upload_date,
            }

        webpage = self._webpage_read_content(
            full_response, url, video_id, prefix=first_bytes)

        self.report_extraction(video_id)

        # Is it an RSS feed?
        try:
            doc = parse_xml(webpage)
            if doc.tag == 'rss':
                return self._extract_rss(url, video_id, doc)
        except compat_xml_parse_error:
            pass

        # Is it a Camtasia project?
        camtasia_res = self._extract_camtasia(url, video_id, webpage)
        if camtasia_res is not None:
            return camtasia_res

        # Sometimes embedded video player is hidden behind percent encoding
        # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
        # Unescaping the whole page allows to handle those cases in a generic way
        webpage = compat_urllib_parse.unquote(webpage)

        # it's tempting to parse this further, but you would
        # have to take into account all the variations like
        #   Video Title - Site Name
        #   Site Name | Video Title
        #   Video Title - Tagline | Site Name
        # and so on and so forth; it's just not practical
        video_title = self._html_search_regex(
            r'(?s)<title>(.*?)</title>', webpage, 'video title',
            default='video')

        # Try to detect age limit automatically
        age_limit = self._rta_search(webpage)
        # And then there are the jokers who advertise that they use RTA,
        # but actually don't.
        AGE_LIMIT_MARKERS = [
            r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
        ]
        if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
            age_limit = 18

        # video uploader is domain name
        video_uploader = self._search_regex(
            r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')

        # Helper method
        def _playlist_from_matches(matches, getter=None, ie=None):
            urlrs = orderedSet(
                self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
                for m in matches)
            return self.playlist_result(
                urlrs, playlist_id=video_id, playlist_title=video_title)

        # Look for BrightCove:
        bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
        if bc_urls:
            self.to_screen('Brightcove video detected.')
            entries = [{
                '_type': 'url',
                'url': smuggle_url(bc_url, {'Referer': url}),
                'ie_key': 'Brightcove'
            } for bc_url in bc_urls]

            return {
                '_type': 'playlist',
                'title': video_title,
                'id': video_id,
                'entries': entries,
            }

        # Look for embedded (iframe) Vimeo player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
        if mobj:
            player_url = unescapeHTML(mobj.group('url'))
            surl = smuggle_url(player_url, {'Referer': url})
            return self.url_result(surl)

        # Look for embedded (swf embed) Vimeo player
        mobj = re.search(
            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
        if mobj:
            return self.url_result(mobj.group(1))

        # Look for embedded YouTube player
        matches = re.findall(r'''(?x)
            (?:
                <iframe[^>]+?src=|
                data-video-url=|
                <embed[^>]+?src=|
                embedSWF\(?:\s*|
                new\s+SWFObject\(
            )
            (["\'])
                (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
                (?:embed|v|p)/.+?)
            \1''', webpage)
        if matches:
            return _playlist_from_matches(
                matches, lambda m: unescapeHTML(m[1]))

        # Look for lazyYT YouTube embed
        matches = re.findall(
            r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
        if matches:
            return _playlist_from_matches(matches, lambda m: unescapeHTML(m))

        # Look for embedded Dailymotion player
        matches = re.findall(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
        if matches:
            return _playlist_from_matches(
                matches, lambda m: unescapeHTML(m[1]))

        # Look for embedded Dailymotion playlist player (#3822)
        m = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
        if m:
            playlists = re.findall(
                r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
            if playlists:
                return _playlist_from_matches(
                    playlists, lambda p: '//dailymotion.com/playlist/%s' % p)

        # Look for embedded Wistia player
        match = re.search(
            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
        if match:
            embed_url = self._proto_relative_url(
                unescapeHTML(match.group('url')))
            return {
                '_type': 'url_transparent',
                'url': embed_url,
                'ie_key': 'Wistia',
                'uploader': video_uploader,
                'title': video_title,
                'id': video_id,
            }

        match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
        if match:
            return {
                '_type': 'url_transparent',
                'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
                'ie_key': 'Wistia',
                'uploader': video_uploader,
                'title': video_title,
                'id': match.group('id')
            }

        # Look for embedded blip.tv player
        mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
        if mobj:
            return self.url_result('http://blip.tv/a/a-' + mobj.group(1), 'BlipTV')
        mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
        if mobj:
            return self.url_result(mobj.group(1), 'BlipTV')

        # Look for embedded condenast player
        matches = re.findall(
            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
            webpage)
        if matches:
            return {
                '_type': 'playlist',
                'entries': [{
                    '_type': 'url',
                    'ie_key': 'CondeNast',
                    'url': ma,
                } for ma in matches],
                'title': video_title,
                'id': video_id,
            }

        # Look for Bandcamp pages with custom domain
        mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
        if mobj is not None:
            burl = unescapeHTML(mobj.group(1))
            # Don't set the extractor because it can be a track url or an album
            return self.url_result(burl)

        # Look for embedded Vevo player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for Ooyala videos
        mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
        if mobj is not None:
            return OoyalaIE._build_url_result(mobj.group('ec'))

        # Look for Aparat videos
        mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
        if mobj is not None:
            return self.url_result(mobj.group(1), 'Aparat')

        # Look for MPORA videos
        mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
        if mobj is not None:
            return self.url_result(mobj.group(1), 'Mpora')

        # Look for embedded NovaMov-based player
        mobj = re.search(
            r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
                    (?P<url>http://(?:(?:embed|www)\.)?
                        (?:novamov\.com|
                           nowvideo\.(?:ch|sx|eu|at|ag|co)|
                           videoweed\.(?:es|com)|
                           movshare\.(?:net|sx|ag)|
                           divxstage\.(?:eu|net|ch|co|at|ag))
                        /embed\.php.+?)\1''', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))

        # Look for embedded Facebook player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Facebook')

        # Look for embedded VK player
        mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'VK')

        # Look for embedded ivi player
        mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Ivi')

        # Look for embedded Huffington Post player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'HuffPost')

        # Look for embed.ly
        mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'))
        mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
        if mobj is not None:
            return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))

        # Look for funnyordie embed
        matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
        if matches:
            return _playlist_from_matches(
                matches, getter=unescapeHTML, ie='FunnyOrDie')

        # Look for BBC iPlayer embed
        matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
        if matches:
            return _playlist_from_matches(matches, ie='BBCCoUk')

        # Look for embedded RUTV player
        rutv_url = RUTVIE._extract_url(webpage)
        if rutv_url:
            return self.url_result(rutv_url, 'RUTV')

        # Look for embedded TED player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'TED')

        # Look for embedded Ustream videos
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Ustream')

        # Look for embedded arte.tv player
        mobj = re.search(
            r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'ArteTVEmbed')

        # Look for embedded smotri.com player
        smotri_url = SmotriIE._extract_url(webpage)
        if smotri_url:
            return self.url_result(smotri_url, 'Smotri')

        # Look for embeded soundcloud player
        mobj = re.search(
            r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
            webpage)
        if mobj is not None:
            url = unescapeHTML(mobj.group('url'))
            return self.url_result(url)

        # Look for embedded vulture.com player
        mobj = re.search(
            r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
            webpage)
        if mobj is not None:
            url = unescapeHTML(mobj.group('url'))
            return self.url_result(url, ie='Vulture')

        # Look for embedded mtvservices player
        mobj = re.search(
            r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
            webpage)
        if mobj is not None:
            url = unescapeHTML(mobj.group('url'))
            return self.url_result(url, ie='MTVServicesEmbedded')

        # Look for embedded yahoo player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Yahoo')

        # Look for embedded sbs.com.au player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'SBS')

        # Look for embedded Cinchcast player
        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Cinchcast')

        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'MLB')

        mobj = re.search(
            r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
            webpage)
        if mobj is not None:
            return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')

        mobj = re.search(
            r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
            webpage)
        if mobj is not None:
            return self.url_result(mobj.group('url'), 'Livestream')

        def check_video(vurl):
            vpath = compat_urlparse.urlparse(vurl).path
            vext = determine_ext(vpath)
            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')

        def filter_video(urls):
            return list(filter(check_video, urls))

        # Start with something easy: JW Player in SWFObject
        found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
        if not found:
            # Look for gorilla-vid style embedding
            found = filter_video(re.findall(r'''(?sx)
                (?:
                    jw_plugins|
                    JWPlayerOptions|
                    jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
                )
                .*?file\s*:\s*["\'](.*?)["\']''', webpage))
        if not found:
            # Broaden the search a little bit
            found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
        if not found:
            # Broaden the findall a little bit: JWPlayer JS loader
            found = filter_video(re.findall(
                r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
        if not found:
            # Flow player
            found = filter_video(re.findall(r'''(?xs)
                flowplayer\("[^"]+",\s*
                    \{[^}]+?\}\s*,
                    \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                        ["']?url["']?\s*:\s*["']([^"']+)["']
            ''', webpage))
        if not found:
            # Try to find twitter cards info
            found = filter_video(re.findall(
                r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
        if not found:
            # We look for Open Graph info:
            # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
            m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
            # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
            if m_video_type is not None:
                found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
        if not found:
            # HTML5 video
            found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
        if not found:
            found = re.search(
                r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'?([^\'"]+)',
                webpage)
            if found:
                new_url = found.group(1)
                self.report_following_redirect(new_url)
                return {
                    '_type': 'url',
                    'url': new_url,
                }
        if not found:
            raise UnsupportedError(url)

        entries = []
        for video_url in found:
            video_url = compat_urlparse.urljoin(url, video_url)
            video_id = compat_urllib_parse.unquote(os.path.basename(video_url))

            # Sometimes, jwplayer extraction will result in a YouTube URL
            if YoutubeIE.suitable(video_url):
                entries.append(self.url_result(video_url, 'Youtube'))
                continue

            # here's a fun little line of code for you:
            video_id = os.path.splitext(video_id)[0]

            entries.append({
                'id': video_id,
                'url': video_url,
                'uploader': video_uploader,
                'title': video_title,
                'age_limit': age_limit,
            })

        if len(entries) == 1:
            return entries[0]
        else:
            for num, e in enumerate(entries, start=1):
                e['title'] = '%s (%d)' % (e['title'], num)
            return {
                '_type': 'playlist',
                'entries': entries,
            }

Example 14

Project: zenv
Source File: autobuilder.py
View license
def process_change(change_type, path):
    """ Utility to sync vm when a file is changed. """
    if DEBUG:
        print 'Change type |%s| reported for path |%s|' % (change_type, path)

    if not re.match('modified|created|deleted', change_type):
        if DEBUG:
            print '    - skipped: change type |%s| not supported' % change_type
        return

    file_name = basename(path)
    if is_excluded_path(path, file_name):
        if DEBUG:
            print '    - skipped: path pattern |%s| is excluded' % path
        return

    # Travel up the directory structure until we find one
    # with a settings file in it
    #
    extra_dirs = []
    settings_dir = dirname(path)
    while not exists(settings_dir + '/' + BUILD_SETTINGS_FILE) and settings_dir != '/':
        extra_dirs.insert(0, basename(settings_dir) + '/')
        settings_dir = dirname(settings_dir)

    if DEBUG:
        print 'Directory with settings file is |%s|' % settings_dir

    if settings_dir == '/':
        # This is certainly not a workspace
        print 'Could not find a settings file appropriate for %s.' % path
        return

    # Get the kind of rules_to_run that we are looking for
    if change_type == 'modified':
        rules_to_run = {KEY_ALL, KEY_UPDATE, KEY_ADDUP, KEY_UPDEL}
    elif change_type == 'created':
        rules_to_run = {KEY_ALL, KEY_ADD, KEY_ADDUP, KEY_ADDDEL}
    elif change_type == 'deleted':
        rules_to_run = {KEY_ALL, KEY_DELETE, KEY_ADDDEL, KEY_UPDEL}
    else:
        print 'OH GOD AN UNHANDLED CHANGE TYPE'
        return

    # Get the contents of the settings file
    matched_rule = None
    commands = None
    envs = []

    with open(settings_dir + '/' + BUILD_SETTINGS_FILE) as file_ptr:
        while matched_rule is None:
            line = file_ptr.readline()

            # Read the variable settings
            if line.startswith(KEY_SET):
                env_name, env_value = (
                    re.search('%s ([^=]+)=(.*)' % KEY_SET, line).groups()
                )

                envs.append('%s=%s' % (env_name, env_value))
                if DEBUG:
                    print ('Found variable declaration |%s| = |%s|' % (env_name, env_value))
                continue

            elif line.startswith(KEY_COMMENT):
                # This is a comment (wow, so meta!)
                if DEBUG:
                    print 'Found comment |%s|' % line
                continue

            elif re.match(r'^\S', line):
                # This does not start with whitespace, so it must be a rule
                if DEBUG:
                    print 'Found rule |%s|' % line,
                rule = line.rstrip("\n")
                if not re.search(rule, file_name):
                    if DEBUG:
                        print ('Rule |%s| does not match file |%s|' % (rule, file_name))
                    continue

                matched_rule = rule

                # Find out what kind of rule this is
                key_line = file_ptr.readline()
                indent = re.findall(r'^(\s+)', key_line)[0]

                while re.match('^' + indent, key_line):
                    key_line = key_line.strip()
                    if DEBUG:
                        print 'Found key |%s| for rule |%s|' % (key_line, rule)
                    if key_line == "\n":
                        break
                    elif key_line not in rules_to_run:
                        if DEBUG:
                            print (
                                'Rule |{sr}| does not apply to ' +
                                'change |{sc}| on key |{sk}|'.format(
                                    sr='' if not rule else rule,
                                    sk='' if not key_line else key_line,
                                    sc='' if not change_type else change_type
                                )
                            )

                        # Skip the contents of this rule
                        key_line = file_ptr.readline()
                        while re.match('^' + indent + indent, key_line):
                            key_line = file_ptr.readline()
                        continue

                    # last postion is file_ptr.tell()
                    rule_line = file_ptr.readline()
                    rule_contents = []
                    while re.match('^' + indent + indent, rule_line) and rule_line != "\n":
                        rule_contents.append(rule_line.strip())
                        # last postion is file_ptr.tell()
                        rule_line = file_ptr.readline()

                    # Combine the given commands into one line using
                    # so that one failure makes everything stop
                    #
                    commands = ' && '.join(rule_contents)
                    if DEBUG:
                        print "Rule contents are:\n%s" % commands

                continue

            elif line == '' or not line:
                if DEBUG:
                    print 'Done parsing file.'
                break

        if matched_rule is None:
            print 'No matching rule found for file |%s|.' % file_name
            return
        elif DEBUG:
            print ('Matched rule |%s| for file |%s|' % (matched_rule, file_name))

        # Check the changed file_name against each of the regexes from
        # the settings file
        match = re.search(matched_rule, file_name)

        # Create a dict that maps the group numbers to their matched values
        groups = match.groups()
        match_groups = {str(i + 1): groups[i] for i in xrange(len(groups))}

        # Replace autobuild variables $0, $1, $2, etc with
        # Python-esqe $(0)s, $(1)s, etc and use that to pass matched groups
        match_groups['0'] = relpath(path, settings_dir)
        commands = re.sub(r'\$\{([0-9]+)\}', '%(\\1)s',
            re.sub(r'\$([0-9]+)', '%(\\1)s', commands)) % match_groups

        # Add the inherited ZEnv variables
        inherited_envs = '; '.join('%s=%s' %
            (i, quote(os.environ[i]))
            for i in os.environ if i.startswith('ZENV_')
        )

        # Add variable declarations to the beginning of the commands
        var_decls = '; '.join(envs)
        full_program = '; '.join([inherited_envs, var_decls, commands])

        # Run the command
        print 'Running build for path |%s|' % path
        if DEBUG:
            print full_program
        else:
            print commands
        os.system(('cd %s; ' % settings_dir) + full_program)
        print "Build complete.\n"

Example 15

Project: pcbmode
Source File: svgpath.py
View license
    def _getDimensions(self, path):
        """
        """

        last_point = Point()
        abs_point = Point()
     
        bbox_top_left = Point()
        bbox_bot_right = Point()
     
        # for the t/T (shorthand bezier) command, we need to keep track
        # of the last bezier control point from previous Q/q/T/t command
        last_bezier_control_point = Point()
     
        for i in range(0, len(path)):
     
            # 'move to' command
            if re.match('m', path[i][0]):
     
                if i == 0:
                    # the first coordinate is the start of both top left and bottom right
                    abs_point.assign(path[i][1][0], path[i][1][1])
                    bbox_top_left.assign(path[i][1][0], path[i][1][1])
                    bbox_bot_right.assign(path[i][1][0], path[i][1][1])
                else:
                    new_point = Point(path[i][1][0], path[i][1][1])
                    abs_point += new_point
                    bbox_top_left, bbox_bot_right = svg.boundary_box_check(bbox_top_left, 
                                                                       bbox_bot_right, 
                                                                       abs_point)
                    
                # for the rest of the coordinates
                for coord in path[i][2:]:
                    new_point = Point(coord[0], coord[1])
                    abs_point += new_point
                    bbox_top_left, bbox_bot_right = svg.boundary_box_check(bbox_top_left, 
                                                                       bbox_bot_right, 
                                                                       abs_point)
      
            # cubic Bezier curve command 
            elif re.match('c', path[i][0]):
     
                bezier_curve_path = []
                
                for n in range(1, len(path[i])-1, 3):
                    bezier_curve_path.append(abs_point)
                    for m in range(0, 3):
                        coord = path[i][n+m]
                        point = Point(coord[0], coord[1])
                        bezier_curve_path.append(abs_point + point)
                    new_point = Point(path[i][n+m][0], path[i][n+m][1])
                    abs_point += new_point 
     
          
                for n in range(0, len(bezier_curve_path), 4):
     
                    # clear bezier point arrays 
                    bezier_points_x = []
                    bezier_points_y = []
     
                    # split points of bezier into 'x' and 'y' coordinate arrays
                    # as this is what the point array function expects
                    for m in range(0, 4):
                        bezier_points_x.append(bezier_curve_path[n+m].x)
                        bezier_points_y.append(bezier_curve_path[n+m].y)
     
                    # caluclate the individual points along the bezier curve for 'x'
                    # and 'y'
                    points_x = svg.calculate_points_of_cubic_bezier(bezier_points_x, 100)
                    points_y = svg.calculate_points_of_cubic_bezier(bezier_points_y, 100)
     
                    bezier_point_array = []
     
                    # put those points back into a Point type array
                    for n in range(0, len(points_x)):
                        bezier_point_array.append(Point(points_x[n], points_y[n]))
     
                    # check each point if it extends the boundary box
                    for n in range(0, len(bezier_point_array)):
                        bbox_top_left, bbox_bot_right = svg.boundary_box_check(
                            bbox_top_left, 
                            bbox_bot_right, 
                            bezier_point_array[n])
     
     
            # quadratic Bezier curve command 
            elif re.match('q', path[i][0]):
     
                bezier_curve_path = []
                
                for n in range(1, len(path[i])-1, 2):
                    bezier_curve_path.append(abs_point)
                    for m in range(0, 2):
                        coord = path[i][n+m]
                        point = Point(coord[0], coord[1])
                        bezier_curve_path.append(abs_point + point)
                        # inject a second, identical control point so this quadratic
                        # bezier looks like a cubic one
                        if m == 1:
                            bezier_curve_path.append(abs_point + point)
                        if m == 0:
                            last_bezier_control_point = abs_point + point
                    new_point = Point(path[i][n+m][0], path[i][n+m][1])
                    abs_point += new_point   
     
          
                for n in range(0, len(bezier_curve_path), 4):
     
                    # clear bezier point arrays 
                    bezier_points_x = []
                    bezier_points_y = []
     
                    # split points of bezier into 'x' and 'y' coordinate arrays
                    # as this is what the point array function expects
                    for m in range(0, 4):
                        bezier_points_x.append(bezier_curve_path[n+m].x)
                        bezier_points_y.append(bezier_curve_path[n+m].y)
     
                    # caluclate the individual points along the bezier curve for 'x'
                    # and 'y'
                    points_x = svg.calculate_points_of_cubic_bezier(bezier_points_x, 100)
                    points_y = svg.calculate_points_of_cubic_bezier(bezier_points_y, 100)
     
                    bezier_point_array = []
     
                    # put those points back into a Point type array
                    for n in range(0, len(points_x)):
                        bezier_point_array.append(Point(points_x[n], points_y[n]))
     
                    # check each point if it extends the boundary box
                    for n in range(0, len(bezier_point_array)):
                        bbox_top_left, bbox_bot_right = svg.boundary_box_check(
                                bbox_top_left, 
                                bbox_bot_right, 
                                bezier_point_array[n])
     
     
            # simple cubic Bezier curve command 
            elif re.match('t', path[i][0]):
                bezier_curve_path = []
     
                for n in range(1, len(path[i])):
                    bezier_curve_path.append(abs_point)
                    coord = path[i][n]
                    point = Point(coord[0], coord[1])
                    end_point = abs_point + point
                    diff = Point(abs_point.x - last_bezier_control_point.x, 
                                 abs_point.y - last_bezier_control_point.y)
                    control_point = abs_point + diff
                    bezier_curve_path.append(control_point)
                    bezier_curve_path.append(end_point)
                    bezier_curve_path.append(end_point)
                    last_bezier_control_point = control_point
                    new_point = Point(path[i][n][0], path[i][n][1])
                    abs_point += new_point
     
                    
                for n in range(0, len(bezier_curve_path), 4):
     
                    # clear bezier point arrays 
                    bezier_points_x = []
                    bezier_points_y = []
     
                    # split points of bezier into 'x' and 'y' coordinate arrays
                    # as this is what the point array function expects
                    for m in range(0, 4):
                        bezier_points_x.append(bezier_curve_path[n+m].x)
                        bezier_points_y.append(bezier_curve_path[n+m].y)
     
                    # caluclate the individual points along the bezier curve for 'x'
                    # and 'y'
                    points_x = svg.calculate_points_of_cubic_bezier(bezier_points_x, 100)
                    points_y = svg.calculate_points_of_cubic_bezier(bezier_points_y, 100)
     
                    bezier_point_array = []
     
                    # put those points back into a Point type array
                    for n in range(0, len(points_x)):
                        bezier_point_array.append(Point(points_x[n], points_y[n]))
     
                    # check each point if it extends the boundary box
                    for m in range(0, len(bezier_point_array)):
                        bbox_top_left, bbox_bot_right = svg.boundary_box_check(
                                bbox_top_left, 
                                bbox_bot_right, 
                                bezier_point_array[m])
     
     
       
    #        elif re.match('S', path[i][0], re.I):
    #            pass
     
            # 'line to'  command
            elif re.match('l', path[i][0]):
                for coord in path[i][1:]:
                    new_point = Point(coord[0], coord[1])
                    abs_point += new_point
                    bbox_top_left, bbox_bot_right = svg.boundary_box_check(bbox_top_left, 
                                                                       bbox_bot_right, 
                                                                       abs_point)
                
            # 'horizontal line' command
            elif re.match('h', path[i][0]):
                for coord in path[i][1:]:
                    new_point = Point(coord[0], 0)
                    abs_point += new_point
                    bbox_top_left, bbox_bot_right = svg.boundary_box_check(bbox_top_left, 
                                                                       bbox_bot_right, 
                                                                       abs_point)
     
            # 'vertical line' command
            elif re.match('v', path[i][0]):
                for coord in path[i][1:]:
                    new_point = Point(0, coord[0])
                    abs_point += new_point
                    bbox_top_left, bbox_bot_right = svg.boundary_box_check(bbox_top_left, 
                                                                       bbox_bot_right, 
                                                                       abs_point)
     
            # 'close shape' command
            elif re.match('Z', path[i][0], re.I):
                pass
     
            else:
                print("ERROR: found an unsupported SVG path command " + str(path[i][0]))

        self._bbox_top_left = bbox_top_left
        self._bbox_bot_right = bbox_bot_right        
        #self._width = (bbox_bot_right.x - bbox_top_left.x)
        #self._height = abs(bbox_bot_right.y - bbox_top_left.y)
        return (bbox_bot_right.x - bbox_top_left.x), abs(bbox_bot_right.y - bbox_top_left.y)

Example 16

Project: pcbmode
Source File: svgpath.py
View license
    def _makeCoordList(self, path, steps, length):
        """
        """
     
        # absolute position
        ap = Point()
     
        # path origin
        po = Point()
        
        points = []
        p = []

        # TODO: legacy
        pd = path
     
        last_bezier_control_point = Point()
     
        for i in range(0, len(pd)):
     
            cmd = pd[i][0]
     
            # 'move to' command
            if re.match('m', cmd):
                if i == 0:
                    coord = Point(pd[i][1][0], pd[i][1][1])
                    ap.assign(coord.x, coord.y)
                    p.append(ap)
                    po.assign(coord.x, coord.y)
                else:
                    coord_tmp = Point(pd[i][1][0], pd[i][1][1])
                    ap += coord_tmp
                    # a marker that a new path is starting after a previous one closed
                    points.append(p)
                    p = []
                    p.append(ap)
                    po = ap
                    
                for coord_tmp in pd[i][2:]:
                    coord = Point(coord_tmp[0], coord_tmp[1])
                    ap += coord
                    p.append(ap)
     
            # cubic (two control points) Bezier curve command 
            elif re.match('c', cmd):
     
                bezier_curve_path = []
                
                for n in range(1, len(pd[i])-1, 3):
                    bezier_curve_path.append(ap)
                    for m in range(0, 3):
                        coord = pd[i][n+m]
                        point = Point(coord[0], coord[1])
                        bezier_curve_path.append(ap + point)
                    new_point = Point(pd[i][n+m][0], pd[i][n+m][1])
                    ap += new_point 
     
          
                for n in range(0, len(bezier_curve_path), 4):
     
                    # clear bezier point arrays 
                    bezier_points_x = []
                    bezier_points_y = []
     
                    # split points of bezier into 'x' and 'y' coordinate arrays
                    # as this is what the point array function expects
                    for m in range(0, 4):
                        bezier_points_x.append(bezier_curve_path[n+m].x)
                        bezier_points_y.append(bezier_curve_path[n+m].y)
     
     
                    # caluclate the individual points along the bezier curve for 'x'
                    # and 'y'
                    points_x = self._linearizeCubicBezier(bezier_points_x, steps)
                    points_y = self._linearizeCubicBezier(bezier_points_y, steps)

                    path_length = self._getCubicBezierLength(points_x, points_y)

                    if path_length == 0:
                        steps_tmp = 1
                    else:
                        steps_tmp = ceil(path_length / length)
                    skip = int(ceil(steps / steps_tmp))
     
                    bezier_point_array = []
     
                    # put thos points back into a Point type array
                    for n in range(0, len(points_x), skip):
                        bezier_point_array.append(Point(points_x[n], points_y[n]))
                    bezier_point_array.append(Point(points_x[len(points_x)-1], points_y[len(points_x)-1]))
     
                    p += bezier_point_array
     
     
            # quadratic (single control point) Bezier curve command 
            elif re.match('q', cmd):
     
                bezier_curve_path = []
                
                for n in range(1, len(pd[i])-1, 2):
                    bezier_curve_path.append(ap)
                    for m in range(0, 2):
                        coord = pd[i][n+m]
                        point = Point(coord[0], coord[1])
                        bezier_curve_path.append(ap + point)
                        # inject a second, identical control point so this quadratic
                        # bezier looks like a cubic one
                        if m == 1:
                            bezier_curve_path.append(ap+point)
                        if m == 0:
                            last_bezier_control_point = ap + point
                    new_point = Point(pd[i][n+m][0], pd[i][n+m][1])
                    ap += new_point   
     
     
                for n in range(0, len(bezier_curve_path), 4):
     
                    # clear bezier point arrays 
                    bezier_points_x = []
                    bezier_points_y = []
     
                    # split points of bezier into 'x' and 'y' coordinate arrays
                    # as this is what the point array function expects
                    for m in range(0, 4):
                        bezier_points_x.append(bezier_curve_path[n+m].x)
                        bezier_points_y.append(bezier_curve_path[n+m].y)
     
     
                    # caluclate the individual points along the bezier curve for 'x'
                    # and 'y'
                    points_x = self._linearizeCubicBezier(bezier_points_x, steps)
                    points_y = self._linearizeCubicBezier(bezier_points_y, steps)

                    path_length = self._getCubicBezierLength(points_x, points_y)
                    skip = int(ceil(steps / (path_length / length)))
     
                    bezier_point_array = []
     
                    # put those points back into a Point type array
                    for n in range(0, len(points_x), skip):
                        bezier_point_array.append(Point(points_x[n], points_y[n]))            
                    bezier_point_array.append(Point(points_x[len(points_x)-1], points_y[len(points_x)-1]))
     
                    p += bezier_point_array
     
     
     
            # simple cubic Bezier curve command 
            elif re.match('t', cmd):
     
                bezier_curve_path = []
     
                for n in range(1, len(pd[i])):
                    bezier_curve_path.append(ap)
                    coord = pd[i][n]
                    point = Point(coord[0], coord[1])
                    end_point = ap + point
                    diff = Point(ap.x - last_bezier_control_point.x, ap.y - last_bezier_control_point.y)
                    control_point = ap + diff
                    bezier_curve_path.append(control_point)
                    bezier_curve_path.append(end_point)
                    bezier_curve_path.append(end_point)
                    last_bezier_control_point = control_point
                    new_point = Point(pd[i][n][0], pd[i][n][1])
                    ap += new_point
     
                for n in range(0, len(bezier_curve_path), 4):
     
                    # clear bezier point arrays 
                    bezier_points_x = []
                    bezier_points_y = []
     
                    # split points of bezier into 'x' and 'y' coordinate arrays
                    # as this is what the point array function expects
                    for m in range(0, 4):
                        bezier_points_x.append(bezier_curve_path[n+m].x)
                        bezier_points_y.append(bezier_curve_path[n+m].y)
     
                    # caluclate the individual points along the bezier curve for 'x'
                    # and 'y'
                    points_x = self._linearizeCubicBezier(bezier_points_x, steps)
                    points_y = self._linearizeCubicBezier(bezier_points_y, steps)

                    path_length = self._getCubicBezierLength(points_x, points_y)
                    skip = int(ceil(steps / (path_length / length)))
     
                    bezier_point_array = []
     
                    # put those points back into a Point type array
                    for m in range(0, len(points_x), skip):
                        bezier_point_array.append(Point(points_x[m], points_y[m]))
                    bezier_point_array.append(Point(points_x[len(points_x)-1], points_y[len(points_x)-1]))
     
                    p += bezier_point_array
     
     
    #        elif re.match('s', cmd):
    #            pass
     
            # 'line to'  command
            elif re.match('l', cmd):
                for coord_tmp in pd[i][1:]:
                    coord = Point(coord_tmp[0], coord_tmp[1])
                    ap += coord
                    p.append(ap)
     
            # 'horizontal line' command
            elif re.match('h', cmd):
                for coord_tmp in pd[i][1:]:
                    coord = Point(coord_tmp[0], 0)
                    ap += coord
                    p.append(ap)            
     
            # 'vertical line' command
            elif re.match('v', cmd):
                for coord_tmp in pd[i][1:]:
                    coord = Point(0, coord_tmp[0])
                    ap += coord
                    p.append(ap)
     
            # 'close shape' command
            elif re.match('z', cmd):
                ap = ap + (po - ap)
     
     
            else:
                msg.error("Found an unsupported SVG path command, '%s'" % cmd)
     
     
        points.append(p)

        return points

Example 17

Project: veyepar
Source File: tsraw.py
View license
def get_start( pathname, time_source ):

    """
    get the start of this clip
    dv.filename generally looks like this: 2012-01-14/10:01:34.dv
    parse the dir and filename strings.
    or use the filesystem_create 
    """
    
    # 3 ways of getting the datetime this file started

    def fs_time(pathname):
        # get timestamp from filesystem
        st = os.stat(pathname)    
        ts_start=datetime.datetime.fromtimestamp( st.st_mtime )
        return ts_start

    def parse_name(pathname):
        # print("parse_name")
        # parse string into datetime
        # expects room/yyy-mm-dd/hh_mm_ss-x.ext

        # remove extention
        filename = os.path.splitext(pathname)[0]

        # dvswitch appends -n in the event of filename colisions. 
        # for start, dupe time is fine, so drop the -n for parsing 
        if filename[-2]=='-': filename = filename[:-2] 

        # for now, the last dir is the date, and the file is time:
        filename='/'.join(filename.split('/')[-2:])

        # swap : for _ (so either : or _ can be used in the filename)
        filename = filename.replace(':','_')

        # parse
        start=datetime.datetime.strptime(filename,'%Y-%m-%d/%H_%M_%S')
        return start

    def re_name(pathname):
        # print("re_name...")
        # parse string into datetime useing RE

        dt_re = r".*/(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+).*/(?P<hour>\d+)_(?P<minute>\d+)_(?P<second>\d+)"


        dt_o = re.match(dt_re, pathname)
        dt_parts = dt_o.groupdict()
        print(dt_parts)

        dt_parts = {k:int(v) for k,v in list(dt_parts.items())}
        print(dt_parts)

        start=datetime.datetime( **dt_parts )
        print(start)

        return start

    def PyExifTool(pathname):
        # use PyExifTool to read Time Code
        """
        >>> metadata['H264:TimeCode']
        u'03:25:30:12'
         u'H264:DateTimeOriginal': u'2015:11:03 10:22:23+00:00',
        """

        with exiftool.ExifTool() as et:
            metadata = et.get_metadata(pathname)
        dt = metadata['H264:DateTimeOriginal']
        
        start=datetime.datetime.strptime(dt,'%Y:%m:%d %H:%M:%S+00:00')
        # print(start)
        return start

    def frame_time(pathname):
        # get timestamp from first frame
        pass

    def gst_discover_start(pathname):
        # get start time using gstreamer to read the media file header
        # print("gst_discover_start")

        """
     If you add an 'audioparse' element (or 'rawaudioparse' in >= 1.9/git
     master) after filesrc and configure it with the right properties, it
     should be able to report the duration correctly.
        """

        discoverer = GstPbutils.Discoverer()
        d = discoverer.discover_uri('file://{}'.format(pathname))
        tags= d.get_tags()
        dt=tags.get_date_time("datetime")[1]

        # import code; code.interact(local=locals())

        # print(dt.to_iso8601_string())
        start = datetime.datetime(
                year=dt.get_year(),
                month=dt.get_month(),
                day=dt.get_day(),
                hour=dt.get_hour(),
                minute=dt.get_minute(),
                second=dt.get_second(),
                )

        return start

    def un(pathname):
        # files from the UN.

         # date is in the file name
         # time is in other: time_code_of_first_frame
         # 1672828_DMOICT Open Camps CR7 8AM-12PM 16 JULY 16.mov

        year = "2016"
        month = 'JULY' 
        day = pathname.split(month)[0].split()[-1]

        media_info = MediaInfo.parse(pathname)
        t3=media_info.tracks[3]
        time = t3.time_code_of_first_frame

        dt = "{year}-{month}-{day} {time}".format(
                year=year, month=month, day=day, time=time)

        # start = datetime.datetime.strptime("16 JULY 2016 07:50:00;00", "%d %B %Y %H:%M:%S;00")
        start = datetime.datetime.strptime(dt, "%Y-%B-%d %H:%M:%S;00")
        print( start )

        return start

        # d+_DMOICT...move stuff so it errors if it finds something else
        start_date_re = r".*/" + date_re + ".*/\d+_DMOICT.*\.mov"

        start_date_o =  re.match(start_date_re, pathname)
        dt_parts = start_date_o.groupdict()
        print("date_parts:", dt_parts)

        
        cmd = ['mediainfo', 
                '--Inform=Video;%TimeCode_FirstFrame%', 
                pathname ]
        p = subprocess.Popen(
                cmd,
                stdout = subprocess.PIPE )
        stdout = p.stdout.read()
        # '07:50:00:00\n'

        # time_code = stdout.strip().split(':')

        start_time_re = time_re + rb":\d\d\n"

        start_time_o =  re.match(start_time_re, stdout)
        start_time_d = start_time_o.groupdict()
        print("start_time_d:",start_time_d)

        dt_parts.update(start_time_d)
        pprint.pprint(dt_parts)

        dt_parts = {k:int(v) for k,v in list(dt_parts.items())}
        print(dt_parts)

        start=datetime.datetime( **dt_parts )
        print(start)

        return start


    def auto(pathname):
        # try to figure out what to use
        # print("auto...")

        time_re = r".*(?P<h>\d+)_(?P<m>\d+)_(?P<s>\d+)\."
        ext = os.path.splitext(pathname)[1]

        if re.match( time_re, pathname ) is not None:
            start = re_name(pathname)

        elif ext == ".dv":
            start = parse_name(pathname)

        elif ext == ".MTS":
            start = PyExifTool(pathname)

        else:
            start = gst_discover_start(pathname)

        return start

    # get_start() starts here..

    # wacky python case statement 
    start = {'fn':parse_name,
             'fs':fs_time,
             'frame':frame_time,
             'gst':gst_discover_start,
             'un':un,
             'auto':auto,
             }[time_source](pathname)

    return start

Example 18

Project: nzbget-subliminal
Source File: DesktopEntry.py
View license
    def checkKey(self, key, value, group):
        # standard keys     
        if key == "Type":
            if value == "ServiceType" or value == "Service" or value == "FSDevice":
                self.warnings.append("Type=%s is a KDE extension" % key)
            elif value == "MimeType":
                self.warnings.append("Type=MimeType is deprecated")
            elif not (value == "Application" or value == "Link" or value == "Directory"):
                self.errors.append("Value of key 'Type' must be Application, Link or Directory, but is '%s'" % value)

            if self.fileExtension == ".directory" and not value == "Directory":
                self.warnings.append("File extension is .directory, but Type is '%s'" % value)
            elif self.fileExtension == ".desktop" and value == "Directory":
                self.warnings.append("Files with Type=Directory should have the extension .directory")

            if value == "Application":
                if "Exec" not in self.content[group]:
                    self.warnings.append("Type=Application needs 'Exec' key")
            if value == "Link":
                if "URL" not in self.content[group]:
                    self.warnings.append("Type=Link needs 'URL' key")

        elif key == "Version":
            self.checkValue(key, value)

        elif re.match("^Name"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif re.match("^GenericName"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif key == "NoDisplay":
            self.checkValue(key, value, type="boolean")

        elif re.match("^Comment"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif re.match("^Icon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)

        elif key == "Hidden":
            self.checkValue(key, value, type="boolean")

        elif key == "OnlyShowIn":
            self.checkValue(key, value, list=True)
            self.checkOnlyShowIn(value)

        elif key == "NotShowIn":
            self.checkValue(key, value, list=True)
            self.checkOnlyShowIn(value)

        elif key == "TryExec":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Exec":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Path":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Terminal":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "Application")
        
        elif key == "Actions":
            self.checkValue(key, value, list=True)
            self.checkType(key, "Application")

        elif key == "MimeType":
            self.checkValue(key, value, list=True)
            self.checkType(key, "Application")

        elif key == "Categories":
            self.checkValue(key, value)
            self.checkType(key, "Application")
            self.checkCategories(value)
        
        elif re.match("^Keywords"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value, type="localestring", list=True)
            self.checkType(key, "Application")

        elif key == "StartupNotify":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "Application")

        elif key == "StartupWMClass":
            self.checkType(key, "Application")

        elif key == "URL":
            self.checkValue(key, value)
            self.checkType(key, "URL")

        # kde extensions
        elif key == "ServiceTypes":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "DocPath":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "InitialPreference":
            self.checkValue(key, value, type="numeric")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "Dev":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "FSType":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "MountPoint":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "ReadOnly":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif re.match("^UnmountIcon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        # deprecated keys
        elif key == "Encoding":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif re.match("^MiniIcon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "TerminalOptions":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "DefaultApp":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "Protocols":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "Extensions":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "BinaryPattern":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "MapNotify":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif re.match("^SwallowTitle"+xdg.Locale.regex+"$", key):
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "SwallowExec":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "FilePattern":
            self.checkValue(key, value, type="regex", list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "SortOrder":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        # "X-" extensions
        elif re.match("^X-[a-zA-Z0-9-]+", key):
            pass

        else:
            self.errors.append("Invalid key: %s" % key)

Example 19

Project: haproxy-dconv
Source File: haproxy-dconv.py
View license
def convert(pctxt, infile, outfile, base=''):
    global document, keywords, keywordsCount, chapters, keyword_conflicts

    if len(base) > 0 and base[:-1] != '/':
        base += '/'

    hasSummary = False

    data = []
    fd = file(infile,"r")
    for line in fd:
        line.replace("\t", " " * 8)
        line = line.rstrip()
        data.append(line)
    fd.close()

    parsers = init_parsers(pctxt)

    pctxt.context = {
            'headers':  {},
            'document': "",
            'base':     base,
    }

    sections = []
    currentSection = {
            "details": getTitleDetails(""),
            "content": "",
    }

    chapters = {}

    keywords = {}
    keywordsCount = {}

    specialSections = {
            "default": {
                    "hasKeywords": True,
            },
            "4.1": {
                    "hasKeywords": True,
            },
    }

    pctxt.keywords = keywords
    pctxt.keywordsCount = keywordsCount
    pctxt.chapters = chapters

    print >> sys.stderr, "Importing %s..." % infile

    nblines = len(data)
    i = j = 0
    while i < nblines:
        line = data[i].rstrip()
        if i < nblines - 1:
            next = data[i + 1].rstrip()
        else:
            next = ""
        if (line == "Summary" or re.match("^[0-9].*", line)) and (len(next) > 0) and (next[0] == '-') \
                and ("-" * len(line)).startswith(next):  # Fuzzy underline length detection
            sections.append(currentSection)
            currentSection = {
                "details": getTitleDetails(line),
                "content": "",
            }
            j = 0
            i += 1 # Skip underline
            while not data[i + 1].rstrip():
                i += 1 # Skip empty lines

        else:
            if len(line) > 80:
                print >> sys.stderr, "Line `%i' exceeds 80 columns" % (i + 1)

            currentSection["content"] = currentSection["content"] + line + "\n"
            j += 1
            if currentSection["details"]["title"] == "Summary" and line != "":
                hasSummary = True
                # Learn chapters from the summary
                details = getTitleDetails(line)
                if details["chapter"]:
                    chapters[details["chapter"]] = details
        i += 1
    sections.append(currentSection)

    chapterIndexes = sorted(chapters.keys(), key=lambda chapter: map(int, chapter.split('.')))

    document = ""

    # Complete the summary
    for section in sections:
        details = section["details"]
        title = details["title"]
        if title:
            fulltitle = title
            if details["chapter"]:
                #documentAppend("<a name=\"%s\"></a>" % details["chapter"])
                fulltitle = details["chapter"] + ". " + title
                if not details["chapter"] in chapters:
                    print >> sys.stderr, "Adding '%s' to the summary" % details["title"]
                    chapters[details["chapter"]] = details
                    chapterIndexes = sorted(chapters.keys())

    for section in sections:
        details = section["details"]
        pctxt.details = details
        level = details["level"]
        title = details["title"]
        content = section["content"].rstrip()

        print >> sys.stderr, "Parsing chapter %s..." % title

        if (title == "Summary") or (title and not hasSummary):
            summaryTemplate = pctxt.templates.get_template('summary.html')
            documentAppend(summaryTemplate.render(
                pctxt = pctxt,
                chapters = chapters,
                chapterIndexes = chapterIndexes,
            ))
            if title and not hasSummary:
                hasSummary = True
            else:
                continue

        if title:
            documentAppend('<a class="anchor" id="%s" name="%s"></a>' % (details["chapter"], details["chapter"]))
            if level == 1:
                documentAppend("<div class=\"page-header\">", False)
            documentAppend('<h%d id="chapter-%s" data-target="%s"><small><a class="small" href="#%s">%s.</a></small> %s</h%d>' % (level, details["chapter"], details["chapter"], details["chapter"], details["chapter"], cgi.escape(title, True), level))
            if level == 1:
                documentAppend("</div>", False)

        if content:
            if False and title:
                # Display a navigation bar
                documentAppend('<ul class="well pager">')
                documentAppend('<li><a href="#top">Top</a></li>', False)
                index = chapterIndexes.index(details["chapter"])
                if index > 0:
                    documentAppend('<li class="previous"><a href="#%s">Previous</a></li>' % chapterIndexes[index - 1], False)
                if index < len(chapterIndexes) - 1:
                    documentAppend('<li class="next"><a href="#%s">Next</a></li>' % chapterIndexes[index + 1], False)
                documentAppend('</ul>', False)
            content = cgi.escape(content, True)
            content = re.sub(r'section ([0-9]+(.[0-9]+)*)', r'<a href="#\1">section \1</a>', content)

            pctxt.set_content(content)

            if not title:
                lines = pctxt.get_lines()
                pctxt.context['headers'] = {
                    'title':    '',
                    'subtitle': '',
                    'version':  '',
                    'author':   '',
                    'date':     ''
                }
                if re.match("^-+$", pctxt.get_line().strip()):
                    # Try to analyze the header of the file, assuming it follows
                    # those rules :
                    # - it begins with a "separator line" (several '-' chars)
                    # - then the document title
                    # - an optional subtitle
                    # - a new separator line
                    # - the version
                    # - the author
                    # - the date
                    pctxt.next()
                    pctxt.context['headers']['title'] = pctxt.get_line().strip()
                    pctxt.next()
                    subtitle = ""
                    while not re.match("^-+$", pctxt.get_line().strip()):
                        subtitle += " " + pctxt.get_line().strip()
                        pctxt.next()
                    pctxt.context['headers']['subtitle'] += subtitle.strip()
                    if not pctxt.context['headers']['subtitle']:
                        # No subtitle, try to guess one from the title if it
                        # starts with the word "HAProxy"
                        if pctxt.context['headers']['title'].startswith('HAProxy '):
                            pctxt.context['headers']['subtitle'] = pctxt.context['headers']['title'][8:]
                            pctxt.context['headers']['title'] = 'HAProxy'
                    pctxt.next()
                    pctxt.context['headers']['version'] = pctxt.get_line().strip()
                    pctxt.next()
                    pctxt.context['headers']['author'] = pctxt.get_line().strip()
                    pctxt.next()
                    pctxt.context['headers']['date'] = pctxt.get_line().strip()
                    pctxt.next()
                    if HAPROXY_GIT_VERSION:
                        pctxt.context['headers']['version'] = 'version ' + HAPROXY_GIT_VERSION

                    # Skip header lines
                    pctxt.eat_lines()
                    pctxt.eat_empty_lines()

            documentAppend('<div>', False)

            delay = []
            while pctxt.has_more_lines():
                try:
                    specialSection = specialSections[details["chapter"]]
                except:
                    specialSection = specialSections["default"]

                line = pctxt.get_line()
                if i < nblines - 1:
                    nextline = pctxt.get_line(1)
                else:
                    nextline = ""

                oldline = line
                pctxt.stop = False
                for parser in parsers:
                    line = parser.parse(line)
                    if pctxt.stop:
                        break
                if oldline == line:
                    # nothing has changed,
                    # delays the rendering
                    if delay or line != "":
                        delay.append(line)
                    pctxt.next()
                elif pctxt.stop:
                    while delay and delay[-1].strip() == "":
                        del delay[-1]
                    if delay:
                        remove_indent(delay)
                        documentAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
                    delay = []
                    documentAppend(line, False)
                else:
                    while delay and delay[-1].strip() == "":
                        del delay[-1]
                    if delay:
                        remove_indent(delay)
                        documentAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
                    delay = []
                    documentAppend(line, True)
                    pctxt.next()

            while delay and delay[-1].strip() == "":
                del delay[-1]
            if delay:
                remove_indent(delay)
                documentAppend('<pre class="text">%s\n</pre>' % "\n".join(delay), False)
            delay = []
            documentAppend('</div>')

    if not hasSummary:
        summaryTemplate = pctxt.templates.get_template('summary.html')
        print chapters
        document = summaryTemplate.render(
            pctxt = pctxt,
            chapters = chapters,
            chapterIndexes = chapterIndexes,
        ) + document


    # Log warnings for keywords defined in several chapters
    keyword_conflicts = {}
    for keyword in keywords:
        keyword_chapters = list(keywords[keyword])
        keyword_chapters.sort()
        if len(keyword_chapters) > 1:
            print >> sys.stderr, 'Multi section keyword : "%s" in chapters %s' % (keyword, list(keyword_chapters))
            keyword_conflicts[keyword] = keyword_chapters

    keywords = list(keywords)
    keywords.sort()

    createLinks()

    # Add the keywords conflicts to the keywords list to make them available in the search form
    # And remove the original keyword which is now useless
    for keyword in keyword_conflicts:
        sections = keyword_conflicts[keyword]
        offset = keywords.index(keyword)
        for section in sections:
            keywords.insert(offset, "%s (%s)" % (keyword, chapters[section]['title']))
            offset += 1
        keywords.remove(keyword)

    try:
        footerTemplate = pctxt.templates.get_template('footer.html')
        footer = footerTemplate.render(
            pctxt = pctxt,
            headers = pctxt.context['headers'],
            document = document,
            chapters = chapters,
            chapterIndexes = chapterIndexes,
            keywords = keywords,
            keywordsCount = keywordsCount,
            keyword_conflicts = keyword_conflicts,
            version = VERSION,
            date = datetime.datetime.now().strftime("%Y/%m/%d"),
        )
    except TopLevelLookupException:
        footer = ""

    return {
            'pctxt': pctxt,
            'headers': pctxt.context['headers'],
            'base': base,
            'document': document,
            'chapters': chapters,
            'chapterIndexes': chapterIndexes,
            'keywords': keywords,
            'keywordsCount': keywordsCount,
            'keyword_conflicts': keyword_conflicts,
            'version': VERSION,
            'date': datetime.datetime.now().strftime("%Y/%m/%d"),
            'footer': footer
    }

Example 20

Project: nzbToMedia
Source File: DesktopEntry.py
View license
    def checkKey(self, key, value, group):
        # standard keys     
        if key == "Type":
            if value == "ServiceType" or value == "Service" or value == "FSDevice":
                self.warnings.append("Type=%s is a KDE extension" % key)
            elif value == "MimeType":
                self.warnings.append("Type=MimeType is deprecated")
            elif not (value == "Application" or value == "Link" or value == "Directory"):
                self.errors.append("Value of key 'Type' must be Application, Link or Directory, but is '%s'" % value)

            if self.fileExtension == ".directory" and not value == "Directory":
                self.warnings.append("File extension is .directory, but Type is '%s'" % value)
            elif self.fileExtension == ".desktop" and value == "Directory":
                self.warnings.append("Files with Type=Directory should have the extension .directory")

            if value == "Application":
                if "Exec" not in self.content[group]:
                    self.warnings.append("Type=Application needs 'Exec' key")
            if value == "Link":
                if "URL" not in self.content[group]:
                    self.warnings.append("Type=Link needs 'URL' key")

        elif key == "Version":
            self.checkValue(key, value)

        elif re.match("^Name"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif re.match("^GenericName"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif key == "NoDisplay":
            self.checkValue(key, value, type="boolean")

        elif re.match("^Comment"+xdg.Locale.regex+"$", key):
            pass # locale string

        elif re.match("^Icon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)

        elif key == "Hidden":
            self.checkValue(key, value, type="boolean")

        elif key == "OnlyShowIn":
            self.checkValue(key, value, list=True)
            self.checkOnlyShowIn(value)

        elif key == "NotShowIn":
            self.checkValue(key, value, list=True)
            self.checkOnlyShowIn(value)

        elif key == "TryExec":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Exec":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Path":
            self.checkValue(key, value)
            self.checkType(key, "Application")

        elif key == "Terminal":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "Application")
        
        elif key == "Actions":
            self.checkValue(key, value, list=True)
            self.checkType(key, "Application")

        elif key == "MimeType":
            self.checkValue(key, value, list=True)
            self.checkType(key, "Application")

        elif key == "Categories":
            self.checkValue(key, value)
            self.checkType(key, "Application")
            self.checkCategories(value)
        
        elif re.match("^Keywords"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value, type="localestring", list=True)
            self.checkType(key, "Application")

        elif key == "StartupNotify":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "Application")

        elif key == "StartupWMClass":
            self.checkType(key, "Application")

        elif key == "URL":
            self.checkValue(key, value)
            self.checkType(key, "URL")

        # kde extensions
        elif key == "ServiceTypes":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "DocPath":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "InitialPreference":
            self.checkValue(key, value, type="numeric")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "Dev":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "FSType":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "MountPoint":
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif key == "ReadOnly":
            self.checkValue(key, value, type="boolean")
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        elif re.match("^UnmountIcon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)
            self.checkType(key, "FSDevice")
            self.warnings.append("Key '%s' is a KDE extension" % key)

        # deprecated keys
        elif key == "Encoding":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif re.match("^MiniIcon"+xdg.Locale.regex+"$", key):
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "TerminalOptions":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "DefaultApp":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "Protocols":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "Extensions":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "BinaryPattern":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "MapNotify":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif re.match("^SwallowTitle"+xdg.Locale.regex+"$", key):
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "SwallowExec":
            self.checkValue(key, value)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "FilePattern":
            self.checkValue(key, value, type="regex", list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        elif key == "SortOrder":
            self.checkValue(key, value, list=True)
            self.warnings.append("Key '%s' is deprecated" % key)

        # "X-" extensions
        elif re.match("^X-[a-zA-Z0-9-]+", key):
            pass

        else:
            self.errors.append("Invalid key: %s" % key)

Example 21

Project: stonix
Source File: ConfigureMACPolicy.py
View license
    def fixSELinux(self):
        '''
        enable and configure selinux. self.rulesuccess will be updated if this
        method does not succeed.

        @author bemalmbe
        @change: dwalker 4/04/2014 implemented commandhelper, added more
            accurate implementation per system basis for apt-get systems
            especially.
        '''
        if not self.ConfigureMAC.getcurrvalue():
            return
        self.detailedresults = ""
        if not self.kernel:
            return
        #clear out event history so only the latest fix is recorded
        self.iditerator = 0
        eventlist = self.statechglogger.findrulechanges(self.rulenumber)
        for event in eventlist:
            self.statechglogger.deleteentry(event)

        if not self.pkghelper.check(self.selinux):
            if self.pkghelper.checkAvailable(self.selinux):
                if not self.pkghelper.install(self.selinux):
                    self.rulesuccess = False
                    self.detailedresults += "selinux was not able to be \
installed\n"
                    self.formatDetailedResults("report", self.compliant,
                                               self.detailedresults)
                    self.logdispatch.log(LogPriority.INFO, self.detailedresults)
                    return
                else:
                    self.seinstall = True
            else:
                self.detailedresults += "selinux package not available \
for install on this linux distribution\n"
                self.rulesuccess = False
                self.formatDetailedResults("report", self.rulesuccess,
                                           self.detailedresults)
                return
        self.f1 = readFile(self.path1, self.logger)
        self.f2 = readFile(self.path2, self.logger)
        if self.f1:
            if not checkPerms(self.path1, [0, 0, 420], self.logger):
                self.iditerator += 1
                myid = iterate(self.iditerator, self.rulenumber)
                setPerms(self.path1, [0, 0, 420], self.logger,
                         self.statechglogger, myid)
                self.detailedresults += "Corrected permissions on file: \
" + self.path1 + "\n"
            val1 = ""
            tempstring = ""
            for line in self.f1:
                if re.match("^#", line) or re.match("^\s*$", line):
                    tempstring += line
                    continue
                if re.match("^SELINUX\s{0,1}=", line.strip()):
                    if re.search("=", line.strip()):
                        temp = line.split("=")
                        if temp[1].strip() == "permissive" or temp[1].strip() == "enforcing":
                            val1 = temp[1].strip()
                        if val1 != self.modeci.getcurrvalue():
                            val1 = self.modeci.getcurrvalue()
                            continue
                if re.match("^SELINUXTYPE", line.strip()):
                    continue
                else:
                    tempstring += line
            tempstring += self.universal
            if val1:
                tempstring += "SELINUX=" + val1 + "\n"
            else:
                tempstring += "SELINUX=permissive\n"
            tempstring += "SELINUXTYPE=" + self.setype + "\n"

        else:
            tempstring = ""
            tempstring += self.universal
            tempstring += "SELINUX=permissive\n"
            tempstring += "SELINUXTYPE=" + self.setype + "\n"
        if writeFile(self.tpath1, tempstring, self.logger):
            self.iditerator += 1
            myid = iterate(self.iditerator, self.rulenumber)
            event = {"eventtype": "conf",
                     "filepath": self.path1}
            self.statechglogger.recordchgevent(myid, event)
            self.statechglogger.recordfilechange(self.path1, self.tpath1,
                                                 myid)
            os.rename(self.tpath1, self.path1)
            os.chown(self.path1, 0, 0)
            os.chmod(self.path1, 420)
            resetsecon(self.path1)
            self.detailedresults += "Corrected the contents of the file: \
" + self.path1 + " to be compliant\n"
        else:
            self.rulesuccess = False
        if self.f2:
            if not checkPerms(self.path2, self.perms2, self.logger):
                self.iditerator += 1
                myid = iterate(self.iditerator, self.rulenumber)
                setPerms(self.path2, self.perms2, self.logger,
                         self.statechglogger, myid)
                self.detailedresults += "Corrected permissions on file: \
" + self.path2 + "\n"
            if self.pkghelper.manager == "apt-get":
                tempstring = ""
                for line in self.f2:
                    if re.match("^GRUB_CMDLINE_LINUX_DEFAULT", line.strip()):
                        newstring = re.sub("security=[a-zA-Z0-9]+", "", line)
                        newstring = re.sub("selinux=[a-zA-Z0-9]+", "", newstring)
                        newstring = re.sub("\s+", " ", newstring)
                        tempstring += newstring + "\n"
                    else:
                        tempstring += line
            else:
                tempstring = ""
                for line in self.f2:
                    if re.match("^kernel", line):
                        temp = line.strip().split()
                        i = 0
                        for item in temp:
                            if re.search("selinux", item):
                                temp.pop(i)
                                i += 1
                                continue
                            if re.search("enforcing", item):
                                temp.pop(i)
                                i += 1
                                continue
                            i += 1
                        tempstringtemp = ""
                        for item in temp:
                            tempstringtemp += item
                        tempstringtemp += "\n"
                        tempstring += tempstringtemp
                    else:
                        tempstring += line
            if tempstring:
                if writeFile(self.tpath2, tempstring, self.logger):
                    self.iditerator += 1
                    myid = iterate(self.iditerator, self.rulenumber)
                    event = {"eventtype": "conf",
                             "filepath": self.path2}
                    self.statechglogger.recordchgevent(myid, event)
                    self.statechglogger.recordfilechange(self.path2,
                                                         self.tpath2, myid)
                    os.rename(self.tpath2, self.path2)
                    os.chown(self.path2, self.perms2[0], self.perms2[1])
                    os.chmod(self.path2, self.perms2[2])
                    resetsecon(self.path2)
                    self.detailedresults += "Corrected the contents of \
the file: " + self.path2 + " to be compliant\n"
                else:
                    self.rulesuccess = False
        if not self.seinstall:
            if self.pkghelper.manager == "apt-get":
                if re.search("Debian", self.environ.getostype()):
                    cmd = ["/usr/sbin/selinux-activate"]
                elif re.search("Ubuntu", self.environ.getostype()):
                    cmd = ["/usr/sbin/setenforce", "Enforcing"]
                if self.ch.executeCommand(cmd):
                    if not self.ch.getReturnCode() == 0:
                        self.rulesuccess = False

Example 22

Project: pyqso
Source File: adif.py
View license
    def is_valid(self, field_name, data, data_type):
        """ Validate the data in a field with respect to the ADIF specification.

        :arg str field_name: The name of the ADIF field.
        :arg str data: The data of the ADIF field to validate.
        :arg str data_type: The type of data to be validated. See http://www.adif.org/304/ADIF_304.htm#Data_Types for the full list with descriptions.
        :returns: True or False to indicate whether the data is valid or not.
        :rtype: bool
        """

        logging.debug("Validating the following data in field '%s': %s" % (field_name, data))

        # Allow an empty string, in case the user doesn't want
        # to fill in this field.
        if(data == ""):
            return True

        if(data_type == "N"):
            # Allow a decimal point before and/or after any numbers,
            # but don't allow a decimal point on its own.
            m = re.match(r"-?(([0-9]+\.?[0-9]*)|([0-9]*\.?[0-9]+))", data)
            if(m is None):
                # Did not match anything.
                return False
            else:
                # Make sure we match the whole string,
                # otherwise there may be an invalid character after the match.
                return (m.group(0) == data)

        elif(data_type == "B"):
            # Boolean
            m = re.match(r"(Y|N)", data)
            if(m is None):
                return False
            else:
                return (m.group(0) == data)

        elif(data_type == "D"):
            # Date
            pattern = re.compile(r"([0-9]{4})")
            m_year = pattern.match(data, 0)
            if((m_year is None) or (int(m_year.group(0)) < 1930)):
                # Did not match anything.
                return False
            else:
                pattern = re.compile(r"([0-9]{2})")
                m_month = pattern.match(data, 4)
                if((m_month is None) or int(m_month.group(0)) > 12 or int(m_month.group(0)) < 1):
                    # Did not match anything.
                    return False
                else:
                    pattern = re.compile(r"([0-9]{2})")
                    m_day = pattern.match(data, 6)
                    days_in_month = calendar.monthrange(int(m_year.group(0)), int(m_month.group(0)))
                    if((m_day is None) or int(m_day.group(0)) > days_in_month[1] or int(m_day.group(0)) < 1):
                        # Did not match anything.
                        return False
                    else:
                        # Make sure we match the whole string,
                        # otherwise there may be an invalid character after the match.
                        return (len(data) == 8)

        elif(data_type == "T"):
            # Time
            pattern = re.compile(r"([0-9]{2})")
            m_hour = pattern.match(data, 0)
            if((m_hour is None) or (int(m_hour.group(0)) < 0) or (int(m_hour.group(0)) > 23)):
                # Did not match anything.
                return False
            else:
                pattern = re.compile(r"([0-9]{2})")
                m_minutes = pattern.match(data, 2)
                if((m_minutes is None) or int(m_minutes.group(0)) < 0 or int(m_minutes.group(0)) > 59):
                    # Did not match anything.
                    return False
                else:
                    if(len(data) == 4):
                        # HHMM format
                        return True
                    pattern = re.compile(r"([0-9]{2})")
                    m_seconds = pattern.match(data, 4)
                    if((m_seconds is None) or int(m_seconds.group(0)) < 0 or int(m_seconds.group(0)) > 59):
                        # Did not match anything.
                        return False
                    else:
                        # Make sure we match the whole string,
                        # otherwise there may be an invalid character after the match.
                        return (len(data) == 6)  # HHMMSS format

        # FIXME: Need to make sure that the "S" and "M" data types accept ASCII-only characters
        # in the range 32-126 inclusive.
        elif(data_type == "S"):
            # String
            m = re.match(r"(.+)", data)
            if(m is None):
                return False
            else:
                return (m.group(0) == data)

        elif(data_type == "I"):
            # IntlString
            m = re.match(r"(.+)", data, re.UNICODE)
            if(m is None):
                return False
            else:
                return (m.group(0) == data)

        elif(data_type == "G"):
            # IntlMultilineString
            m = re.match(r"(.+(\r\n)*.*)", data, re.UNICODE)
            if(m is None):
                return False
            else:
                return (m.group(0) == data)

        elif(data_type == "M"):
            # MultilineString
            # m = re.match(r"(.+(\r\n)*.*)", data)
            # if(m is None):
            #   return False
            # else:
            #   return (m.group(0) == data)
            return True

        elif(data_type == "L"):
            # Location
            pattern = re.compile(r"([EWNS]{1})", re.IGNORECASE)
            m_directional = pattern.match(data, 0)
            if(m_directional is None):
                # Did not match anything.
                return False
            else:
                pattern = re.compile(r"([0-9]{3})")
                m_degrees = pattern.match(data, 1)
                if((m_degrees is None) or int(m_degrees.group(0)) < 0 or int(m_degrees.group(0)) > 180):
                    # Did not match anything.
                    return False
                else:
                    pattern = re.compile(r"([0-9]{2}\.[0-9]{3})")
                    m_minutes = pattern.match(data, 4)
                    if((m_minutes is None) or float(m_minutes.group(0)) < 0 or float(m_minutes.group(0)) > 59.999):
                        # Did not match anything.
                        return False
                    else:
                        # Make sure we match the whole string,
                        # otherwise there may be an invalid character after the match.
                        return (len(data) == 10)

        elif(data_type == "E" or data_type == "A"):
            # Enumeration, AwardList.
            if(field_name == "MODE"):
                return (data in list(MODES.keys()))
            elif(field_name == "BAND"):
                return (data in BANDS)
            else:
                return True

        else:
            return True

Example 23

Project: rivescript-python
Source File: brain.py
View license
    def _getreply(self, user, msg, context='normal', step=0, ignore_object_errors=True):
        """The internal reply getter function.

        DO NOT CALL THIS YOURSELF.

        :param str user: The user ID as passed to ``reply()``.
        :param str msg: The formatted user message.
        :param str context: The reply context, one of ``begin`` or ``normal``.
        :param int step: The recursion depth counter.
        :param bool ignore_object_errors: Whether to ignore errors from within
            Python object macros and not raise an ``ObjectError`` exception.

        :return str: The reply output.
        """
        # Needed to sort replies?
        if 'topics' not in self.master._sorted:
            raise RepliesNotSortedError("You must call sort_replies() once you are done loading RiveScript documents")

        # Initialize the user's profile?
        topic = self.master.get_uservar(user, "topic")
        if topic in [None, "undefined"]:
            topic = "random"
            self.master.set_uservar(user, "topic", topic)

        # Collect data on the user.
        stars     = []
        thatstars = []  # For %Previous's.
        reply     = ''

        # Avoid letting them fall into a missing topic.
        if topic not in self.master._topics:
            self.warn("User " + user + " was in an empty topic named '" + topic + "'")
            topic = "random"
            self.master.set_uservar(user, "topic", topic)

        # Avoid deep recursion.
        if step > self.master._depth:
            raise DeepRecursionError

        # Are we in the BEGIN statement?
        if context == 'begin':
            topic = '__begin__'

        # Initialize this user's history.
        history = self.master.get_uservar(user, "__history__")
        if type(history) is not dict or "input" not in history or "reply" not in history:
            history = self.default_history()
            self.master.set_uservar(user, "__history__", history)

        # More topic sanity checking.
        if topic not in self.master._topics:
            # This was handled before, which would mean topic=random and
            # it doesn't exist. Serious issue!
            raise NoDefaultRandomTopicError("no default topic 'random' was found")

        # Create a pointer for the matched data when we find it.
        matched        = None
        matchedTrigger = None
        foundMatch     = False

        # See if there were any %Previous's in this topic, or any topic related
        # to it. This should only be done the first time -- not during a
        # recursive redirection. This is because in a redirection, "lastreply"
        # is still gonna be the same as it was the first time, causing an
        # infinite loop!
        if step == 0:
            allTopics = [topic]
            if topic in self.master._includes or topic in self.master._lineage:
                # Get all the topics!
                allTopics = inherit_utils.get_topic_tree(self.master, topic)

            # Scan them all!
            for top in allTopics:
                self.say("Checking topic " + top + " for any %Previous's.")
                if top in self.master._sorted["thats"]:
                    self.say("There is a %Previous in this topic!")

                    # Do we have history yet?
                    lastReply = history["reply"][0]

                    # Format the bot's last reply the same way as the human's.
                    lastReply = self.format_message(lastReply, botreply=True)
                    self.say("lastReply: " + lastReply)

                    # See if it's a match.
                    for trig in self.master._sorted["thats"][top]:
                        pattern = trig[0]
                        botside = self.reply_regexp(user, pattern)
                        self.say("Try to match lastReply (" + lastReply + ") to " + pattern)

                        # Match??
                        match = re.match(botside, lastReply)
                        if match:
                            # Huzzah! See if OUR message is right too.
                            self.say("Bot side matched!")
                            thatstars = match.groups()

                            # Compare the triggers to the user's message.
                            user_side = trig[1]
                            subtrig = self.reply_regexp(user, user_side["trigger"])
                            self.say("Now try to match " + msg + " to " + user_side["trigger"])

                            match = re.match(subtrig, msg)
                            if match:
                                self.say("Found a match!")
                                matched = trig[1]
                                matchedTrigger = subtrig
                                foundMatch = True

                                # Get the stars!
                                stars = match.groups()
                                break

                        # Break if we found a match.
                        if foundMatch:
                            break
                # Break if we found a match.
                if foundMatch:
                    break

        # Search their topic for a match to their trigger.
        if not foundMatch:
            for trig in self.master._sorted["topics"][topic]:
                pattern = trig[0]

                # Process the triggers.
                regexp = self.reply_regexp(user, pattern)
                self.say("Try to match %r against %r (%r)" % (msg, pattern, regexp.pattern))

                # Python's regular expression engine is slow. Try a verbatim
                # match if this is an atomic trigger.
                isAtomic = utils.is_atomic(pattern)
                isMatch = False
                if isAtomic:
                    # Only look for exact matches, no sense running atomic triggers
                    # through the regexp engine.
                    if msg == pattern:
                        isMatch = True
                else:
                    # Non-atomic triggers always need the regexp.
                    match = re.match(regexp, msg)
                    if match:
                        # The regexp matched!
                        isMatch = True

                        # Collect the stars.
                        stars = match.groups()

                if isMatch:
                    self.say("Found a match!")

                    matched = trig[1]
                    foundMatch = True
                    matchedTrigger = pattern
                    break

        # Store what trigger they matched on. If their matched trigger is None,
        # this will be too, which is great.
        self.master.set_uservar(user, "__lastmatch__", matchedTrigger)

        if matched:
            for nil in [1]:
                # See if there are any hard redirects.
                if matched["redirect"]:
                    self.say("Redirecting us to " + matched["redirect"])
                    redirect = self.process_tags(user, msg, matched["redirect"], stars, thatstars, step,
                                                  ignore_object_errors)
                    self.say("Pretend user said: " + redirect)
                    reply = self._getreply(user, redirect, step=(step + 1), ignore_object_errors=ignore_object_errors)
                    break

                # Check the conditionals.
                for con in matched["condition"]:
                    halves = re.split(RE.cond_split, con)
                    if halves and len(halves) == 2:
                        condition = re.match(RE.cond_parse, halves[0])
                        if condition:
                            left     = condition.group(1)
                            eq       = condition.group(2)
                            right    = condition.group(3)
                            potreply = halves[1]
                            self.say("Left: " + left + "; eq: " + eq + "; right: " + right + " => " + potreply)

                            # Process tags all around.
                            left  = self.process_tags(user, msg, left, stars, thatstars, step, ignore_object_errors)
                            right = self.process_tags(user, msg, right, stars, thatstars, step, ignore_object_errors)

                            # Defaults?
                            if len(left) == 0:
                                left = 'undefined'
                            if len(right) == 0:
                                right = 'undefined'

                            self.say("Check if " + left + " " + eq + " " + right)

                            # Validate it.
                            passed = False
                            if eq == 'eq' or eq == '==':
                                if left == right:
                                    passed = True
                            elif eq == 'ne' or eq == '!=' or eq == '<>':
                                if left != right:
                                    passed = True
                            else:
                                # Gasp, dealing with numbers here...
                                try:
                                    left, right = int(left), int(right)
                                    if eq == '<':
                                        if left < right:
                                            passed = True
                                    elif eq == '<=':
                                        if left <= right:
                                            passed = True
                                    elif eq == '>':
                                        if left > right:
                                            passed = True
                                    elif eq == '>=':
                                        if left >= right:
                                            passed = True
                                except:
                                    self.warn("Failed to evaluate numeric condition!")

                            # How truthful?
                            if passed:
                                reply = potreply
                                break

                # Have our reply yet?
                if len(reply) > 0:
                    break

                # Process weights in the replies.
                bucket = []
                for text in matched["reply"]:
                    weight = 1
                    match  = re.match(RE.weight, text)
                    if match:
                        weight = int(match.group(1))
                        if weight <= 0:
                            self.warn("Can't have a weight <= 0!")
                            weight = 1
                    for i in range(0, weight):
                        bucket.append(text)

                # Get a random reply.
                reply = random.choice(bucket)
                break

        # Still no reply?
        if not foundMatch:
            raise NoMatchError
        elif len(reply) == 0:
            raise NoReplyError

        self.say("Reply: " + reply)

        # Process tags for the BEGIN block.
        if context == "begin":
            # BEGIN blocks can only set topics and uservars. The rest happen
            # later!
            reTopic = re.findall(RE.topic_tag, reply)
            for match in reTopic:
                self.say("Setting user's topic to " + match)
                self.master.set_uservar(user, "topic", match)
                reply = reply.replace('{{topic={match}}}'.format(match=match), '')

            reSet = re.findall(RE.set_tag, reply)
            for match in reSet:
                self.say("Set uservar " + str(match[0]) + "=" + str(match[1]))
                self.master.set_uservar(user, match[0], match[1])
                reply = reply.replace('<set {key}={value}>'.format(key=match[0], value=match[1]), '')
        else:
            # Process more tags if not in BEGIN.
            reply = self.process_tags(user, msg, reply, stars, thatstars, step, ignore_object_errors)

        return reply

Example 24

Project: pseudo
Source File: code_generator.py
View license
    def _parse_template(self, code, label):
        '''
        Pare smart indented templates

        Takes a template a returns a list of sub-templates, taking in account
        the indentation of the original code based on the first line indentation(0)
        Special treatment of whitespace: returns special Offset and INTERNAL_WHITESPACE, so the generation can be configurable
        It auto detects the indentation width used, as the indent of the first indented line
        >>> indented("""
          def %<code>
            e =
            %<code2>
          """)
        ['def', INTERNAL_WHITESPACE, Placeholder('code', 0), NEWLINE,
          Offset(1),'e', INTERNAL_WHITESPACE, '=', NEWLINE,
          Placeholder('code2', 1), NEWLINE]
        '''

        if isinstance(code, tuple):
            return tuple(self._parse_template(c, label) for c in code)
        elif isinstance(code, dict):
            return {
                k: self._parse_template(v, label) if k != '_key' else v
                for k, v
                in code.items()
            }
        elif not isinstance(code, str):
            return []

        lines = code.split('\n')
        parsed = []
        if len(lines) == 1:
            i = re.match(r'^( +)', lines[0])
            indent_size = len(i.group()) if i else 0
            indent = 1 if i else 0
            actual = lines
            base = 0
        else:
            base = len(re.match(r'^( *)', lines[1]).group())
            rebased = [line[base:] for line in lines]
            for line in rebased:
                i = re.match(r'^( +)', line)
                if i:
                    indent_size = len(i.group())
                    break
            else:
                indent_size = 0
            actual = rebased[1:]

        for line in actual:
            j = LINE_FIRS.match(line)
            indent = len(j.group()) // indent_size if j else 0
            if parsed:
                parsed.append(Offset(indent))
            in_placeholder = False
            in_action = False
            in_args = False
            in_string_arg = False
            in_double_arg = False
            in_type = False
            c = int(indent * indent_size)
            m = c
            placeholder = ''
            while m < len(line):
                # print(m, line[m], 'place:', in_placeholder, 'act:', in_action, 'a:', in_args, 's:', in_string_arg, yaml.dump(parsed))
                f = line[m]
                next_f = line[m + 1] if m < len(line) - 1 else None
                if f == '%' and not in_placeholder and next_f == '<':
                    m += 2
                    in_placeholder = True
                    placeholder = ''
                    continue
                elif f == ':' and in_placeholder:
                    m += 1
                    in_placeholder = False
                    in_action = True
                    action = ''
                    continue
                elif f == ' ' and in_placeholder:
                    m += 1
                    continue
                elif f == ' ' and in_action:
                    m += 1
                    in_action = False
                    in_args = True
                    args = ['']
                    continue
                elif f == ' ' and (in_string_arg or in_double_arg):
                    args[-1] += f
                    m += 1
                    continue
                elif f == ' ' and in_args:
                    m += 1
                    args.append('')
                    continue
                elif f == '\'' and in_args:
                    m += 1
                    if in_string_arg:
                        in_string_arg = False
                        if args[-1] == '\\n':
                            args[-1] = '\n'
                        args[-1] += f

                    elif in_double_arg:
                        args[-1] += f
                    else:
                        in_string_arg = True

                    continue
                elif f == '"' and in_args:
                    m += 1
                    if in_double_arg:
                        in_double_arg = False
                        if args[-1] == '\\n':
                            args[-1] = '\n'
                        args[-1] += f
                    elif in_string_arg:
                        args[-1] += f
                    else:
                        in_string_arg = True

                    continue
                elif f == '>' and in_args and not in_string_arg and not in_double_arg:
                    m += 1
                    if args[-1] == '':
                        args = args[:-1]
                    args = [arg[:-1] if arg[-1] == '\'' else int(arg) for arg in args]
                    in_args = False
                    parsed.append(Action(placeholder, action, args))
                    continue
                elif f == '>' and in_action:
                    m += 1
                    in_action = False
                    parsed.append(Action(placeholder, action, []))
                elif f == '>' and in_placeholder:
                    m += 1
                    q = None
                    # if '.' in placeholder[1:]:
                    #     input(placeholder)
                    if placeholder[0] == '#':
                        q = Function(placeholder[1:])
                    elif placeholder[0] == '@':
                        q = PseudoType(placeholder[1:].split('.'))
                    elif placeholder[0] == '.':
                        q = SubTemplate(label, placeholder[1:])
                    elif '.' in placeholder:
                        q = SubElement(placeholder.split('.'))
                    else:
                        q = Placeholder(placeholder)
                    in_placeholder = False
                    parsed.append(q)
                elif f == ' ':
                    m += 1
                    parsed.append(INTERNAL_WHITESPACE)
                elif in_placeholder:
                    m += 1
                    placeholder += f
                elif in_action:
                    m += 1
                    action += f
                elif in_args:
                    m += 1
                    args[-1] += f
                else:
                    m += 1
                    if parsed and isinstance(parsed[-1], str):
                        parsed[-1] += f
                    else:
                        parsed.append(f)
            if len(actual) > 1:
                parsed.append(NEWLINE)
        return parsed

Example 25

Project: dash-hack
Source File: hdr_parser.py
View license
    def parse_func_decl(self, decl_str):
        """
        Parses the function or method declaration in the form:
        [([CV_EXPORTS] <rettype>) | CVAPI(rettype)]
            [~]<function_name>
            (<arg_type1> <arg_name1>[=<default_value1>] [, <arg_type2> <arg_name2>[=<default_value2>] ...])
            [const] {; | <function_body>}

        Returns the function declaration entry:
        [<func name>, <return value C-type>, <list of modifiers>, <list of arguments>] (see above)
        """

        if self.wrap_mode:
            if not (("CV_EXPORTS_AS" in decl_str) or ("CV_EXPORTS_W" in decl_str) or \
                ("CV_WRAP" in decl_str) or ("CV_WRAP_AS" in decl_str)):
                return []

        # ignore old API in the documentation check (for now)
        if "CVAPI(" in decl_str and self.wrap_mode:
            return []

        top = self.block_stack[-1]
        func_modlist = []

        npos = decl_str.find("CV_EXPORTS_AS")
        if npos >= 0:
            arg, npos3 = self.get_macro_arg(decl_str, npos)
            func_modlist.append("="+arg)
            decl_str = decl_str[:npos] + decl_str[npos3+1:]
        npos = decl_str.find("CV_WRAP_AS")
        if npos >= 0:
            arg, npos3 = self.get_macro_arg(decl_str, npos)
            func_modlist.append("="+arg)
            decl_str = decl_str[:npos] + decl_str[npos3+1:]

        # filter off some common prefixes, which are meaningless for Python wrappers.
        # note that we do not strip "static" prefix, which does matter;
        # it means class methods, not instance methods
        decl_str = self.batch_replace(decl_str, [("virtual", ""), ("static inline", ""), ("inline", ""),\
            ("CV_EXPORTS_W", ""), ("CV_EXPORTS", ""), ("CV_CDECL", ""), ("CV_WRAP ", " "), ("static CV_INLINE", ""), ("CV_INLINE", "")]).strip()

        static_method = False
        context = top[0]
        if decl_str.startswith("static") and (context == "class" or context == "struct"):
            decl_str = decl_str[len("static"):].lstrip()
            static_method = True

        args_begin = decl_str.find("(")
        if decl_str.startswith("CVAPI"):
            rtype_end = decl_str.find(")", args_begin+1)
            if rtype_end < 0:
                print "Error at %d. no terminating ) in CVAPI() macro: %s" % (self.lineno, decl_str)
                sys.exit(-1)
            decl_str = decl_str[args_begin+1:rtype_end] + " " + decl_str[rtype_end+1:]
            args_begin = decl_str.find("(")
        if args_begin < 0:
            print "Error at %d: no args in '%s'" % (self.lineno, decl_str)
            sys.exit(-1)

        decl_start = decl_str[:args_begin].strip()
        # handle operator () case
        if decl_start.endswith("operator"):
            args_begin = decl_str.find("(", args_begin+1)
            if args_begin < 0:
                print "Error at %d: no args in '%s'" % (self.lineno, decl_str)
                sys.exit(-1)
            decl_start = decl_str[:args_begin].strip()
            # TODO: normalize all type of operators
            if decl_start.endswith("()"):
                decl_start = decl_start[0:-2].rstrip() + " ()"

        # constructor/destructor case
        if bool(re.match(r'^(\w+::)*(?P<x>\w+)::~?(?P=x)$', decl_start)):
            decl_start = "void " + decl_start

        rettype, funcname, modlist, argno = self.parse_arg(decl_start, -1)

        if argno >= 0:
            classname = top[1]
            if rettype == classname or rettype == "~" + classname:
                rettype, funcname = "", rettype
            else:
                if bool(re.match('\w+\s+\(\*\w+\)\s*\(.*\)', decl_str)):
                    return [] # function typedef
                elif bool(re.match('\w+\s+\(\w+::\*\w+\)\s*\(.*\)', decl_str)):
                    return [] # class method typedef
                elif bool(re.match('[A-Z_]+', decl_start)):
                    return [] # it seems to be a macro instantiation
                elif "__declspec" == decl_start:
                    return []
                elif bool(re.match(r'\w+\s+\(\*\w+\)\[\d+\]', decl_str)):
                    return [] # exotic - dynamic 2d array
                else:
                    #print rettype, funcname, modlist, argno
                    print "Error at %s:%d the function/method name is missing: '%s'" % (self.hname, self.lineno, decl_start)
                    sys.exit(-1)

        if self.wrap_mode and (("::" in funcname) or funcname.startswith("~")):
            # if there is :: in function name (and this is in the header file),
            # it means, this is inline implementation of a class method.
            # Thus the function has been already declared within the class and we skip this repeated
            # declaration.
            # Also, skip the destructors, as they are always wrapped
            return []

        funcname = self.get_dotted_name(funcname)

        if not self.wrap_mode:
            decl = self.parse_func_decl_no_wrap(decl_str, static_method)
            decl[0] = funcname
            return decl

        arg_start = args_begin+1
        npos = arg_start-1
        balance = 1
        angle_balance = 0
        # scan the argument list; handle nested parentheses
        args_decls = []
        args = []
        argno = 1

        while balance > 0:
            npos += 1
            t, npos = self.find_next_token(decl_str, ["(", ")", ",", "<", ">"], npos)
            if not t:
                print "Error: no closing ')' at %d" % (self.lineno,)
                print decl_str
                print decl_str[arg_start:]
                sys.exit(-1)
            if t == "<":
                angle_balance += 1
            if t == ">":
                angle_balance -= 1
            if t == "(":
                balance += 1
            if t == ")":
                balance -= 1

            if (t == "," and balance == 1 and angle_balance == 0) or balance == 0:
                # process next function argument
                a = decl_str[arg_start:npos].strip()
                #print "arg = ", a
                arg_start = npos+1
                if a:
                    eqpos = a.find("=")
                    defval = ""
                    modlist = []
                    if eqpos >= 0:
                        defval = a[eqpos+1:].strip()
                    else:
                        eqpos = a.find("CV_DEFAULT")
                        if eqpos >= 0:
                            defval, pos3 = self.get_macro_arg(a, eqpos)
                        else:
                            eqpos = a.find("CV_WRAP_DEFAULT")
                            if eqpos >= 0:
                                defval, pos3 = self.get_macro_arg(a, eqpos)
                    if defval == "NULL":
                        defval = "0"
                    if eqpos >= 0:
                        a = a[:eqpos].strip()
                    arg_type, arg_name, modlist, argno = self.parse_arg(a, argno)
                    if self.wrap_mode:
                        if arg_type == "InputArray":
                            arg_type = "Mat"
                        elif arg_type == "InputOutputArray":
                            arg_type = "Mat"
                            modlist.append("/IO")
                        elif arg_type == "OutputArray":
                            arg_type = "Mat"
                            modlist.append("/O")
                        elif arg_type == "InputArrayOfArrays":
                            arg_type = "vector_Mat"
                        elif arg_type == "InputOutputArrayOfArrays":
                            arg_type = "vector_Mat"
                            modlist.append("/IO")
                        elif arg_type == "OutputArrayOfArrays":
                            arg_type = "vector_Mat"
                            modlist.append("/O")
                        defval = self.batch_replace(defval, [("InputArrayOfArrays", "vector<Mat>"),
                                                             ("InputOutputArrayOfArrays", "vector<Mat>"),
                                                             ("OutputArrayOfArrays", "vector<Mat>"),
                                                             ("InputArray", "Mat"),
                                                             ("InputOutputArray", "Mat"),
                                                             ("OutputArray", "Mat"),
                                                             ("noArray", arg_type)]).strip()
                    args.append([arg_type, arg_name, defval, modlist])
                npos = arg_start-1

        npos = decl_str.replace(" ", "").find("=0", npos)
        if npos >= 0:
            # skip pure virtual functions
            return []

        if static_method:
            func_modlist.append("/S")

        return [funcname, rettype, func_modlist, args]

Example 26

Project: kay
Source File: media_compiler.py
View license
def compile_js_(tag_name, js_config, force):
  if IS_APPSERVER:
    return

  def needs_update(media_info):
    if js_config['tool'] != 'goog_calcdeps':
      # update if target file does not exist
      target_path = make_output_path_(js_config, js_config['subdir'],
                                      js_config['output_filename'])
      if not os.path.exists(target_path):
        return True

    # update if it lacks required info in _media.yaml
    last_info = media_info.get(js_config['subdir'], tag_name)
    if not last_info:
      return True
    last_config = last_info.get('config')
    if not last_config:
      return True

    # update if any configuration setting is changed
    if not equal_object_(last_config, js_config):
      return True

    if 'related_files' not in last_info:
      return True
    for path, mtime in last_info['related_files']:
      if mtime != os.path.getmtime(path):
        return True
      
  def jsminify(js_path):
    from StringIO import StringIO
    from kay.ext.media_compressor.jsmin import JavascriptMinify
    ifile = open(js_path)
    outs = StringIO()
    JavascriptMinify().minify(ifile, outs)
    ret = outs.getvalue()
    if len(ret) > 0 and ret[0] == '\n':
      ret = ret[1:]
    return ret

  def concat(js_path):
    print_status(" concat %s" % js_path)
    ifile = open(js_path)
    js = ifile.read()
    ifile.close()
    return js

  def goog_calcdeps():
    deps_config = copy.deepcopy(js_config['goog_common'])
    deps_config.update(js_config['goog_calcdeps'])

    if deps_config.get('method') not in \
          ['separate', 'concat', 'concat_refs', 'compile']:
      print_status("COMPILE_MEDIA_JS['goog_calcdeps']['method'] setting is"
                   " invalid; unknown method `%s'" % deps_config.get('method'))
      sys.exit(1)

    output_urls = []
    if deps_config['method'] == 'separate':
      source_files, output_urls = goog_calcdeps_separate(deps_config)
    elif deps_config['method'] == 'concat':
      source_files, output_urls = goog_calcdeps_concat(deps_config)
    elif deps_config['method'] == 'concat_refs':
      source_files, output_urls = goog_calcdeps_concat_refs(deps_config)
    elif deps_config['method'] == 'compile':
      source_files, output_urls = goog_calcdeps_compile(deps_config)
      source_files = [file[0] for file in source_files]

    related_files = union_list(source_files, 
                               [make_input_path_(path)
                                  for path in js_config['source_files']])
    related_file_info = [(path, os.path.getmtime(path))
                           for path in related_files]
    
    # create yaml info
    last_info = {'config': copy.deepcopy(js_config),
                 'related_files': related_file_info,
                 'result_urls': output_urls}
    media_info.set(js_config['subdir'], tag_name, last_info)
    media_info.save()

  def goog_calcdeps_separate(deps_config):
    source_files = goog_calcdeps_list(deps_config)
    (output_urls, extern_urls) = goog_calcdeps_copy_files(deps_config,
                                                          source_files)
    return (source_files, extern_urls + output_urls)

  def goog_calcdeps_concat(deps_config):
    source_files = goog_calcdeps_list(deps_config)
    (output_urls, extern_urls) = goog_calcdeps_concat_files(deps_config,
                                                            source_files)
    return (source_files, extern_urls + output_urls)

  def goog_calcdeps_concat_refs(deps_config):
    source_files = goog_calcdeps_list(deps_config)
    original_files = [make_input_path_(path)
                      for path in js_config['source_files']]
    ref_files = [path for path in source_files if path not in original_files]
    (output_urls, extern_urls) = goog_calcdeps_concat_files(deps_config,
                                                            ref_files)
    original_urls = [path[len(kay.PROJECT_DIR):] for path in original_files]
    return (source_files, extern_urls + output_urls + original_urls)

  def goog_calcdeps_compile(deps_config):
    comp_config = copy.deepcopy(js_config['goog_common'])
    comp_config.update(js_config['goog_compiler'])

    source_files = []
    extern_urls = []

    command = '%s -o compiled -c "%s" ' % (deps_config['path'],
                                                 comp_config['path'])
    for path in deps_config.get('search_paths', []):
      command += '-p %s ' % make_input_path_(path)
    for path in js_config['source_files']:
      path = make_input_path_(path)
      command += '-i %s ' % path
      source_files.append((path, os.path.getmtime(path)))

    if comp_config['level'] == 'minify':
      level = 'WHITESPACE_ONLY'
    elif comp_config['level'] == 'advanced':
      level = 'ADVANCED_OPTIMIZATIONS'
    else:
      level = 'SIMPLE_OPTIMIZATIONS'
    flags = '--compilation_level=%s' % level
#    for path in comp_config.get('externs', []):
#      flags += '--externs=%s ' % make_input_path_(path)
#    if comp_config.get('externs'):
#      flags += ' --externs=%s ' % " ".join(comp_config['externs'])
    command += '-f "%s" ' % flags
    print_status(command)
    command_output = os.popen(command).read()

    output_path = make_output_path_(js_config, js_config['subdir'],
                                    js_config['output_filename'])
    ofile = create_file_(output_path)
    try:
      for path in comp_config.get('externs', []):
        if re.match(r'^https?://', path):
          extern_urls.append(path)
          continue
        path = make_input_path_(path)
        ifile = open(path)
        try:
          ofile.write(ifile.read())
        finally:
          ifile.close()
        source_files.append((path, os.path.getmtime(path)))
      ofile.write(command_output)
    finally:
      ofile.close()
    return (source_files, extern_urls + [output_path[len(kay.PROJECT_DIR):]])

  def goog_calcdeps_list(deps_config):
    source_files = []

    command = '%s -o list ' % deps_config['path']
    for path in deps_config['search_paths']:
      command += '-p %s ' % make_input_path_(path)
    for path in js_config['source_files']:
      command += '-i %s ' % make_input_path_(path)
    print_status(command)
    command_output = os.popen(command).read()
    for path in command_output.split("\n"):
      if path == '': continue
      source_files.append(path)
    return source_files

  def goog_calcdeps_copy_files(deps_config, source_files):
    extern_urls = []
    output_urls = []

    output_dir_base = make_output_path_(js_config, 'separated_js')

    if not os.path.exists(output_dir_base):
      os.makedirs(output_dir_base)
    if not deps_config.get('use_dependency_file', True):
      output_path = os.path.join(output_dir_base, '__goog_nodeps.js')
      ofile = open(output_path, "w")
      output_urls.append(output_path[len(kay.PROJECT_DIR):])
      try:
        ofile.write('CLOSURE_NO_DEPS = true;')
      finally:
        ofile.close()

    output_dirs = {}
    search_paths = [make_input_path_(path)
                    for path in deps_config['search_paths']]
    for path in search_paths:
      output_dirs[path] = os.path.join(output_dir_base,
                                       md5.new(path).hexdigest())

    all_paths = [make_input_path_(path)
                 for path in deps_config.get('externs', [])]
    all_paths.extend(source_files)
    for path in all_paths:
      if re.match(r'^https?://', path):
        extern_urls.append(path)
        continue

      path = make_input_path_(path)
      output_path = os.path.join(output_dir_base, re.sub('^/', '', path))
      for dir in search_paths:
        if path[0:len(dir)] == dir:
          output_path = os.path.join(output_dirs[dir],
                                     re.sub('^/', '', path[len(dir):]))
          break
      output_dir = os.path.dirname(output_path)

      if not os.path.exists(output_dir):
        os.makedirs(output_dir)
      shutil.copy2(path, output_path)
      output_urls.append(output_path[len(kay.PROJECT_DIR):])
    return (output_urls, extern_urls)
    
  def goog_calcdeps_concat_files(deps_config, source_files):
    extern_urls = []

    output_path = make_output_path_(js_config, js_config['subdir'],
                                    js_config['output_filename'])
    ofile = create_file_(output_path)
    try:
      if not deps_config.get('use_dependency_file', True):
        ofile.write('CLOSURE_NO_DEPS = true;')
      all_paths = [make_input_path_(path)
                   for path in deps_config.get('externs', [])]
      all_paths.extend(source_files)
      for path in all_paths:
        if re.match(r'^https?://', path):
          extern_urls.append(path)
          continue
        ifile = open(make_input_path_(path))
        ofile.write(ifile.read())
        ifile.close()
    finally:
      ofile.close()

    return ([output_path[len(kay.PROJECT_DIR):]], extern_urls)

  selected_tool = js_config['tool']

  if selected_tool not in \
        (None, 'jsminify', 'concat', 'goog_calcdeps', 'goog_compiler'):
    print_status("COMPILE_MEDIA_JS['tool'] setting is invalid;"
                 " unknown tool `%s'" % selected_tool)
    sys.exit(1)

  global media_info
  if media_info is None:
    media_info = MediaInfo.load()

  if not force and not needs_update(media_info):
    print_status(' up to date.')
    return

  if selected_tool == 'goog_calcdeps':
    return goog_calcdeps()

  if selected_tool is None:
    last_info = {'config': copy.deepcopy(js_config),
                 'result_urls': ['/'+f for f in js_config['source_files']]}
    media_info.set(js_config['subdir'], tag_name, last_info)
    media_info.save()
    return

  dest_path = make_output_path_(js_config, js_config['subdir'],
                                js_config['output_filename'])
  ofile = create_file_(dest_path)
  try:
    if selected_tool == 'jsminify':
      for path in js_config['source_files']:
        src_path = make_input_path_(path)
        ofile.write(jsminify(src_path))
    elif selected_tool == 'concat':
      for path in js_config['source_files']:
        src_path = make_input_path_(path)
        ofile.write(concat(src_path))
  finally:
    ofile.close()
  
  if selected_tool == 'goog_compiler':
    comp_config = copy.deepcopy(js_config['goog_common'])
    comp_config.update(js_config['goog_compiler'])
    if comp_config['level'] == 'minify':
      level = 'WHITESPACE_ONLY'
    elif comp_config['level'] == 'advanced':
      level = 'ADVANCED_OPTIMIZATIONS'
    else:
      level = 'SIMPLE_OPTIMIZATIONS'
    command_args = '--compilation_level=%s' % level
    for path in js_config['source_files']:
      command_args += ' --js %s' % make_input_path_(path)
    command_args += ' --js_output_file %s' % dest_path
    command = 'java -jar %s %s' % (comp_config['path'], command_args)
    command_output = os.popen(command).read()

  info = copy.deepcopy(js_config)
  info['output_filename'] = make_output_path_(js_config, js_config['subdir'],
                                              js_config['output_filename'],
                                              relative=True)
  info['result_urls'] = ['/'+info['output_filename']]
  media_info.set(js_config['subdir'], tag_name, info)
  media_info.save()

Example 27

Project: groper
Source File: groper.py
View license
def OptionsMeta(print_func=None):
    '''Creates a private scope for the options manupulation functions and returns them.

    This function us used to create a module-wide global options object and its 
    manipulation functions. It may be used to generate local options objects, for 
    example for unit testing.
    '''

    print_func = print_func or print # Pass in a custom print function to use, e.g. stderr

    option_definitions = {}
    cp = RawConfigParser()
    adapters = {
        bool: cp.getboolean,
        float: cp.getfloat,
        int: cp.getint,
    }

    config_file_def = {
        'section': None,
        'optname': None,
        'filename': None,
    }

    # Variables we will return
    options = OptionObject()
    cmdargs = []
    cmdarg_defs = {
        'count': None,
        'args': None,
    }
    _type = type

    try:
        _basestring = basestring
    except NameError:
        _basestring = str

    try:
        _long = long
    except NameError:
        _long = int

    def generate_sample_config():
        '''Returns a string containing a sample configuration file based on the defined options.'''
        
        f = StringIO()
        try:
            for section in option_definitions:
                f.write('[{0}]\n'.format(section))

                for name, opt in option_definitions[section].items():
                    if opt.cmd_only:
                        continue

                    opt_name = name if hasattr(opt, 'default') else '#{0}'.format(name)
                    opt_val = '{0}'.format(opt.default) if hasattr(opt, 'default') else '<{0}>'.format(name.upper())
                    
                    f.write('{0} = {1}\n'.format(opt_name, opt_val))

                f.write("\n")

            return f.getvalue()
        finally:
            f.close()
        
    def _option_usage(option):
        '''Create an option usage line part based on option definition.
        
            Returns a tuple of (short_str, long_str) to be added.
        '''
        s, l = None, None

        wrap_optional = lambda option, s: s if option.required else ('[{0}]'.format(s))

        if option.cmd_short_name:
            if option.type != bool:
                s = wrap_optional(option, '-{0} <{1}>'.format(option.cmd_short_name, option.cmd_name or option.name))
            else:
                s = wrap_optional(option, '-{0}'.format(option.cmd_short_name))
        elif option.cmd_name and option.required:
            if option.type != bool:
                s = wrap_optional(option, '--{0}=<{1}>'.format(option.cmd_name, option.cmd_name or option.name))
            else:
                s = wrap_optional(option, '--{0}'.format(option.cmd_name))
       
        
        if option.cmd_name:
            if option.type != bool:
                l = wrap_optional(option, '--{0}=<{1}>'.format(option.cmd_name, option.cmd_name or option.name))
            else:
                l = wrap_optional(option, '--{0}'.format(option.cmd_name))
        elif option.cmd_short_name and option.required:
            if option.type != bool:
                l = wrap_optional(option, '-{0} <{1}>'.format((option.cmd_short_name, option.cmd_name or option.name)))
            else:
                l = wrap_optional(option, '-{0}'.format(option.cmd_short_name))

        return s, l

    def _args_usage(cmdargs_def):
        if cmdarg_defs['count'] == -1:
            return '[{0}] ...'.format(cmdarg_defs['args'][0])
        elif cmdarg_defs['count'] == -2:
            return '<{0}> [{1}] ...'.format(cmdarg_defs['args'][0], cmdarg_defs['args'][0])
        elif cmdarg_defs['args']:
            return ' '.join(['<{0}>'.format(s) for s in cmdarg_defs['args']])

    def usage(cmd_name=None):
        '''Returns usage/help string based on defined options.'''

        cmd_name = cmd_name or os.path.basename(sys.argv[0])
        
        lines = ['Usage:', '',]

        # Group all options
        cmd_options = {}
        for section in option_definitions:
            for name, opt in option_definitions[section].items():
                if opt.cmd_name or opt.cmd_short_name:
                    if opt.cmd_group not in cmd_options:
                        cmd_options[opt.cmd_group] = []
                    cmd_options[opt.cmd_group].append(opt)

        if not cmd_options and cmdarg_defs['count']:
            arg_line = _args_usage(cmdarg_defs)
            lines.append('{0} {1}'.format(cmd_name, arg_line))

        # Create lines
        for group in cmd_options.values():
            short_line = []
            long_line = []

            group.sort(key=lambda a: a.name) # Sort alphabetically
            group.sort(key=lambda a: int(a.required)) # Sort by required options first
            
            for option in group:
                s, l = _option_usage(option)
                if s:
                    short_line.append(s)
                if l:
                    long_line.append(l)

            arg_line = _args_usage(cmdarg_defs)

            if arg_line:
                short_line.append(arg_line)
                long_line.append(arg_line)

            if short_line:
                lines.append('{0} {1}'.format(cmd_name, ' '.join(short_line)))
            if long_line:
                lines.append('{0} {1}'.format(cmd_name, ' '.join(long_line)))

        return '\n'.join(lines)
 
    def define_args(args=None):
        '''Defines required/optional arguments.

        The args parameter can be in the following forms:
          - (num, name): num is the number of arguments expected, and name is the name
            to be printed when program usage is being shown.
            NOTE: num can be -1 for "0 or more agruments" and -2 for "one or more arguments"
          - (arg1, arg2, arg3): Require three arguments, each with a different name.
        '''

        if len(args) == 2 and type(args[0]) in set((int, _long)) and isinstance(args[1], _basestring):
            cmdarg_defs['count'] = args[0]
            cmdarg_defs['args'] = [args[1]] * abs(args[0])
            return
        elif hasattr(args, '__iter__'):
            cmdarg_defs['count'] = len(args)
            cmdarg_defs['args'] = tuple(args)
            return

        raise OptionsError('Define either (count, argname) (use -1 for zero or more, -2 for one or more) or a list of argument names.')

    def define_opt(section, name, cmd_name=None, cmd_short_name=None, cmd_only=False, type=_type(''), is_config_file=False, is_help=False, help=None, cmd_group='default', **kwargs):
        '''Defines an option. Should be run before init_options().
        
           Note that you may pass in one additional kwarg: default.
           If this argument is not specified, the option is required, and
           will have to be set from either a config file or the command line.
        '''

        if not isinstance(section, _basestring):
            raise OptionsError('Section name {0} must be a string, not a {1}'.format(section, _type(section)))

        if not isinstance(name, _basestring):
            raise OptionsError('Option name {0} must be a string, not a {1}'.format(name, _type(name)))

        if cmd_name and not isinstance(cmd_name, _basestring):
            raise OptionsError('cmd_name {0} must be a string, not a {1}'.format(cmd_name, _type(cmd_name)))

        if cmd_short_name and not isinstance(cmd_short_name, _basestring):
            raise OptionsError('cmd_short_name {0} must be a string, not a {1}'.format(cmd_short_name, _type(cmd_short_name)))

        section = section.lower().strip()
        name = name.lower().strip()
        if cmd_name:
            cmd_name = cmd_name.lower().strip()

        if not re.match('^[a-z_]+[a-z0-9_]*$', section):
            raise OptionsError('{0} is not a valid section name. It must contain only letters, numbers and underscores.'.format(section))
        
        if not re.match('^[a-z_]+[a-z0-9_]*$', name):
            raise OptionsError('{0} is not a valid name. It must contain only letters, numbers and underscores.'.format(name))

        if cmd_name and not re.match('^[a-z0-9]+[a-z0-9-]*$', cmd_name):
            raise OptionsError('{0} is not a valid cmd_name. It must contain only letters, numbers and dashes.'.format(cmd_short_name))

        if cmd_short_name and (len(cmd_short_name) != 1 or not re.match('^[a-zA-Z0-9]{1}$', cmd_short_name)):
            raise OptionsError('{0} is not a valid cmd_short_name. It must contain only letters or numbers and be of length 1.'.format(cmd_short_name))

        if not hasattr(options, section):
            setattr(options, section, OptionObject())
            option_definitions[section] = {}

        if name in option_definitions[section]:
            raise OptionsError('Option {0}.{1} is already defined.'.format(section, name))

        if cmd_only and not (cmd_name or cmd_short_name):
            raise OptionsError('Option {0}.{1} is defined as cmd_only, but neither cmd_name nor cmd_short_name are set.'.format(section, name))

        if is_config_file and not isinstance(type(), _basestring):
            raise OptionsError('Option {0}.{1} is defined as is_config_file, but with {2} instead of {3}.'.format(section, name, type, _type('')))

        if is_config_file and config_file_def['section']:
            raise OptionsError('Duplicate is_config_file options {0}.{1} and {2}.{3}.'.format(section, name, config_file_def['section'], config_file_def['name']))

        if is_config_file and not (cmd_name or cmd_short_name):
            raise OptionsError('Option {0}.{1} is defined as is_config_file, but cmd_name and cmd_short_name are not specified.'.format(section, name))

        if is_help and not isinstance(type(), bool):
            raise OptionsError('Option {0}.{1} is defined as is_help, but with {2} instead of {3}.'.format(section, name, type, bool))

        option_definitions[section][name] = OptionObject(
            section=section,
            name=name,
            cmd_name=cmd_name,
            cmd_short_name=cmd_short_name,
            required=False,
            type=type,
            is_config_file=is_config_file,
            is_help=is_help,
            cmd_group=cmd_group,
            cmd_only=cmd_only or is_config_file or is_help,
            set_by=None,
        )

        if 'default' in kwargs:
            option_definitions[section][name].default = kwargs['default']
        elif type == bool:
            option_definitions[section][name].default = False
        else:
            option_definitions[section][name].required = True
            

        if is_config_file:
            config_file_def['section'] = section
            config_file_def['optname'] = name
            
            if 'default' in kwargs:
                config_file_def['filename'] = kwargs['default']

    def parse_config(config_file=None):
        '''Parses a configuration file.
        
        This function sets option values if not already set by the parse_args() function.'''

        if not config_file:
            if not config_file_def['filename']:
                raise OptionsError('You must pass a config_file path to parse_config() or define a command line option is_config_file=True with an optional default.')
            config_file = config_file_def['filename']

        config_file = os.path.abspath(config_file)
        if not os.path.exists(config_file):
            raise OptionsUserError('Configuration file {0} does not exist.'.format(config_file))

        cp.readfp(codecs.open(config_file, 'r', 'utf-8'))

        for section in option_definitions:
            if cp.has_section(section):
                for name in option_definitions[section]:
                    if option_definitions[section][name].set_by is not None:
                        continue

                    if option_definitions[section][name].cmd_only:
                        continue

                    if name in option_definitions[section]:
                        opt = option_definitions[section][name]

                        try:
                            if opt.type in adapters:
                                setattr(getattr(options, section), name, adapters[opt.type](section, name))
                            else:
                                value = cp.get(section, name)
                                setattr(getattr(options, section), name, opt.type(value))
                        except ValueError as e:
                            print(e)
                            raise OptionsUserError('Could not parse configuration file {0}: section {1} option {2} must be of type {3}, not {4}'.format(config_file, section, name, opt.type.__name__, type(getattr(getattr(options, section), name))))
                        except NoOptionError:
                            if option_definitions[section][name].set_by or hasattr(option_definitions[section][name], 'default'):
                                continue
                            raise OptionsUserError('Could not parse configuration file {0}: section {1} option {2} was not found'.format(config_file, section, name))
                        option_definitions[section][name].set_by = parse_config

    def parse_args(argv):
        '''Parses command line arguments and sets option values as well as the cmdargs list.'''

        short_args = []
        long_args = []
        cmd_options = {}

        for section in option_definitions:
            for name, opt in option_definitions[section].items():
                if not opt.cmd_name and not opt.cmd_short_name:
                    continue

                if opt.cmd_name:
                    if opt.type == bool:
                        long_args.append(opt.cmd_name)
                    else:
                        long_args.append('{0}='.format(opt.cmd_name))
                    cmd_options['--{0}'.format(opt.cmd_name)] = opt

                if opt.cmd_short_name:
                    if opt.type == bool:
                        short_args.append(opt.cmd_short_name)
                    else:
                        short_args.append('{0}:'.format(opt.cmd_short_name))
                    cmd_options['-{0}'.format(opt.cmd_short_name)] = opt

        try:
            opts, args = getopt.getopt(argv, ''.join(short_args), long_args)
        except getopt.GetoptError as err:
            raise OptionsUserError(err)

        # Empty a non-local scope list, in case parse_args is called twice
        if len(cmdargs) > 0:
            [cmdargs.pop() for _ in range(len(cmdargs))]
        
        for arg in args:
            cmdargs.append(arg)

        for key, val in opts:
            if key in cmd_options:
                opt = cmd_options[key]
                if opt.is_help:
                    print_func(usage())
                    sys.exit(0)

                if opt.type == bool:
                    setattr(getattr(options, opt.section), opt.name, True)
                else:
                    try:
                        setattr(getattr(options, opt.section), opt.name, opt.type(val))
                    except ValueError:
                        raise OptionsUserError('Could not parse command line option {0}: it must be of type {1}.'.format(opt.name, opt.type.__name__))
                option_definitions[opt.section][opt.name].set_by = parse_args
            else:
                raise OptionsUserError('Unknown command line parameter {0}.'.format(key))

        if config_file_def['section'] and hasattr(getattr(options, config_file_def['section']), config_file_def['optname']):
            config_file_def['filename'] = getattr(getattr(options, config_file_def['section']), config_file_def['optname'])

    def init_options(argv=None, config_file=None):
        """Shortcut method for initializing all the options.

        Uses no configuration file unless a command line option has been defined 
        as is_config_file=True.
        """
        
        if argv is None:
            argv = sys.argv[1:]

        try:
            parse_args(argv)
            if config_file or config_file_def['filename']:
                parse_config(config_file)

            set_defaults()
            verify_all_options()

        except OptionsUserError as e:
            print_func(e)
            print_func('')
            print_func(usage())
            sys.exit(os.EX_USAGE)

    def set_defaults():
        '''Sets the default option values if they have not already been specified.'''

        for section in option_definitions:
            for name, opt in option_definitions[section].items():
                if not hasattr(option_definitions[section][name], 'default'):
                    continue

                if option_definitions[section][name].set_by is not None:
                    continue

                default = getattr(option_definitions[section][name], 'default')
                setattr(getattr(options, section), name, default)

    def verify_all_options():
        '''Raises an error if required options have not been specified by the user.'''

        if config_file_def['section'] and not config_file_def['filename']:
            option = option_definitions[config_file_def['section']][config_file_def['optname']]

            if option.cmd_name:
                error = 'Required command line option --{0} was not specified.'.format(option.cmd_name)
            elif option.cmd_short_name:
                error = 'Required command line option -{0} was not specified.'.format(option.cmd_short_name)
            raise OptionsUserError(error)

        errors = []
        for section in option_definitions:
            for name, opt in option_definitions[section].items():
                if option_definitions[section][name].required:
                    if not hasattr(getattr(options, section), name):

                        if not option_definitions[section][name].cmd_only:
                            final_words = ', and {0}.{1} could not be found in the config file.'.format(section, name)
                        else:
                            final_words = '.'

                        if option_definitions[section][name].cmd_name:
                            error = 'Required command line option --{0} was not specified{1}'.format(option_definitions[section][name].cmd_name, final_words)
                        elif option_definitions[section][name].cmd_short_name:
                            error = 'Required command line option -{0} was not specified{1}'.format(option_definitions[section][name].cmd_short_name, final_words)
                        else:
                            error = 'Required option {0}.{1} was not specified in the config file.'.format(section, name,)

                        errors.append(error)

        if cmdarg_defs['count'] == -1:
            pass # zero args required
        elif cmdarg_defs['count'] == -2:
            if len(cmdargs) < 1:
                errors.append('At least one <{0}> argument required.'.format(cmdarg_defs['args']))
        elif cmdarg_defs['args'] is not None:
            if len(cmdargs) != cmdarg_defs['count']:
                errors.append('Required arguments were not specified: {0}.'.format(' '.join(['<{0}>'.format(s) for s in cmdarg_defs['args']])))

        if len(errors) > 0:
            raise OptionsUserError('\n'.join(errors))

    return options, cmdargs, define_opt, define_args, parse_config, parse_args, set_defaults, verify_all_options, init_options, generate_sample_config, usage

Example 28

Project: yum-plugin-replace
Source File: replace.py
View license
    def doCommand(self, base, basecmd, extcmds):
        logger = logging.getLogger("yum.verbose.main")
        print "Replacing packages takes time, please be patient..."
        global pkgs_to_remove
        pkgs_to_install = []
        pkgs_to_not_remove = []
        deps_to_resolve = []
        pkgs_with_same_srpm = []

        def msg(x):
            logger.log(logginglevels.INFO_2, x)
        def msg_warn(x):
            logger.warn(x)


        opts = base.plugins.cmdline[0]
        if len(base.plugins.cmdline[1]) <= 1:
            raise UpdateError, "Must specify a package to be replaced (i.e yum replace pkg --replace-with pkgXY)"
        if not opts.replace_with:
            raise UpdateError, "Replacement package name required (--replace-with)" 

        orig_pkg = base.plugins.cmdline[1][1]
        new_pkg = opts.replace_with

        if not base.isPackageInstalled(orig_pkg):
            raise RemoveError, "Package '%s' is not installed." % orig_pkg

        # get pkg object
        res = base.rpmdb.searchNevra(name=orig_pkg)
        if len(res) > 1:
            raise RemoveError, \
                "Multiple packages found matching '%s'.  Please remove manually." % \
                orig_pkg
        orig_pkgobject = res[0]
        pkgs_to_remove.append(orig_pkgobject)

        # find all other installed packages with same srpm (orig_pkg's subpackages)
        for pkg in base.rpmdb:
            if pkg.sourcerpm == orig_pkgobject.sourcerpm:
                pkgs_to_remove.append(pkg)
                for dep in pkg.provides_names:
                    deps_to_resolve.append(dep)

        # get new pkg object
        new_pkgs = []
        res = base.pkgSack.returnNewestByName(new_pkg)
        for i in res:
            if platform.machine() == i.arch:
                if i not in new_pkgs:
                    new_pkgs.append(i)

        # if no archs matched (maybe a i686 / i386 issue) then pass them all
        if len(new_pkgs) == 0:
            new_pkgs = res

        # clean up duplicates, for some reason yum creates duplicate package objects
        # that are the same, but different object ref so they don't compare.  here
        # we compare against returnEVR().
        final_pkgs = []
        for i in new_pkgs:
            add = True 
            for i2 in final_pkgs:
                if i.returnEVR() == i2.returnEVR() and i.arch == i2.arch:
                    add = False
            if add and i not in final_pkgs:
                final_pkgs.append(i)

        if len(final_pkgs) > 1:
            raise UpdateError, \
                "Multiple packages found matching '%s'.  Please upgrade manually." % \
                new_pkg
        new_pkgobject = new_pkgs[0]
        pkgs_to_install.append(new_pkgobject)

        orig_prefix = orig_pkg
        new_prefix = new_pkg

        # Find the original and new prefixes of packages based on their sourcerpm name
        m = re.match('(.*)-%s-%s' % (orig_pkgobject.version, orig_pkgobject.release),\
            orig_pkgobject.sourcerpm)
        if m:
            orig_prefix = m.group(1)

        m = re.match('(.*)-%s-%s' % (new_pkgobject.version, new_pkgobject.release),\
            new_pkgobject.sourcerpm)
        if m:
            new_prefix = m.group(1)

        # don't remove pkgs that rely on orig_pkg (yum tries to remove them)
        for pkg in base.rpmdb:
            for req in pkg.requires_names:
                if req in deps_to_resolve:
                    if pkg not in pkgs_to_not_remove and pkg not in pkgs_to_remove:
                        pkgs_to_not_remove.append(pkg)

        # determine all new_pkg subpackages that provide missing deps
        providers = {}
        for pkg in base.pkgSack:
            if pkg.sourcerpm == new_pkgobject.sourcerpm:
                pkgs_with_same_srpm.append(pkg)
                for dep in pkg.provides_names:
                    if dep in deps_to_resolve:
                        if pkg not in pkgs_to_remove:
                            
                            # build a list of all packages matching provides
                            if not providers.has_key(dep):
                                providers[dep] = []
                            providers[dep].append(pkg)

        # We now have a complete list of package providers we care about
        if providers:
            resolved = False
            for key, pkgs in providers.items():
                if len(pkgs) == 1:
                    pkgs_to_install.append(pkgs[0])
                    deps_to_resolve.remove(key)
                    resolved = True

                elif len(pkgs) > 1:
                    # Attempt to auto resolve multiple provides
                    for rpkg in pkgs_to_remove:
                        npkg = rpkg.name.replace(orig_prefix, new_prefix)
                        for pkg in pkgs:
                            if npkg == pkg.name:
                                pkgs_to_install.append(pkg)
                                resolved = True

                    # we've completed our auto resolve,
                    # if resolved lets remove the key
                    if resolved:
                        deps_to_resolve.remove(key)

                    if not resolved:
                        print '\nWARNING: Multiple Providers found for %s' % key
                        print "  %s" % [str(i) for i in pkgs]

                # remove the dep from dict since it should be handled.
                del(providers[key])

        # This is messy: determine if any of the pkgs_to_not_remove have
        # counterparts as part of same 'base name' set (but different srpm, i.e. 
        # php and php-pear has different source rpms but you want phpXY-pear too).
        while pkgs_to_not_remove:
            pkg = pkgs_to_not_remove.pop()
            m = re.match('%s-(.*)' % orig_prefix, pkg.name)
            if not m:
                continue
            replace_name = "%s-%s" % (new_prefix, m.group(1))
            for pkg2 in base.pkgSack: 
                if pkg2.name == replace_name:
                    if pkg not in pkgs_to_remove:
                        pkgs_to_remove.append(pkg)
                        if pkg in pkgs_to_not_remove:
                            pkgs_to_not_remove.remove(pkg)
                    if pkg2 not in pkgs_to_install:
                        pkgs_to_install.append(pkg2)
                        if pkg2 in pkgs_to_not_remove:
                            pkgs_to_not_remove.remove(pkg2)

        # clean up duplicates (multiple versions)
        _pkgs_to_install = []
        for pkg in pkgs_to_install:
            latest_pkg = base.pkgSack.returnNewestByName(pkg.name)[0]
            if latest_pkg not in _pkgs_to_install:
                _pkgs_to_install.append(latest_pkg)
        pkgs_to_install = _pkgs_to_install

        # Its common that newer/replacement packages won't provide all the same things.
        # Give the user the chance to bail if they are scared.
        if len(deps_to_resolve) > 0:
            print
            print "WARNING: Unable to resolve all providers: %s" % deps_to_resolve
            print
            if not opts.assumeyes:
                res = raw_input("This may be normal depending on the package.  Continue? [y/N] ")
                if not res.strip('\n').lower() in ['y', 'yes']:
                    sys.exit(1)
        
        # remove old packages
        for pkg in pkgs_to_remove:
            base.remove(pkg)
        # install new/replacement packages
        for pkg in pkgs_to_install:
            base.install(pkg) 

        return 2, ["Run transaction to replace '%s' with '%s'" % (orig_pkg, new_pkg)]

Example 29

Project: bacpypes
Source File: pdu.py
View license
    def decode_address(self, addr):
        """Initialize the address from a string.  Lots of different forms are supported."""
        if _debug: Address._debug("decode_address %r (%s)", addr, type(addr))

        # start out assuming this is a local station
        self.addrType = Address.localStationAddr
        self.addrNet = None

        if addr == "*":
            if _debug: Address._debug("    - localBroadcast")

            self.addrType = Address.localBroadcastAddr
            self.addrNet = None
            self.addrAddr = None
            self.addrLen = None

        elif addr == "*:*":
            if _debug: Address._debug("   - globalBroadcast")

            self.addrType = Address.globalBroadcastAddr
            self.addrNet = None
            self.addrAddr = None
            self.addrLen = None

        elif isinstance(addr, int):
            if _debug: Address._debug("    - int")
            if (addr < 0) or (addr >= 256):
                raise ValueError("address out of range")

            self.addrAddr = struct.pack('B', addr)
            self.addrLen = 1

        elif isinstance(addr, str):
            if _debug: Address._debug("    - str")

            m = ip_address_mask_port_re.match(addr)
            if m:
                if _debug: Address._debug("    - IP address")

                net, addr, mask, port = m.groups()
                if not mask: mask = '32'
                if not port: port = '47808'
                if _debug: Address._debug("    - net, addr, mask, port: %r, %r, %r, %r", net, addr, mask, port)

                if net:
                    net = int(net)
                    if (net >= 65535):
                        raise ValueError("network out of range")
                    self.addrType = Address.remoteStationAddr
                    self.addrNet = net

                self.addrPort = int(port)
                self.addrTuple = (addr, self.addrPort)

                addrstr = socket.inet_aton(addr)
                self.addrIP = struct.unpack('!L', addrstr)[0]
                self.addrMask = (_long_mask << (32 - int(mask))) & _long_mask
                self.addrHost = (self.addrIP & ~self.addrMask)
                self.addrSubnet = (self.addrIP & self.addrMask)

                bcast = (self.addrSubnet | ~self.addrMask)
                self.addrBroadcastTuple = (socket.inet_ntoa(struct.pack('!L', bcast & _long_mask)), self.addrPort)

                self.addrAddr = addrstr + struct.pack('!H', self.addrPort & _short_mask)
                self.addrLen = 6

            elif ethernet_re.match(addr):
                if _debug: Address._debug("    - ethernet")

                self.addrAddr = xtob(addr, ':')
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+$", addr):
                if _debug: Address._debug("    - int")

                addr = int(addr)
                if (addr > 255):
                    raise ValueError("address out of range")

                self.addrAddr = struct.pack('B', addr)
                self.addrLen = 1

            elif re.match(r"^\d+:[*]$", addr):
                if _debug: Address._debug("    - remote broadcast")

                addr = int(addr[:-2])
                if (addr >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteBroadcastAddr
                self.addrNet = addr
                self.addrAddr = None
                self.addrLen = None

            elif re.match(r"^\d+:\d+$",addr):
                if _debug: Address._debug("    - remote station")

                net, addr = addr.split(':')
                net = int(net)
                addr = int(addr)
                if (net >= 65535):
                    raise ValueError("network out of range")
                if (addr > 255):
                    raise ValueError("address out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = struct.pack('B', addr)
                self.addrLen = 1

            elif re.match(r"^0x([0-9A-Fa-f][0-9A-Fa-f])+$",addr):
                if _debug: Address._debug("    - modern hex string")

                self.addrAddr = xtob(addr[2:])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^X'([0-9A-Fa-f][0-9A-Fa-f])+'$",addr):
                if _debug: Address._debug("    - old school hex string")

                self.addrAddr = xtob(addr[2:-1])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+:0x([0-9A-Fa-f][0-9A-Fa-f])+$",addr):
                if _debug: Address._debug("    - remote station with modern hex string")

                net, addr = addr.split(':')
                net = int(net)
                if (net >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = xtob(addr[2:])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+:X'([0-9A-Fa-f][0-9A-Fa-f])+'$",addr):
                if _debug: Address._debug("    - remote station with old school hex string")

                net, addr = addr.split(':')
                net = int(net)
                if (net >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = xtob(addr[2:-1])
                self.addrLen = len(self.addrAddr)

            else:
                raise ValueError("unrecognized format")

        elif isinstance(addr, tuple):
            addr, port = addr
            self.addrPort = int(port)

            if isinstance(addr, str):
                if not addr:
                    # when ('', n) is passed it is the local host address, but that
                    # could be more than one on a multihomed machine, the empty string
                    # means "any".
                    addrstr = b'\0\0\0\0'
                else:
                    addrstr = socket.inet_aton(addr)
                self.addrTuple = (addr, self.addrPort)

            elif isinstance(addr, (int, long)):
                addrstr = struct.pack('!L', addr & _long_mask)
                self.addrTuple = (socket.inet_ntoa(addrstr), self.addrPort)

            else:
                raise TypeError("tuple must be (string, port) or (long, port)")
            if _debug: Address._debug("    - addrstr: %r", addrstr)

            self.addrIP = struct.unpack('!L', addrstr)[0]
            self.addrMask = _long_mask
            self.addrHost = None
            self.addrSubnet = None
            self.addrBroadcastTuple = self.addrTuple

            self.addrAddr = addrstr + struct.pack('!H', self.addrPort & _short_mask)
            self.addrLen = 6

        else:
            raise TypeError("integer, string or tuple required")

Example 30

Project: bacpypes
Source File: pdu.py
View license
    def decode_address(self, addr):
        """Initialize the address from a string.  Lots of different forms are supported."""
        if _debug: Address._debug("decode_address %r (%s)", addr, type(addr))

        # start out assuming this is a local station
        self.addrType = Address.localStationAddr
        self.addrNet = None

        if addr == "*":
            if _debug: Address._debug("    - localBroadcast")

            self.addrType = Address.localBroadcastAddr
            self.addrNet = None
            self.addrAddr = None
            self.addrLen = None

        elif addr == "*:*":
            if _debug: Address._debug("   - globalBroadcast")

            self.addrType = Address.globalBroadcastAddr
            self.addrNet = None
            self.addrAddr = None
            self.addrLen = None

        elif isinstance(addr, int):
            if _debug: Address._debug("    - int")
            if (addr < 0) or (addr >= 256):
                raise ValueError("address out of range")

            self.addrAddr = struct.pack('B', addr)
            self.addrLen = 1

        elif isinstance(addr, (bytes, bytearray)):
            if _debug: Address._debug("    - bytes or bytearray")

            self.addrAddr = bytes(addr)
            self.addrLen = len(addr)

            if self.addrLen == 6:
                self.addrIP = struct.unpack('!L', addr[:4])[0]
                self.addrMask = (1 << 32) - 1
                self.addrHost = (self.addrIP & ~self.addrMask)
                self.addrSubnet = (self.addrIP & self.addrMask)
                self.addrPort = struct.unpack(">H", addr[4:])[0]

                self.addrTuple = (socket.inet_ntoa(addr[:4]), self.addrPort)
                self.addrBroadcastTuple = ('255.255.255.255', self.addrPort)

        elif isinstance(addr, str):
            if _debug: Address._debug("    - str")

            m = ip_address_mask_port_re.match(addr)
            if m:
                if _debug: Address._debug("    - IP address")

                net, addr, mask, port = m.groups()
                if not mask: mask = '32'
                if not port: port = '47808'
                if _debug: Address._debug("    - net, addr, mask, port: %r, %r, %r, %r", net, addr, mask, port)

                if net:
                    net = int(net)
                    if (net >= 65535):
                        raise ValueError("network out of range")
                    self.addrType = Address.remoteStationAddr
                    self.addrNet = net

                self.addrPort = int(port)
                self.addrTuple = (addr, self.addrPort)

                addrstr = socket.inet_aton(addr)
                self.addrIP = struct.unpack('!L', addrstr)[0]
                self.addrMask = (_long_mask << (32 - int(mask))) & _long_mask
                self.addrHost = (self.addrIP & ~self.addrMask)
                self.addrSubnet = (self.addrIP & self.addrMask)

                bcast = (self.addrSubnet | ~self.addrMask)
                self.addrBroadcastTuple = (socket.inet_ntoa(struct.pack('!L', bcast & _long_mask)), self.addrPort)

                self.addrAddr = addrstr + struct.pack('!H', self.addrPort & _short_mask)
                self.addrLen = 6

            elif ethernet_re.match(addr):
                if _debug: Address._debug("    - ethernet")

                self.addrAddr = xtob(addr, ':')
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+$", addr):
                if _debug: Address._debug("    - int")

                addr = int(addr)
                if (addr > 255):
                    raise ValueError("address out of range")

                self.addrAddr = struct.pack('B', addr)
                self.addrLen = 1

            elif re.match(r"^\d+:[*]$", addr):
                if _debug: Address._debug("    - remote broadcast")

                addr = int(addr[:-2])
                if (addr >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteBroadcastAddr
                self.addrNet = addr
                self.addrAddr = None
                self.addrLen = None

            elif re.match(r"^\d+:\d+$",addr):
                if _debug: Address._debug("    - remote station")

                net, addr = addr.split(':')
                net = int(net)
                addr = int(addr)
                if (net >= 65535):
                    raise ValueError("network out of range")
                if (addr > 255):
                    raise ValueError("address out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = struct.pack('B', addr)
                self.addrLen = 1

            elif re.match(r"^0x([0-9A-Fa-f][0-9A-Fa-f])+$",addr):
                if _debug: Address._debug("    - modern hex string")

                self.addrAddr = xtob(addr[2:])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^X'([0-9A-Fa-f][0-9A-Fa-f])+'$",addr):
                if _debug: Address._debug("    - old school hex string")

                self.addrAddr = xtob(addr[2:-1])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+:0x([0-9A-Fa-f][0-9A-Fa-f])+$",addr):
                if _debug: Address._debug("    - remote station with modern hex string")

                net, addr = addr.split(':')
                net = int(net)
                if (net >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = xtob(addr[2:])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+:X'([0-9A-Fa-f][0-9A-Fa-f])+'$",addr):
                if _debug: Address._debug("    - remote station with old school hex string")

                net, addr = addr.split(':')
                net = int(net)
                if (net >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = xtob(addr[2:-1])
                self.addrLen = len(self.addrAddr)

            else:
                raise ValueError("unrecognized format")

        elif isinstance(addr, tuple):
            addr, port = addr
            self.addrPort = int(port)

            if isinstance(addr, str):
                if not addr:
                    # when ('', n) is passed it is the local host address, but that
                    # could be more than one on a multihomed machine, the empty string
                    # means "any".
                    addrstr = b'\0\0\0\0'
                else:
                    addrstr = socket.inet_aton(addr)
                self.addrTuple = (addr, self.addrPort)

            elif isinstance(addr, int):
                addrstr = struct.pack('!L', addr & _long_mask)
                self.addrTuple = (socket.inet_ntoa(addrstr), self.addrPort)

            else:
                raise TypeError("tuple must be (string, port) or (long, port)")
            if _debug: Address._debug("    - addrstr: %r", addrstr)

            self.addrIP = struct.unpack('!L', addrstr)[0]
            self.addrMask = _long_mask
            self.addrHost = None
            self.addrSubnet = None
            self.addrBroadcastTuple = self.addrTuple

            self.addrAddr = addrstr + struct.pack('!H', self.addrPort & _short_mask)
            self.addrLen = 6
        else:
            raise TypeError("integer, string or tuple required")

Example 31

Project: bacpypes
Source File: pdu.py
View license
    def decode_address(self, addr):
        """Initialize the address from a string.  Lots of different forms are supported."""
        if _debug: Address._debug("decode_address %r (%s)", addr, type(addr))

        # start out assuming this is a local station
        self.addrType = Address.localStationAddr
        self.addrNet = None

        if addr == "*":
            if _debug: Address._debug("    - localBroadcast")

            self.addrType = Address.localBroadcastAddr
            self.addrNet = None
            self.addrAddr = None
            self.addrLen = None

        elif addr == "*:*":
            if _debug: Address._debug("   - globalBroadcast")

            self.addrType = Address.globalBroadcastAddr
            self.addrNet = None
            self.addrAddr = None
            self.addrLen = None

        elif isinstance(addr, int):
            if _debug: Address._debug("    - int")
            if (addr < 0) or (addr >= 256):
                raise ValueError("address out of range")

            self.addrAddr = struct.pack('B', addr)
            self.addrLen = 1

        elif isinstance(addr, (bytes, bytearray)):
            if _debug: Address._debug("    - bytes or bytearray")

            self.addrAddr = bytes(addr)
            self.addrLen = len(addr)

            if self.addrLen == 6:
                self.addrIP = struct.unpack('!L', addr[:4])[0]
                self.addrMask = (1 << 32) - 1
                self.addrHost = (self.addrIP & ~self.addrMask)
                self.addrSubnet = (self.addrIP & self.addrMask)
                self.addrPort = struct.unpack(">H", addr[4:])[0]

                self.addrTuple = (socket.inet_ntoa(addr[:4]), self.addrPort)
                self.addrBroadcastTuple = ('255.255.255.255', self.addrPort)

        elif isinstance(addr, str):
            if _debug: Address._debug("    - str")

            m = ip_address_mask_port_re.match(addr)
            if m:
                if _debug: Address._debug("    - IP address")

                net, addr, mask, port = m.groups()
                if not mask: mask = '32'
                if not port: port = '47808'
                if _debug: Address._debug("    - net, addr, mask, port: %r, %r, %r, %r", net, addr, mask, port)

                if net:
                    net = int(net)
                    if (net >= 65535):
                        raise ValueError("network out of range")
                    self.addrType = Address.remoteStationAddr
                    self.addrNet = net

                self.addrPort = int(port)
                self.addrTuple = (addr, self.addrPort)

                addrstr = socket.inet_aton(addr)
                self.addrIP = struct.unpack('!L', addrstr)[0]
                self.addrMask = (_long_mask << (32 - int(mask))) & _long_mask
                self.addrHost = (self.addrIP & ~self.addrMask)
                self.addrSubnet = (self.addrIP & self.addrMask)

                bcast = (self.addrSubnet | ~self.addrMask)
                self.addrBroadcastTuple = (socket.inet_ntoa(struct.pack('!L', bcast & _long_mask)), self.addrPort)

                self.addrAddr = addrstr + struct.pack('!H', self.addrPort & _short_mask)
                self.addrLen = 6

            elif ethernet_re.match(addr):
                if _debug: Address._debug("    - ethernet")

                self.addrAddr = xtob(addr, ':')
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+$", addr):
                if _debug: Address._debug("    - int")

                addr = int(addr)
                if (addr > 255):
                    raise ValueError("address out of range")

                self.addrAddr = struct.pack('B', addr)
                self.addrLen = 1

            elif re.match(r"^\d+:[*]$", addr):
                if _debug: Address._debug("    - remote broadcast")

                addr = int(addr[:-2])
                if (addr >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteBroadcastAddr
                self.addrNet = addr
                self.addrAddr = None
                self.addrLen = None

            elif re.match(r"^\d+:\d+$",addr):
                if _debug: Address._debug("    - remote station")

                net, addr = addr.split(':')
                net = int(net)
                addr = int(addr)
                if (net >= 65535):
                    raise ValueError("network out of range")
                if (addr > 255):
                    raise ValueError("address out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = struct.pack('B', addr)
                self.addrLen = 1

            elif re.match(r"^0x([0-9A-Fa-f][0-9A-Fa-f])+$",addr):
                if _debug: Address._debug("    - modern hex string")

                self.addrAddr = xtob(addr[2:])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^X'([0-9A-Fa-f][0-9A-Fa-f])+'$",addr):
                if _debug: Address._debug("    - old school hex string")

                self.addrAddr = xtob(addr[2:-1])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+:0x([0-9A-Fa-f][0-9A-Fa-f])+$",addr):
                if _debug: Address._debug("    - remote station with modern hex string")

                net, addr = addr.split(':')
                net = int(net)
                if (net >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = xtob(addr[2:])
                self.addrLen = len(self.addrAddr)

            elif re.match(r"^\d+:X'([0-9A-Fa-f][0-9A-Fa-f])+'$",addr):
                if _debug: Address._debug("    - remote station with old school hex string")

                net, addr = addr.split(':')
                net = int(net)
                if (net >= 65535):
                    raise ValueError("network out of range")

                self.addrType = Address.remoteStationAddr
                self.addrNet = net
                self.addrAddr = xtob(addr[2:-1])
                self.addrLen = len(self.addrAddr)

            else:
                raise ValueError("unrecognized format")

        elif isinstance(addr, tuple):
            addr, port = addr
            self.addrPort = int(port)

            if isinstance(addr, str):
                if not addr:
                    # when ('', n) is passed it is the local host address, but that
                    # could be more than one on a multihomed machine, the empty string
                    # means "any".
                    addrstr = b'\0\0\0\0'
                else:
                    addrstr = socket.inet_aton(addr)
                self.addrTuple = (addr, self.addrPort)

            elif isinstance(addr, int):
                addrstr = struct.pack('!L', addr & _long_mask)
                self.addrTuple = (socket.inet_ntoa(addrstr), self.addrPort)

            else:
                raise TypeError("tuple must be (string, port) or (long, port)")
            if _debug: Address._debug("    - addrstr: %r", addrstr)

            self.addrIP = struct.unpack('!L', addrstr)[0]
            self.addrMask = _long_mask
            self.addrHost = None
            self.addrSubnet = None
            self.addrBroadcastTuple = self.addrTuple

            self.addrAddr = addrstr + struct.pack('!H', self.addrPort & _short_mask)
            self.addrLen = 6
        else:
            raise TypeError("integer, string or tuple required")

Example 32

Project: biocode
Source File: batch_attributor.py
View license
def main():
    parser = argparse.ArgumentParser( description='Generates a shell script of Attributor runs given a range of pipeline IDs')

    ## output file to be written
    parser.add_argument('-o', '--output_file', type=str, required=True, help='Path to an output shell file to be created' )
    parser.add_argument('-od', '--output_directory', type=str, required=True, help='Directory where output files will be written' )
    args = parser.parse_args()

    # Either set this variable to a path, or use the pipeline_min/pipeline_max below
    #  If you dont' want to use this, set it to None
    srs_id_list_file = '/path/to/some.id.list'
    srs_ids_to_keep = list()

    # inclusive.  These are NOT read if the srs_id_list_file is defined above
    #pipeline_min = 10927248802
    #pipeline_max = 11214274766
    pipeline_min = 11305294429
    pipeline_max = 11373273817

    # CONFIG
    project_area = '/local/hmp/dacc/t3/'
    owners = ['hhuot', 'cmccracken']
    config_template = '/home/hhuot/git/Attributor/assign_functional_annotation.example_hhc.config'
    # assumes all child names are like: SRS147134.rapsearch2.m8.gz
    rapsearch2_base = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmgi/rapsearch'
    attributor_path = '/home/hhuot/git/Attributor/assign_functional_annotation.py'
    # names are like either SRS045739.metagenemark.gff3 or SRS045763.metagenemark3.gff3
    gff_base_dir = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmorf/gff3'
    # names are like either SRS018791.metagenemark.faa or SRS018794.metagenemark3.faa
    fasta_base_dir = '/local/hmp/dacc/t3/hhs/genome/microbiome/wgs/analysis/hmorf/faa'

    # key = pipeline ID, value = SRS ID
    pipelines = dict()

    # We're either getting our SRS ID list to keep from a file or a range of pipeline IDs
    if srs_id_list_file is not None:
        for line in open(srs_id_list_file):
            line = line.rstrip()
            srs_ids_to_keep.append(line)

    ## make sure the config template exists
    if not os.path.exists(config_template):
        raise Exception("ERROR: config template file not found: {0}".format(config_template))

    for pipeline_id in os.listdir("{0}/workflow/runtime/pipeline".format(project_area)):
        m = re.match("^\d+$", pipeline_id)
        if m:
            pipeline_id = int(pipeline_id)
            if srs_id_list_file is None:
                if pipeline_id > pipeline_max or pipeline_id < pipeline_min:
                    continue
        else:
            continue

        pipeline_path = "{0}/workflow/runtime/pipeline/{1}/pipeline.xml".format(project_area, pipeline_id)

        owner = find_owner(pipeline_path)
        if owner not in owners:
            continue

        pipeline_comment_path = "{0}/workflow/runtime/pipeline/{1}/pipeline.xml.comment".format(project_area, pipeline_id)
        if os.path.exists(pipeline_comment_path):
            comment = open(pipeline_comment_path).read()

            m = re.search("(SRS\d+)", comment)
            if m:
                srs_id = m.group(1)
                pipelines[pipeline_id] = srs_id
            else:
                m = re.search("(\d+\..{2,4})", comment)
                if m:
                    srs_id = m.group(1)
                    pipelines[pipeline_id] = srs_id
                else:
                    print("WARNING: Pipeline skipped without an SRS comment: {0}".format(pipeline_comment_path), file=sys.stderr)

    print("INFO: found {0} pipelines".format(len(pipelines)))

    batch_fh = open(args.output_file, 'wt')

    for pipeline_id in pipelines:
        # do we have a limit on the SRS IDs?
        if srs_id_list_file is not None:
            if pipelines[pipeline_id] not in srs_ids_to_keep:
                continue
        
        config_path = "{0}/{1}.config".format(args.output_directory, pipelines[pipeline_id])
        tmp_config_path = "/tmp/{0}.config".format(pipeline_id)

        ## get the source FASTA path
        if os.path.exists("{0}/{1}.metagenemark.faa".format(fasta_base_dir, pipelines[pipeline_id])):
            fasta_path = "{0}/{1}.metagenemark.faa".format(fasta_base_dir, pipelines[pipeline_id])
        elif os.path.exists("{0}/{1}.metagenemark3.faa".format(fasta_base_dir, pipelines[pipeline_id])):
            fasta_path = "{0}/{1}.metagenemark3.faa".format(fasta_base_dir, pipelines[pipeline_id])
        else:
            print("WARNING: failed to find FASTA file for {0}".format(pipelines[pipeline_id]), file=sys.stderr)
            continue
            #raise Exception("ERROR: failed to find FASTA file for {0}".format(pipelines[pipeline_id]))

        ## get the source GFF path
        if os.path.exists("{0}/{1}.metagenemark.gff3".format(gff_base_dir, pipelines[pipeline_id])):
            gff3_path = "{0}/{1}.metagenemark.gff3".format(gff_base_dir, pipelines[pipeline_id])
        elif os.path.exists("{0}/{1}.metagenemark3.gff3".format(gff_base_dir, pipelines[pipeline_id])):
            gff3_path = "{0}/{1}.metagenemark3.gff3".format(gff_base_dir, pipelines[pipeline_id])
        else:
            print("WARNING: failed to find GFF3 file for {0}".format(pipelines[pipeline_id]), file=sys.stderr)
            continue
            #raise Exception("ERROR: failed to find GFF3 file for {0}".format(pipelines[pipeline_id]))

        ## copy the config file
        shutil.copy(config_template, config_path)

        ## modify it with these paths
        ofh = open(tmp_config_path, 'wt')
        last_label = None

        for line in open(config_path):
            line = line.rstrip()
            m = re.match("   - label: (.+)", line)
            if m:
                last_label = m.group(1)

            m = re.match("     path: (.+)", line)
            if m:
                if last_label == 'coding_hmm_lib__equivalog':
                    ofh.write("     path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'rapsearch2__uniref100__all_full':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'rapsearch2__uniref100__all_partial':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'rapsearch2__uniref100__trusted_full':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'rapsearch2__uniref100__trusted_partial':
                    ofh.write("     path: {0}/{1}.rapsearch2.m8\n".format(rapsearch2_base, pipelines[pipeline_id]))
                    continue
                elif last_label == 'coding_hmm_lib__equivalog_domain':
                    ofh.write("     path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'coding_hmm_lib__all_trusted':
                    ofh.write("     path: {0}/output_repository/hmmpfam3/{1}_default/hmmpfam3.htab.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'tmhmm':
                    ofh.write("     path: {0}/output_repository/tmhmm/{1}_default/tmhmm.raw.list\n".format(project_area, pipeline_id))
                    continue
                elif last_label == 'lipoprotein_motif':
                    ofh.write("     path: {0}/output_repository/lipoprotein_motif/{1}_default/lipoprotein_motif.bsml.list\n".format(project_area, pipeline_id))
                    continue

            m = re.match("   polypeptide_fasta: (\S+)", line)
            if m:
                ofh.write("   polypeptide_fasta: {0}\n".format(fasta_path))
                continue

            m = re.match("   gff3: (\S+)", line)
            if m:
                ofh.write("   gff3: {0}\n".format(gff3_path))
                continue

            ofh.write("{0}\n".format(line))

        ofh.close()
        shutil.move(tmp_config_path, config_path)
        batch_fh.write("{0} -f gff3 -c {1} -o {2}/{3}\n".format(attributor_path, config_path, args.output_directory,
                                                        pipelines[pipeline_id]))

Example 33

Project: PyHAML
Source File: parse.py
View license
    def _parse_statement(self):

        line = self._peek_buffer()

        # Escaping.
        if line.startswith('\\'):
            return (
                nodes.Content(line[1:]),
                ''
            )

        # HTML comments.
        m = re.match(r'/(\[if[^\]]+])?(.*)$', line)
        if m:
            return (
                nodes.HTMLComment(m.group(2).strip(), (m.group(1) or '').rstrip()),
                ''
            )

        # Expressions.
        m = re.match(r'''
            (&?)                  # HTML escaping flag
            =
            (?:\|(\w+(?:,\w+)*))? # mako filters
            \s*
            (.*)                  # expression content
            $
        ''', line, re.X)
        if m:
            add_escape, filters, content = m.groups()
            filters = filters or ''
            if add_escape:
                filters = filters + (',' if filters else '') + 'h'
            return (
                nodes.Expression(content, filters),
                ''
            )

        # SASS Mixins
        m = re.match(r'@(\w+)', line)
        if m:
            name = m.group(1)
            line = line[m.end():]
            self._replace_buffer(line)
            argspec = self._match_python_expr(first=set('('), last=set(')'))
            if argspec:
                argspec = argspec[1:-1]
                line = self._peek_buffer()
            return (
                nodes.MixinDef(name, argspec),
                line
            )

        m = re.match(r'\+([\w.]+)', line)
        if m:
            name = m.group(1)
            line = line[m.end():]
            self._replace_buffer(line)
            argspec = self._match_python_expr(first=set('('), last=set(')'))
            if argspec:
                argspec = argspec[1:-1]
                line = self._peek_buffer()
            return (
                nodes.MixinCall(name, argspec),
                line
            )

        # HAML Filters.
        m = re.match(r':(\w+)(?:\s+(.+))?$', line)
        if m:
            filter, content = m.groups()
            return (
                nodes.Filter(content, filter),
                ''
            )

        # HAML comments
        if line.startswith('-#'):
            return (
                nodes.HAMLComment(line[2:].lstrip()),
                ''
            ) 
        
        # XML Doctype
        if line.startswith('!!!'):
            return (
                nodes.Doctype(*line[3:].strip().split()),
                ''
            )

        # Tags.
        m = re.match(r'''
            (?:%(%?(?:\w+:)?[\w-]*))? # tag name. the extra % is for mako
            (?:
              \[(.+?)(?:,(.+?))?\]    # object reference and prefix
            )? 
            (                         
              (?:\#[\w-]+|\.[\w-]+)+  # id/class
            )?
        ''', line, re.X)                                
        # The match only counts if we have a tag name or id/class.
        if m and (m.group(1) is not None or m.group(4)):
            name, object_reference, object_reference_prefix, raw_id_class = m.groups()
            
            # Extract id value and class list.
            id, class_ = None, []
            for m2 in re.finditer(r'(#|\.)([\w-]+)', raw_id_class or ''):
                type, value = m2.groups()
                if type == '#':
                    id = value
                else:
                    class_.append(value)
            line = line[m.end():]

            # Extract the kwargs expression.
            self._replace_buffer(line)
            kwargs_expr = self._match_python_expr(first=set('('), last=set(')'))
            if kwargs_expr:
                kwargs_expr = kwargs_expr[1:-1]
                line = self._peek_buffer()

            # Whitespace stripping
            m2 = re.match(r'([<>]+)', line)
            strip_outer = strip_inner = False
            if m2:
                strip_outer = '>' in m2.group(1)
                strip_inner = '<' in m2.group(1)
                line = line[m2.end():]

            # Self-closing tags
            self_closing = bool(line and line[0] == '/')
            line = line[int(self_closing):].lstrip()

            return (
                nodes.Tag(
                    name=name,
                    id=id,
                    class_=' '.join(class_),
                    
                    kwargs_expr=kwargs_expr,
                    object_reference=object_reference,
                    object_reference_prefix=object_reference_prefix, 
                    self_closing=self_closing,
                    strip_inner=strip_inner,
                    strip_outer=strip_outer,
                ),
                line
            )

        # Control statements.
        m = re.match(r'''
            -
            \s*
            (for|if|while|elif) # control type
        ''', line, re.X)
        if m:
            control_type = m.group(1)
            self._replace_buffer(line[m.end():].lstrip())
            expr = self._match_python_expr(last=set(':'))
            return (
                nodes.Control(control_type, expr[:-1]),
                self._peek_buffer()
            )
        m = re.match(r'-\s*else\s*:', line, re.X)
        if m:
            return (
                nodes.Control('else', None),
                line[m.end():].lstrip()
            )
        
        # Python source.
        if line.startswith('-'):
            if line.startswith('-!'):
                return (
                    nodes.Python(line[2:].lstrip(), module=True),
                    ''
                )
            else:
                return (
                    nodes.Python(line[1:].lstrip(), module=False),
                    ''
                )

        # Content
        return (
            nodes.Content(line),
            ''
        )

Example 34

Project: PyHAML
Source File: parse.py
View license
    def _parse_statement(self):

        line = self._peek_buffer()

        # Escaping.
        if line.startswith('\\'):
            return (
                nodes.Content(line[1:]),
                ''
            )

        # HTML comments.
        m = re.match(r'/(\[if[^\]]+])?(.*)$', line)
        if m:
            return (
                nodes.HTMLComment(m.group(2).strip(), (m.group(1) or '').rstrip()),
                ''
            )

        # Expressions.
        m = re.match(r'''
            (&?)                  # HTML escaping flag
            =
            (?:\|(\w+(?:,\w+)*))? # mako filters
            \s*
            (.*)                  # expression content
            $
        ''', line, re.X)
        if m:
            add_escape, filters, content = m.groups()
            filters = filters or ''
            if add_escape:
                filters = filters + (',' if filters else '') + 'h'
            return (
                nodes.Expression(content, filters),
                ''
            )

        # SASS Mixins
        m = re.match(r'@(\w+)', line)
        if m:
            name = m.group(1)
            line = line[m.end():]
            self._replace_buffer(line)
            argspec = self._match_python_expr(first=set('('), last=set(')'))
            if argspec:
                argspec = argspec[1:-1]
                line = self._peek_buffer()
            return (
                nodes.MixinDef(name, argspec),
                line
            )

        m = re.match(r'\+([\w.]+)', line)
        if m:
            name = m.group(1)
            line = line[m.end():]
            self._replace_buffer(line)
            argspec = self._match_python_expr(first=set('('), last=set(')'))
            if argspec:
                argspec = argspec[1:-1]
                line = self._peek_buffer()
            return (
                nodes.MixinCall(name, argspec),
                line
            )

        # HAML Filters.
        m = re.match(r':(\w+)(?:\s+(.+))?$', line)
        if m:
            filter, content = m.groups()
            return (
                nodes.Filter(content, filter),
                ''
            )

        # HAML comments
        if line.startswith('-#'):
            return (
                nodes.HAMLComment(line[2:].lstrip()),
                ''
            ) 
        
        # XML Doctype
        if line.startswith('!!!'):
            return (
                nodes.Doctype(*line[3:].strip().split()),
                ''
            )

        # Tags.
        m = re.match(r'''
            (?:%(%?(?:\w+:)?[\w-]*))? # tag name. the extra % is for mako
            (?:
              \[(.+?)(?:,(.+?))?\]    # object reference and prefix
            )? 
            (                         
              (?:\#[\w-]+|\.[\w-]+)+  # id/class
            )?
        ''', line, re.X)                                
        # The match only counts if we have a tag name or id/class.
        if m and (m.group(1) is not None or m.group(4)):
            name, object_reference, object_reference_prefix, raw_id_class = m.groups()
            
            # Extract id value and class list.
            id, class_ = None, []
            for m2 in re.finditer(r'(#|\.)([\w-]+)', raw_id_class or ''):
                type, value = m2.groups()
                if type == '#':
                    id = value
                else:
                    class_.append(value)
            line = line[m.end():]

            # Extract the kwargs expression.
            self._replace_buffer(line)
            kwargs_expr = self._match_python_expr(first=set('('), last=set(')'))
            if kwargs_expr:
                kwargs_expr = kwargs_expr[1:-1]
                line = self._peek_buffer()

            # Whitespace stripping
            m2 = re.match(r'([<>]+)', line)
            strip_outer = strip_inner = False
            if m2:
                strip_outer = '>' in m2.group(1)
                strip_inner = '<' in m2.group(1)
                line = line[m2.end():]

            # Self-closing tags
            self_closing = bool(line and line[0] == '/')
            line = line[int(self_closing):].lstrip()

            return (
                nodes.Tag(
                    name=name,
                    id=id,
                    class_=' '.join(class_),
                    
                    kwargs_expr=kwargs_expr,
                    object_reference=object_reference,
                    object_reference_prefix=object_reference_prefix, 
                    self_closing=self_closing,
                    strip_inner=strip_inner,
                    strip_outer=strip_outer,
                ),
                line
            )

        # Control statements.
        m = re.match(r'''
            -
            \s*
            (for|if|while|elif) # control type
        ''', line, re.X)
        if m:
            control_type = m.group(1)
            self._replace_buffer(line[m.end():].lstrip())
            expr = self._match_python_expr(last=set(':'))
            return (
                nodes.Control(control_type, expr[:-1]),
                self._peek_buffer()
            )
        m = re.match(r'-\s*else\s*:', line, re.X)
        if m:
            return (
                nodes.Control('else', None),
                line[m.end():].lstrip()
            )
        
        # Python source.
        if line.startswith('-'):
            if line.startswith('-!'):
                return (
                    nodes.Python(line[2:].lstrip(), module=True),
                    ''
                )
            else:
                return (
                    nodes.Python(line[1:].lstrip(), module=False),
                    ''
                )

        # Content
        return (
            nodes.Content(line),
            ''
        )

Example 35

Project: Arelle
Source File: validateSBRnl.py
View license
def checkDTSdocument(val, modelDocument, *args, **kwargs):
    modelXbrl = val.modelXbrl
    if modelDocument.type in (ModelDocument.Type.SCHEMA, ModelDocument.Type.LINKBASE):
        isSchema = modelDocument.type == ModelDocument.Type.SCHEMA
        docinfo = modelDocument.xmlDocument.docinfo
        if docinfo and docinfo.xml_version != "1.0":
            modelXbrl.error("SBR.NL.2.2.0.02" if isSchema else "SBR.NL.2.3.0.02",
                    _('%(docType)s xml version must be "1.0" but is "%(xmlVersion)s"'),
                    modelObject=modelDocument, docType=modelDocument.gettype().title(),
                    xmlVersion=docinfo.xml_version)
        if modelDocument.documentEncoding.lower() != "utf-8":
            modelXbrl.error("SBR.NL.2.2.0.03" if isSchema else "SBR.NL.2.3.0.03",
                    _('%(docType)s encoding must be "utf-8" but is "%(xmlEncoding)s"'),
                    modelObject=modelDocument, docType=modelDocument.gettype().title(),
                    xmlEncoding=modelDocument.documentEncoding)
        lookingForPrecedingComment = True
        for commentNode in modelDocument.xmlRootElement.itersiblings(preceding=True):
            if isinstance(commentNode, etree._Comment):
                if lookingForPrecedingComment:
                    lookingForPrecedingComment = False
                else:
                    modelXbrl.error("SBR.NL.2.2.0.05" if isSchema else "SBR.NL.2.3.0.05",
                            _('%(docType)s must have only one comment node before schema element'),
                            modelObject=modelDocument, docType=modelDocument.gettype().title())
        if lookingForPrecedingComment:
            modelXbrl.error("SBR.NL.2.2.0.04" if isSchema else "SBR.NL.2.3.0.04",
                _('%(docType)s must have comment node only on line 2'),
                modelObject=modelDocument, docType=modelDocument.gettype().title())
        
        # check namespaces are used
        for prefix, ns in modelDocument.xmlRootElement.nsmap.items():
            if ((prefix not in val.valUsedPrefixes) and
                (modelDocument.type != ModelDocument.Type.SCHEMA or ns != modelDocument.targetNamespace)):
                modelXbrl.error("SBR.NL.2.2.0.11" if modelDocument.type == ModelDocument.Type.SCHEMA else "SBR.NL.2.3.0.08",
                    _('%(docType)s namespace declaration "%(declaration)s" is not used'),
                    modelObject=modelDocument, docType=modelDocument.gettype().title(),
                    declaration=("xmlns" + (":" + prefix if prefix else "") + "=" + ns))
                
        if isSchema and val.annotationsCount > 1:
            modelXbrl.error("SBR.NL.2.2.0.22",
                _('Schema has %(annotationsCount)s xs:annotation elements, only 1 allowed'),
                modelObject=modelDocument, annotationsCount=val.annotationsCount)
    if modelDocument.type == ModelDocument.Type.LINKBASE:
        if not val.containsRelationship:
            modelXbrl.error("SBR.NL.2.3.0.12",
                "Linkbase has no relationships",
                modelObject=modelDocument)
        # check file name suffixes
        extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
        if extLinkElt is not None:
            expectedSuffix = None
            if extLinkElt.localName == "labelLink":
                anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*")
                if anyLabel is not None:
                    xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang")
                    expectedSuffix = "-lab-{0}.xml".format(xmlLang)
            else:
                expectedSuffix = {"referenceLink": "-ref.xml",
                                  "presentationLink": "-pre.xml",
                                  "definitionLink": "-def.xml"}.get(extLinkElt.localName, None)
            if expectedSuffix:
                if not modelDocument.uri.endswith(expectedSuffix):
                    modelXbrl.error("SBR.NL.3.2.1.09",
                        "Linkbase filename expected to end with %(expectedSuffix)s: %(filename)s",
                        modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri)
            elif extLinkElt.qname == XbrlConst.qnGenLink:
                anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*")
                if anyLabel is not None:
                    xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang")
                    expectedSuffix = "-generic-lab-{0}.xml".format(xmlLang)
                elif XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "reference") is not None:
                    expectedSuffix = "-generic-ref.xml"
                if expectedSuffix and not modelDocument.uri.endswith(expectedSuffix):
                    modelXbrl.error("SBR.NL.3.2.1.10",
                        "Generic linkbase filename expected to end with %(expectedSuffix)s: %(filename)s",
                        modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri)
        # label checks
        for qnLabel in (XbrlConst.qnLinkLabel, XbrlConst.qnGenLabel):
            for modelLabel in modelDocument.xmlRootElement.iterdescendants(tag=qnLabel.clarkNotation):
                if isinstance(modelLabel, ModelResource):
                    if not modelLabel.text or not modelLabel.text[:1].isupper():
                        modelXbrl.error("SBR.NL.3.2.7.05",
                            _("Labels MUST have a capital first letter, label %(label)s: %(text)s"),
                            modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
                    if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel):
                        if len(modelLabel.text) > 255:
                            modelXbrl.error("SBR.NL.3.2.7.06",
                                _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"),
                                modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
                if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel):
                    if len(modelLabel.text) > 255:
                        modelXbrl.error("SBR.NL.3.2.7.06",
                            _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"),
                            modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
        for modelResource in modelDocument.xmlRootElement.iter():
            # locator checks
            if isinstance(modelResource, ModelLocator):
                hrefModelObject = modelResource.dereference()
                if isinstance(hrefModelObject, ModelObject):
                    expectedLocLabel = hrefModelObject.id + "_loc"
                    if modelResource.xlinkLabel != expectedLocLabel:
                        modelXbrl.error("SBR.NL.3.2.11.01",
                            _("Locator @xlink:label names MUST be concatenated from: @id from the XML node, underscore, 'loc', expected %(expectedLocLabel)s, found %(foundLocLabel)s"),
                            modelObject=modelResource, expectedLocLabel=expectedLocLabel, foundLocLabel=modelResource.xlinkLabel)
            # xlinkLabel checks
            if isinstance(modelResource, ModelResource):
                if re.match(r"[^a-zA-Z0-9_-]", modelResource.xlinkLabel):
                    modelXbrl.error("SBR.NL.3.2.11.03",
                        _("@xlink:label names MUST use a-zA-Z0-9_- characters only: %(xlinkLabel)s"),
                        modelObject=modelResource, xlinkLabel=modelResource.xlinkLabel)
    elif modelDocument.targetNamespace: # SCHEMA with targetNamespace
        # check for unused imports
        for referencedDocument in modelDocument.referencesDocument.keys():
            if (referencedDocument.type == ModelDocument.Type.SCHEMA and
                referencedDocument.targetNamespace not in {XbrlConst.xbrli, XbrlConst.link} and
                referencedDocument.targetNamespace not in val.referencedNamespaces):
                modelXbrl.error("SBR.NL.2.2.0.15",
                    _("A schema import schemas of which no content is being addressed: %(importedFile)s"),
                    modelObject=modelDocument, importedFile=referencedDocument.basename)
        if modelDocument.targetNamespace != modelDocument.targetNamespace.lower():
            modelXbrl.error("SBR.NL.3.2.3.02",
                _("Namespace URI's MUST be lower case: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        if len(modelDocument.targetNamespace) > 255:
            modelXbrl.error("SBR.NL.3.2.3.03",
                _("Namespace URI's MUST NOT be longer than 255 characters: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        if re.match(r"[^a-z0-9_/-]", modelDocument.targetNamespace):
            modelXbrl.error("SBR.NL.3.2.3.04",
                _("Namespace URI's MUST use only signs from a-z0-9_-/: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        if not modelDocument.targetNamespace.startswith('http://www.nltaxonomie.nl'):
            modelXbrl.error("SBR.NL.3.2.3.05",
                _("Namespace URI's MUST start with 'http://www.nltaxonomie.nl': %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        namespacePrefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace)
        if not namespacePrefix:
            modelXbrl.error("SBR.NL.3.2.4.01",
                _("TargetNamespaces MUST have a prefix: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        elif namespacePrefix in val.prefixNamespace:
            modelXbrl.error("SBR.NL.3.2.4.02",
                _("Namespace prefix MUST be unique within the NT but prefix '%(prefix)s' is used by both %(namespaceURI)s and %(namespaceURI2)s."),
                modelObject=modelDocument, prefix=namespacePrefix,
                namespaceURI=modelDocument.targetNamespace, namespaceURI2=val.prefixNamespace[namespacePrefix])
        else:
            val.prefixNamespace[namespacePrefix] = modelDocument.targetNamespace
            val.namespacePrefix[modelDocument.targetNamespace] = namespacePrefix
        if namespacePrefix in {"xsi", "xsd", "xs", "xbrli", "link", "xlink", "xbrldt", "xbrldi", "gen", "xl"}:
            modelXbrl.error("SBR.NL.3.2.4.03",
                _("Namespace prefix '%(prefix)s' reserved by organizations for international specifications is used %(namespaceURI)s."),
                modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace)
        if len(namespacePrefix) > 20:
            modelXbrl.warning("SBR.NL.3.2.4.06",
                _("Namespace prefix '%(prefix)s' SHOULD not exceed 20 characters %(namespaceURI)s."),
                modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace)
        # check every non-targetnamespace prefix against its schema
    requiredLinkrole = None # only set for extension taxonomies
    # check folder names
    if modelDocument.filepathdir.startswith(modelXbrl.uriDir):
        partnerPrefix = None
        pathDir = modelDocument.filepathdir[len(modelXbrl.uriDir) + 1:].replace("\\", "/")
        lastPathSegment = None
        for pathSegment in pathDir.split("/"):
            if pathSegment.lower() != pathSegment:
                modelXbrl.error("SBR.NL.3.2.1.02",
                    _("Folder names must be in lower case: %(folder)s"),
                    modelObject=modelDocument, folder=pathSegment)
            if len(pathSegment) >= 15 :
                modelXbrl.error("SBR.NL.3.2.1.03",
                    _("Folder names must be less than 15 characters: %(folder)s"),
                    modelObject=modelDocument, folder=pathSegment)
            if pathSegment in ("bd", "kvk", "cbs"):
                partnerPrefix = pathSegment + '-'
            lastPathSegment = pathSegment
        if modelDocument.basename.lower() != modelDocument.basename:
            modelXbrl.error("SBR.NL.3.2.1.05",
                _("File names must be in lower case: %(file)s"),
                modelObject=modelDocument, file=modelDocument.basename)
        if partnerPrefix and not modelDocument.basename.startswith(partnerPrefix):
            modelXbrl.error("SBR.NL.3.2.1.14",
                "NT Partner DTS files MUST start with %(partnerPrefix)s consistently: %(filename)s",
                modelObject=modelDocument, partnerPrefix=partnerPrefix, filename=modelDocument.uri)
        if modelDocument.type == ModelDocument.Type.SCHEMA:
            if modelDocument.targetNamespace:
                nsParts = modelDocument.targetNamespace.split("/")
                # [0] = https, [1] = // [2] = nl.taxonomie  [3] = year or version
                nsYrOrVer = nsParts[3]
                requiredNamespace = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/" + modelDocument.basename[:-4]
                requiredLinkrole = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/"
                if modelDocument == modelXbrl.modelDocument:  # entry point
                    nsYr = "{year}"
                    if '2009' <= nsParts[3] < '2020':  # must be a year, use as year
                        nsYr = nsParts[3]
                    else: # look for year in parts of basename of required namespace
                        for nsPart in nsParts:
                            for baseNamePart in nsPart.split('-'):
                                if '2009' <= baseNamePart < '2020':
                                    nsYr = baseNamePart
                                    break
                    if not requiredNamespace.endswith('-' + nsYr):
                        requiredNamespace += '-' + nsYr
                if not modelDocument.targetNamespace.startswith(requiredNamespace):
                    modelXbrl.error("SBR.NL.3.2.3.06",
                        _("Namespace URI's MUST be constructed like %(requiredNamespace)s: %(namespaceURI)s"),
                        modelObject=modelDocument, requiredNamespace=requiredNamespace, namespaceURI=modelDocument.targetNamespace)
            else:
                requiredLinkrole = ''
            # concept checks
            for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
                if isinstance(modelConcept, ModelConcept):
                    # 6.7.16 name not duplicated in standard taxonomies
                    name = modelConcept.get("name")
                    if name:
                        ''' removed per RH 2013-03-25
                        substititutionGroupQname = modelConcept.substitutionGroupQname
                        if substititutionGroupQname:
                            if name.endswith("Member") ^ (substititutionGroupQname.localName == "domainMemberItem" and
                                                          substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
                                modelXbrl.error("SBR.NL.3.2.5.11",
                                    _("Concept %(concept)s must end in Member to be in sbr:domainMemberItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("Domain") ^ (substititutionGroupQname.localName == "domainItem" and
                                                          substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
                                modelXbrl.error("SBR.NL.3.2.5.12",
                                    _("Concept %(concept)s must end in Domain to be in sbr:domainItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("TypedAxis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and
                                                             modelConcept.isTypedDimension):
                                modelXbrl.error("SBR.NL.3.2.5.14",
                                    _("Concept %(concept)s must end in TypedAxis to be in xbrldt:dimensionItem substitution group if they represent a typed dimension"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if (name.endswith("Axis") and 
                                not name.endswith("TypedAxis")) ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and
                                                                   modelConcept.isExplicitDimension):
                                modelXbrl.error("SBR.NL.3.2.5.13",
                                    _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group if they represent an explicit dimension"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
                                modelXbrl.error("SBR.NL.3.2.5.15",
                                    _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("Title") ^ (substititutionGroupQname.localName == "presentationItem" and
                                                         substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
                                modelXbrl.error("SBR.NL.3.2.5.16",
                                    _("Concept %(concept)s must end in Title to be in sbr:presentationItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                        ''' 
                        if len(name) > 200:
                            modelXbrl.error("SBR.NL.3.2.12.02" if modelConcept.isLinkPart 
                                                else "SBR.NL.3.2.5.21" if (modelConcept.isItem or modelConcept.isTuple)
                                                else "SBR.NL.3.2.14.01",
                                _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
                                modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
            # type checks
            for typeType in ("simpleType", "complexType"):
                for modelType in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}" + typeType):
                    if isinstance(modelType, ModelType):
                        name = modelType.get("name")
                        if name is None: 
                            name = ""
                            if modelType.get("ref") is not None:
                                continue    # don't validate ref's here
                        if len(name) > 200:
                            modelXbrl.error("SBR.NL.3.2.5.21",
                                _("Type %(type)s name length %(namelength)s exceeds 200 characters"),
                                modelObject=modelType, type=modelType.qname, namelength=len(name))
                        if modelType.qnameDerivedFrom and modelType.qnameDerivedFrom.namespaceURI != XbrlConst.xbrli:
                            modelXbrl.error("SBR.NL.3.2.8.01",
                                _("Custom datatypes MUST be a restriction from XII defined datatypes: %(type)s"),
                                modelObject=modelType, type=modelType.qname)
                        if re.match(r"[^a-zA-Z0-9_-]", name):
                            modelXbrl.error("SBR.NL.3.2.8.02",
                                _("Datatype names MUST use characters a-zA-Z0-9_- only: %(type)s"),
                                modelObject=modelDocument, type=modelType.qname)
                        if modelType.facets and "enumeration" in modelType.facets:
                            if not modelType.qnameDerivedFrom == XbrlConst.qnXbrliStringItemType:
                                modelXbrl.error("SBR.NL.3.2.13.01",
                                    _("Enumerations MUST use a restriction on xbrli:stringItemType: %(type)s"),
                                    modelObject=modelDocument, type=modelType.qname)
            if lastPathSegment == "entrypoints":
                if not modelDocument.xmlRootElement.id:
                    modelXbrl.error("SBR.NL.2.2.0.23",
                        _("xs:schema/@id MUST be present in schema files in the reports/{NT partner}/entrypoints/ folder"),
                        modelObject=modelDocument)
                    
                
    # check for idObject conflicts
    for id, modelObject in modelDocument.idObjects.items():
        if id in val.idObjects:
            modelXbrl.error("SBR.NL.3.2.6.01",
                _("ID %(id)s must be unique in the DTS but is present on two elements."),
                modelObject=(modelObject, val.idObjects[id]), id=id)
        else:
            val.idObjects[id] = modelObject

            
    for roleURI, modelRoleTypes in modelXbrl.roleTypes.items():
        if not roleURI.startswith("http://www.xbrl.org"):
            usedOns = set.union(*[modelRoleType.usedOns for modelRoleType in modelRoleTypes])
            # check roletypes for linkroles (only)
            if usedOns & {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink,
                          XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote}:
                if len(modelRoleTypes) > 1:
                    modelXbrl.error("SBR.NL.3.2.9.01",
                        _("Linkrole URI's MUST be unique in the NT: %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if roleURI.lower() != roleURI:
                    modelXbrl.error("SBR.NL.3.2.9.02",
                        _("Linkrole URI's MUST be in lowercase: %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if re.match(r"[^a-z0-9_/-]", roleURI):
                    modelXbrl.error("SBR.NL.3.2.9.03",
                        _("Linkrole URI's MUST use characters a-z0-9_-/ only: %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if len(roleURI) > 255:
                    modelXbrl.error("SBR.NL.3.2.9.04",
                        _("Linkrole URI's MUST NOT be longer than 255 characters, length is %(len)s: %(linkrole)s"),
                        modelObject=modelRoleTypes, len=len(roleURI), linkrole=roleURI)
                ''' removed per RH 2013-03-13 e-mail
                if not roleURI.startswith('http://www.nltaxonomie.nl'):
                    modelXbrl.error("SBR.NL.3.2.9.05",
                        _("Linkrole URI's MUST start with 'http://www.nltaxonomie.nl': %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if (requiredLinkrole and 
                    not roleURI.startswith(requiredLinkrole) and 
                    re.match(r".*(domain$|axis$|table$|lineitem$)", roleURI)):
                        modelXbrl.error("SBR.NL.3.2.9.06",
                            _("Linkrole URI's MUST have the following construct: http://www.nltaxonomie.nl / {folder path} / {functional name} - {domain or axis or table or lineitem}: %(linkrole)s"),
                            modelObject=modelRoleTypes, linkrole=roleURI)
                '''
                for modelRoleType in modelRoleTypes:
                    if len(modelRoleType.id) > 255:
                        modelXbrl.error("SBR.NL.3.2.10.02",
                            _("Linkrole @id MUST NOT exceed 255 characters, length is %(length)s: %(linkroleID)s"),
                            modelObject=modelRoleType, length=len(modelRoleType.id), linkroleID=modelRoleType.id)
                partnerPrefix = modelRoleTypes[0].modelDocument.basename.split('-')
                if partnerPrefix:  # first element before dash is prefix
                    urnPartnerLinkroleStart = "urn:{0}:linkrole:".format(partnerPrefix[0])
                    if not roleURI.startswith(urnPartnerLinkroleStart): 
                        modelXbrl.error("SBR.NL.3.2.9.10",
                            _("Linkrole MUST start with urn:{NT partner code}:linkrole:, \nexpecting: %(expectedStart)s..., \nfound: %(linkrole)s"),
                            modelObject=modelRoleType, expectedStart=urnPartnerLinkroleStart, linkrole=roleURI)

Example 36

Project: Arelle
Source File: validateSBRnl.py
View license
def checkDTSdocument(val, modelDocument, *args, **kwargs):
    modelXbrl = val.modelXbrl
    if modelDocument.type in (ModelDocument.Type.SCHEMA, ModelDocument.Type.LINKBASE):
        isSchema = modelDocument.type == ModelDocument.Type.SCHEMA
        docinfo = modelDocument.xmlDocument.docinfo
        if docinfo and docinfo.xml_version != "1.0":
            modelXbrl.error("SBR.NL.2.2.0.02" if isSchema else "SBR.NL.2.3.0.02",
                    _('%(docType)s xml version must be "1.0" but is "%(xmlVersion)s"'),
                    modelObject=modelDocument, docType=modelDocument.gettype().title(),
                    xmlVersion=docinfo.xml_version)
        if modelDocument.documentEncoding.lower() != "utf-8":
            modelXbrl.error("SBR.NL.2.2.0.03" if isSchema else "SBR.NL.2.3.0.03",
                    _('%(docType)s encoding must be "utf-8" but is "%(xmlEncoding)s"'),
                    modelObject=modelDocument, docType=modelDocument.gettype().title(),
                    xmlEncoding=modelDocument.documentEncoding)
        lookingForPrecedingComment = True
        for commentNode in modelDocument.xmlRootElement.itersiblings(preceding=True):
            if isinstance(commentNode, etree._Comment):
                if lookingForPrecedingComment:
                    lookingForPrecedingComment = False
                else:
                    modelXbrl.error("SBR.NL.2.2.0.05" if isSchema else "SBR.NL.2.3.0.05",
                            _('%(docType)s must have only one comment node before schema element'),
                            modelObject=modelDocument, docType=modelDocument.gettype().title())
        if lookingForPrecedingComment:
            modelXbrl.error("SBR.NL.2.2.0.04" if isSchema else "SBR.NL.2.3.0.04",
                _('%(docType)s must have comment node only on line 2'),
                modelObject=modelDocument, docType=modelDocument.gettype().title())
        
        # check namespaces are used
        for prefix, ns in modelDocument.xmlRootElement.nsmap.items():
            if ((prefix not in val.valUsedPrefixes) and
                (modelDocument.type != ModelDocument.Type.SCHEMA or ns != modelDocument.targetNamespace)):
                modelXbrl.error("SBR.NL.2.2.0.11" if modelDocument.type == ModelDocument.Type.SCHEMA else "SBR.NL.2.3.0.08",
                    _('%(docType)s namespace declaration "%(declaration)s" is not used'),
                    modelObject=modelDocument, docType=modelDocument.gettype().title(),
                    declaration=("xmlns" + (":" + prefix if prefix else "") + "=" + ns))
                
        if isSchema and val.annotationsCount > 1:
            modelXbrl.error("SBR.NL.2.2.0.22",
                _('Schema has %(annotationsCount)s xs:annotation elements, only 1 allowed'),
                modelObject=modelDocument, annotationsCount=val.annotationsCount)
    if modelDocument.type == ModelDocument.Type.LINKBASE:
        if not val.containsRelationship:
            modelXbrl.error("SBR.NL.2.3.0.12",
                "Linkbase has no relationships",
                modelObject=modelDocument)
        # check file name suffixes
        extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
        if extLinkElt is not None:
            expectedSuffix = None
            if extLinkElt.localName == "labelLink":
                anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*")
                if anyLabel is not None:
                    xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang")
                    expectedSuffix = "-lab-{0}.xml".format(xmlLang)
            else:
                expectedSuffix = {"referenceLink": "-ref.xml",
                                  "presentationLink": "-pre.xml",
                                  "definitionLink": "-def.xml"}.get(extLinkElt.localName, None)
            if expectedSuffix:
                if not modelDocument.uri.endswith(expectedSuffix):
                    modelXbrl.error("SBR.NL.3.2.1.09",
                        "Linkbase filename expected to end with %(expectedSuffix)s: %(filename)s",
                        modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri)
            elif extLinkElt.qname == XbrlConst.qnGenLink:
                anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*")
                if anyLabel is not None:
                    xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang")
                    expectedSuffix = "-generic-lab-{0}.xml".format(xmlLang)
                elif XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "reference") is not None:
                    expectedSuffix = "-generic-ref.xml"
                if expectedSuffix and not modelDocument.uri.endswith(expectedSuffix):
                    modelXbrl.error("SBR.NL.3.2.1.10",
                        "Generic linkbase filename expected to end with %(expectedSuffix)s: %(filename)s",
                        modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri)
        # label checks
        for qnLabel in (XbrlConst.qnLinkLabel, XbrlConst.qnGenLabel):
            for modelLabel in modelDocument.xmlRootElement.iterdescendants(tag=qnLabel.clarkNotation):
                if isinstance(modelLabel, ModelResource):
                    if not modelLabel.text or not modelLabel.text[:1].isupper():
                        modelXbrl.error("SBR.NL.3.2.7.05",
                            _("Labels MUST have a capital first letter, label %(label)s: %(text)s"),
                            modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
                    if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel):
                        if len(modelLabel.text) > 255:
                            modelXbrl.error("SBR.NL.3.2.7.06",
                                _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"),
                                modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
                if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel):
                    if len(modelLabel.text) > 255:
                        modelXbrl.error("SBR.NL.3.2.7.06",
                            _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"),
                            modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64])
        for modelResource in modelDocument.xmlRootElement.iter():
            # locator checks
            if isinstance(modelResource, ModelLocator):
                hrefModelObject = modelResource.dereference()
                if isinstance(hrefModelObject, ModelObject):
                    expectedLocLabel = hrefModelObject.id + "_loc"
                    if modelResource.xlinkLabel != expectedLocLabel:
                        modelXbrl.error("SBR.NL.3.2.11.01",
                            _("Locator @xlink:label names MUST be concatenated from: @id from the XML node, underscore, 'loc', expected %(expectedLocLabel)s, found %(foundLocLabel)s"),
                            modelObject=modelResource, expectedLocLabel=expectedLocLabel, foundLocLabel=modelResource.xlinkLabel)
            # xlinkLabel checks
            if isinstance(modelResource, ModelResource):
                if re.match(r"[^a-zA-Z0-9_-]", modelResource.xlinkLabel):
                    modelXbrl.error("SBR.NL.3.2.11.03",
                        _("@xlink:label names MUST use a-zA-Z0-9_- characters only: %(xlinkLabel)s"),
                        modelObject=modelResource, xlinkLabel=modelResource.xlinkLabel)
    elif modelDocument.targetNamespace: # SCHEMA with targetNamespace
        # check for unused imports
        for referencedDocument in modelDocument.referencesDocument.keys():
            if (referencedDocument.type == ModelDocument.Type.SCHEMA and
                referencedDocument.targetNamespace not in {XbrlConst.xbrli, XbrlConst.link} and
                referencedDocument.targetNamespace not in val.referencedNamespaces):
                modelXbrl.error("SBR.NL.2.2.0.15",
                    _("A schema import schemas of which no content is being addressed: %(importedFile)s"),
                    modelObject=modelDocument, importedFile=referencedDocument.basename)
        if modelDocument.targetNamespace != modelDocument.targetNamespace.lower():
            modelXbrl.error("SBR.NL.3.2.3.02",
                _("Namespace URI's MUST be lower case: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        if len(modelDocument.targetNamespace) > 255:
            modelXbrl.error("SBR.NL.3.2.3.03",
                _("Namespace URI's MUST NOT be longer than 255 characters: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        if re.match(r"[^a-z0-9_/-]", modelDocument.targetNamespace):
            modelXbrl.error("SBR.NL.3.2.3.04",
                _("Namespace URI's MUST use only signs from a-z0-9_-/: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        if not modelDocument.targetNamespace.startswith('http://www.nltaxonomie.nl'):
            modelXbrl.error("SBR.NL.3.2.3.05",
                _("Namespace URI's MUST start with 'http://www.nltaxonomie.nl': %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        namespacePrefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace)
        if not namespacePrefix:
            modelXbrl.error("SBR.NL.3.2.4.01",
                _("TargetNamespaces MUST have a prefix: %(namespaceURI)s"),
                modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace)
        elif namespacePrefix in val.prefixNamespace:
            modelXbrl.error("SBR.NL.3.2.4.02",
                _("Namespace prefix MUST be unique within the NT but prefix '%(prefix)s' is used by both %(namespaceURI)s and %(namespaceURI2)s."),
                modelObject=modelDocument, prefix=namespacePrefix,
                namespaceURI=modelDocument.targetNamespace, namespaceURI2=val.prefixNamespace[namespacePrefix])
        else:
            val.prefixNamespace[namespacePrefix] = modelDocument.targetNamespace
            val.namespacePrefix[modelDocument.targetNamespace] = namespacePrefix
        if namespacePrefix in {"xsi", "xsd", "xs", "xbrli", "link", "xlink", "xbrldt", "xbrldi", "gen", "xl"}:
            modelXbrl.error("SBR.NL.3.2.4.03",
                _("Namespace prefix '%(prefix)s' reserved by organizations for international specifications is used %(namespaceURI)s."),
                modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace)
        if len(namespacePrefix) > 20:
            modelXbrl.warning("SBR.NL.3.2.4.06",
                _("Namespace prefix '%(prefix)s' SHOULD not exceed 20 characters %(namespaceURI)s."),
                modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace)
        # check every non-targetnamespace prefix against its schema
    requiredLinkrole = None # only set for extension taxonomies
    # check folder names
    if modelDocument.filepathdir.startswith(modelXbrl.uriDir):
        partnerPrefix = None
        pathDir = modelDocument.filepathdir[len(modelXbrl.uriDir) + 1:].replace("\\", "/")
        lastPathSegment = None
        for pathSegment in pathDir.split("/"):
            if pathSegment.lower() != pathSegment:
                modelXbrl.error("SBR.NL.3.2.1.02",
                    _("Folder names must be in lower case: %(folder)s"),
                    modelObject=modelDocument, folder=pathSegment)
            if len(pathSegment) >= 15 :
                modelXbrl.error("SBR.NL.3.2.1.03",
                    _("Folder names must be less than 15 characters: %(folder)s"),
                    modelObject=modelDocument, folder=pathSegment)
            if pathSegment in ("bd", "kvk", "cbs"):
                partnerPrefix = pathSegment + '-'
            lastPathSegment = pathSegment
        if modelDocument.basename.lower() != modelDocument.basename:
            modelXbrl.error("SBR.NL.3.2.1.05",
                _("File names must be in lower case: %(file)s"),
                modelObject=modelDocument, file=modelDocument.basename)
        if partnerPrefix and not modelDocument.basename.startswith(partnerPrefix):
            modelXbrl.error("SBR.NL.3.2.1.14",
                "NT Partner DTS files MUST start with %(partnerPrefix)s consistently: %(filename)s",
                modelObject=modelDocument, partnerPrefix=partnerPrefix, filename=modelDocument.uri)
        if modelDocument.type == ModelDocument.Type.SCHEMA:
            if modelDocument.targetNamespace:
                nsParts = modelDocument.targetNamespace.split("/")
                # [0] = https, [1] = // [2] = nl.taxonomie  [3] = year or version
                nsYrOrVer = nsParts[3]
                requiredNamespace = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/" + modelDocument.basename[:-4]
                requiredLinkrole = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/"
                if modelDocument == modelXbrl.modelDocument:  # entry point
                    nsYr = "{year}"
                    if '2009' <= nsParts[3] < '2020':  # must be a year, use as year
                        nsYr = nsParts[3]
                    else: # look for year in parts of basename of required namespace
                        for nsPart in nsParts:
                            for baseNamePart in nsPart.split('-'):
                                if '2009' <= baseNamePart < '2020':
                                    nsYr = baseNamePart
                                    break
                    if not requiredNamespace.endswith('-' + nsYr):
                        requiredNamespace += '-' + nsYr
                if not modelDocument.targetNamespace.startswith(requiredNamespace):
                    modelXbrl.error("SBR.NL.3.2.3.06",
                        _("Namespace URI's MUST be constructed like %(requiredNamespace)s: %(namespaceURI)s"),
                        modelObject=modelDocument, requiredNamespace=requiredNamespace, namespaceURI=modelDocument.targetNamespace)
            else:
                requiredLinkrole = ''
            # concept checks
            for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
                if isinstance(modelConcept, ModelConcept):
                    # 6.7.16 name not duplicated in standard taxonomies
                    name = modelConcept.get("name")
                    if name:
                        ''' removed per RH 2013-03-25
                        substititutionGroupQname = modelConcept.substitutionGroupQname
                        if substititutionGroupQname:
                            if name.endswith("Member") ^ (substititutionGroupQname.localName == "domainMemberItem" and
                                                          substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
                                modelXbrl.error("SBR.NL.3.2.5.11",
                                    _("Concept %(concept)s must end in Member to be in sbr:domainMemberItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("Domain") ^ (substititutionGroupQname.localName == "domainItem" and
                                                          substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
                                modelXbrl.error("SBR.NL.3.2.5.12",
                                    _("Concept %(concept)s must end in Domain to be in sbr:domainItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("TypedAxis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and
                                                             modelConcept.isTypedDimension):
                                modelXbrl.error("SBR.NL.3.2.5.14",
                                    _("Concept %(concept)s must end in TypedAxis to be in xbrldt:dimensionItem substitution group if they represent a typed dimension"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if (name.endswith("Axis") and 
                                not name.endswith("TypedAxis")) ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and
                                                                   modelConcept.isExplicitDimension):
                                modelXbrl.error("SBR.NL.3.2.5.13",
                                    _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group if they represent an explicit dimension"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
                                modelXbrl.error("SBR.NL.3.2.5.15",
                                    _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if name.endswith("Title") ^ (substititutionGroupQname.localName == "presentationItem" and
                                                         substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")):
                                modelXbrl.error("SBR.NL.3.2.5.16",
                                    _("Concept %(concept)s must end in Title to be in sbr:presentationItem substitution group"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                        ''' 
                        if len(name) > 200:
                            modelXbrl.error("SBR.NL.3.2.12.02" if modelConcept.isLinkPart 
                                                else "SBR.NL.3.2.5.21" if (modelConcept.isItem or modelConcept.isTuple)
                                                else "SBR.NL.3.2.14.01",
                                _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
                                modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
            # type checks
            for typeType in ("simpleType", "complexType"):
                for modelType in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}" + typeType):
                    if isinstance(modelType, ModelType):
                        name = modelType.get("name")
                        if name is None: 
                            name = ""
                            if modelType.get("ref") is not None:
                                continue    # don't validate ref's here
                        if len(name) > 200:
                            modelXbrl.error("SBR.NL.3.2.5.21",
                                _("Type %(type)s name length %(namelength)s exceeds 200 characters"),
                                modelObject=modelType, type=modelType.qname, namelength=len(name))
                        if modelType.qnameDerivedFrom and modelType.qnameDerivedFrom.namespaceURI != XbrlConst.xbrli:
                            modelXbrl.error("SBR.NL.3.2.8.01",
                                _("Custom datatypes MUST be a restriction from XII defined datatypes: %(type)s"),
                                modelObject=modelType, type=modelType.qname)
                        if re.match(r"[^a-zA-Z0-9_-]", name):
                            modelXbrl.error("SBR.NL.3.2.8.02",
                                _("Datatype names MUST use characters a-zA-Z0-9_- only: %(type)s"),
                                modelObject=modelDocument, type=modelType.qname)
                        if modelType.facets and "enumeration" in modelType.facets:
                            if not modelType.qnameDerivedFrom == XbrlConst.qnXbrliStringItemType:
                                modelXbrl.error("SBR.NL.3.2.13.01",
                                    _("Enumerations MUST use a restriction on xbrli:stringItemType: %(type)s"),
                                    modelObject=modelDocument, type=modelType.qname)
            if lastPathSegment == "entrypoints":
                if not modelDocument.xmlRootElement.id:
                    modelXbrl.error("SBR.NL.2.2.0.23",
                        _("xs:schema/@id MUST be present in schema files in the reports/{NT partner}/entrypoints/ folder"),
                        modelObject=modelDocument)
                    
                
    # check for idObject conflicts
    for id, modelObject in modelDocument.idObjects.items():
        if id in val.idObjects:
            modelXbrl.error("SBR.NL.3.2.6.01",
                _("ID %(id)s must be unique in the DTS but is present on two elements."),
                modelObject=(modelObject, val.idObjects[id]), id=id)
        else:
            val.idObjects[id] = modelObject

            
    for roleURI, modelRoleTypes in modelXbrl.roleTypes.items():
        if not roleURI.startswith("http://www.xbrl.org"):
            usedOns = set.union(*[modelRoleType.usedOns for modelRoleType in modelRoleTypes])
            # check roletypes for linkroles (only)
            if usedOns & {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink,
                          XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote}:
                if len(modelRoleTypes) > 1:
                    modelXbrl.error("SBR.NL.3.2.9.01",
                        _("Linkrole URI's MUST be unique in the NT: %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if roleURI.lower() != roleURI:
                    modelXbrl.error("SBR.NL.3.2.9.02",
                        _("Linkrole URI's MUST be in lowercase: %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if re.match(r"[^a-z0-9_/-]", roleURI):
                    modelXbrl.error("SBR.NL.3.2.9.03",
                        _("Linkrole URI's MUST use characters a-z0-9_-/ only: %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if len(roleURI) > 255:
                    modelXbrl.error("SBR.NL.3.2.9.04",
                        _("Linkrole URI's MUST NOT be longer than 255 characters, length is %(len)s: %(linkrole)s"),
                        modelObject=modelRoleTypes, len=len(roleURI), linkrole=roleURI)
                ''' removed per RH 2013-03-13 e-mail
                if not roleURI.startswith('http://www.nltaxonomie.nl'):
                    modelXbrl.error("SBR.NL.3.2.9.05",
                        _("Linkrole URI's MUST start with 'http://www.nltaxonomie.nl': %(linkrole)s"),
                        modelObject=modelRoleTypes, linkrole=roleURI)
                if (requiredLinkrole and 
                    not roleURI.startswith(requiredLinkrole) and 
                    re.match(r".*(domain$|axis$|table$|lineitem$)", roleURI)):
                        modelXbrl.error("SBR.NL.3.2.9.06",
                            _("Linkrole URI's MUST have the following construct: http://www.nltaxonomie.nl / {folder path} / {functional name} - {domain or axis or table or lineitem}: %(linkrole)s"),
                            modelObject=modelRoleTypes, linkrole=roleURI)
                '''
                for modelRoleType in modelRoleTypes:
                    if len(modelRoleType.id) > 255:
                        modelXbrl.error("SBR.NL.3.2.10.02",
                            _("Linkrole @id MUST NOT exceed 255 characters, length is %(length)s: %(linkroleID)s"),
                            modelObject=modelRoleType, length=len(modelRoleType.id), linkroleID=modelRoleType.id)
                partnerPrefix = modelRoleTypes[0].modelDocument.basename.split('-')
                if partnerPrefix:  # first element before dash is prefix
                    urnPartnerLinkroleStart = "urn:{0}:linkrole:".format(partnerPrefix[0])
                    if not roleURI.startswith(urnPartnerLinkroleStart): 
                        modelXbrl.error("SBR.NL.3.2.9.10",
                            _("Linkrole MUST start with urn:{NT partner code}:linkrole:, \nexpecting: %(expectedStart)s..., \nfound: %(linkrole)s"),
                            modelObject=modelRoleType, expectedStart=urnPartnerLinkroleStart, linkrole=roleURI)

Example 37

Project: inventory
Source File: handlers.py
View license
    def read(self, request, key_value_id=None):
        #if keystore get var is set return the whole keystore
        if 'keystore' in request.GET:
            #if key get var is set return the keystore based on the existance of this key
            if 'key' in request.GET:
                base = KeyValue.objects.filter(key=request.GET['keystore']).filter(keyvalue_set__contains=request.GET['key'])
                tmp_list = []
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.system.hostname, r.key)
                    tmp_list[key_name] = r.value
            if 'key' not in request.GET:
                tree = KeyValueTree(request.GET['keystore']).final
                return tree
        elif 'key_type' in request.GET:
            key_type = request.GET['key_type']
            tmp_list = []
            if key_type == 'dhcp_scopes':
                #Get keystores from truth that have dhcp.is_scope = True
                base = TruthKeyValue.objects.filter(key='dhcp.is_scope',value='True')
                #Iterate through the list and get all of the key/value pairs
                for row in base:
                    keyvalue = TruthKeyValue.objects.filter(truth=row.truth)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_list.append(tmp_dict)
                return tmp_list

            if key_type == 'system_by_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                keyvalue_pairs = KeyValue.objects.filter(key__contains='dhcp_scope',value=request.GET['scope']).filter(key__startswith='nic.')
                #Iterate through the list and get all of the key/value pairs
                tmp_list = []
                for row in keyvalue_pairs:
                    keyvalue = KeyValue.objects.filter(obj=row.system)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_dict['hostname'] = row.system.hostname
                    appendable = True
                    for the_items in tmp_list:
                        if 'hostname' not in the_items:
                            appendable = True
                        elif the_items['hostname'] == row.system.hostname:
                            appendable = False
                    if appendable is True:
                        tmp_list.append(tmp_dict)
                    #tmp_list = list(set(tmp_list))
                return tmp_list
            if key_type == 'adapters_by_system':
                #Get keystores from truth that have dhcp.is_scope = True
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs =
                KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        if matches.group(1) not in adapter_ids:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    ipv4_address = ''
                    option_domain_name = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    try:
                        final_list.append({
                            'system_hostname':system.hostname,
                            'ipv4_address':ipv4_address,
                            'adapter_name':adapter_name,
                            'mac_address':mac_address,
                            'dhcp_hostname':dhcp_hostname,
                            'dhcp_filename':dhcp_filename}
                            )
                    except:
                        pass
                #tmp_list.append(tmp_dict)
                return final_list
            if key_type == 'adapters_by_system_and_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                dhcp_scope = request.GET['dhcp_scope']
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs =
                KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        #if matches.group(1) not in adapter_ids and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] is not None and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] == dhcp_scope:
                        dhcp_scope_match = 'nic.%s.dhcp_scope.0' % matches.group(1)
                        if matches.group(1) not in adapter_ids and dhcp_scope_match in tmp_dict and tmp_dict[dhcp_scope_match] == dhcp_scope:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    dhcp_domain_name = ''
                    ipv4_address = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if 'nic.%s.option_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.option_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name.0' % a in tmp_dict:
                        dhcp_domain_name = tmp_dict['nic.%s.dhcp_domain_name.0' % a]
                    final_list.append({'system_hostname':system.hostname, 'ipv4_address':ipv4_address,  'adapter_name':adapter_name, 'mac_address':mac_address, 'dhcp_hostname':dhcp_hostname, 'dhcp_filename':dhcp_filename, 'dhcp_domain_name':dhcp_domain_name})
                #tmp_list.append(tmp_dict)
                return final_list
        elif 'key' in request.GET and request.GET['key'] > '':
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.system.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list
        elif 'value' in request.GET:
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.system.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list
        #print tree
        """for t in tree:

Example 38

Project: inventory
Source File: keyvalue_handler.py
View license
    def read(self, request, key_value_id=None):
        #if keystore get var is set return the whole keystore
        if 'keystore' in request.GET:
            #if key get var is set return the keystore based on the existance of this key
            if 'key' in request.GET:
                base = KeyValue.objects.filter(key=request.GET['keystore']).filter(keyvalue_set__contains=request.GET['key'])
                tmp_list = []
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.system.hostname, r.key)
                    tmp_list[key_name] = r.value
            if 'key' not in request.GET:
                tree = KeyValueTree(request.GET['keystore']).final
                return tree
        elif 'key_type' in request.GET:
            key_type = request.GET['key_type']
            tmp_list = []
            if key_type == 'dhcp_scopes':
                #Get keystores from truth that have dhcp.is_scope = True
                base = TruthKeyValue.objects.filter(key='dhcp.is_scope',value='True')
                #Iterate through the list and get all of the key/value pairs
                for row in base:
                    keyvalue = TruthKeyValue.objects.filter(truth=row.truth)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_list.append(tmp_dict)
                return tmp_list

            if key_type == 'system_by_reverse_dns_zone':

                #Get keystores from truth that have dhcp.is_scope = True
                keyvalue_pairs = KeyValue.objects.filter(key__contains='reverse_dns_zone',value=request.GET['zone']).filter(key__startswith='nic.')
                #Iterate through the list and get all of the key/value pairs
                tmp_list = []
                for row in keyvalue_pairs:
                    keyvalue = KeyValue.objects.filter(obj=row.system)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_dict['hostname'] = row.system.hostname
                    appendable = True
                    for the_items in tmp_list:
                        if 'hostname' not in the_items:
                            appendable = True
                        elif the_items['hostname'] == row.system.hostname:
                            appendable = False
                    if appendable is True:
                        tmp_list.append(tmp_dict)
                    #tmp_list = list(set(tmp_list))
                return tmp_list
            if key_type == 'system_by_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                keyvalue_pairs = KeyValue.objects.filter(key__contains='dhcp_scope',value=request.GET['scope']).filter(key__startswith='nic.')
                #Iterate through the list and get all of the key/value pairs
                tmp_list = []
                for row in keyvalue_pairs:
                    keyvalue = KeyValue.objects.filter(obj=row.system)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_dict['hostname'] = row.system.hostname
                    appendable = True
                    for the_items in tmp_list:
                        if 'hostname' not in the_items:
                            appendable = True
                        elif the_items['hostname'] == row.system.hostname:
                            appendable = False
                    if appendable is True:
                        tmp_list.append(tmp_dict)
                    #tmp_list = list(set(tmp_list))
                return tmp_list
            if key_type == 'adapters_by_system':
                #Get keystores from truth that have dhcp.is_scope = True
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        if matches.group(1) not in adapter_ids:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    ipv4_address = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    try:
                        final_list.append({
                            'system_hostname':system.hostname,
                            'ipv4_address':ipv4_address,
                            'adapter_name':adapter_name,
                            'mac_address':mac_address,
                            'dhcp_hostname':dhcp_hostname,
                            'dhcp_filename':dhcp_filename}
                            )
                    except:
                        pass
                #tmp_list.append(tmp_dict)
                return final_list
            if key_type == 'adapters_by_system_and_zone':
                #Get keystores from truth that have dhcp.is_scope = True
                zone = request.GET['zone']
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        dhcp_scope_match = 'nic.%s.reverse_dns_zone.0' % matches.group(1)
                        if matches.group(1) not in adapter_ids and dhcp_scope_match in tmp_dict and tmp_dict[dhcp_scope_match] == zone:
                        #if matches.group(1) not in adapter_ids and 'nic.%s.dhcp_scope.0' % matches.group(1) in tmp_dict and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] == dhcp_scope:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    dhcp_domain_name = ''
                    ipv4_address = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name.0' % a in tmp_dict:
                        dhcp_domain_name = tmp_dict['nic.%s.dhcp_domain_name.0' % a]
                    final_list.append({'system_hostname':system.hostname, 'ipv4_address':ipv4_address})
                #tmp_list.append(tmp_dict)
                return final_list
            if key_type == 'adapters_by_system_and_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                dhcp_scope = request.GET['dhcp_scope']
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        dhcp_scope_match = 'nic.%s.dhcp_scope.0' % matches.group(1)
                        if matches.group(1) not in adapter_ids and dhcp_scope_match in tmp_dict and tmp_dict[dhcp_scope_match] == dhcp_scope:
                        #if matches.group(1) not in adapter_ids and 'nic.%s.dhcp_scope.0' % matches.group(1) in tmp_dict and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] == dhcp_scope:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    dhcp_domain_name = ''
                    ipv4_address = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict and 'nic.%s.option_hostname.0' % a not in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if  'nic.%s.option_hostname.0' % a not in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.option_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name.0' % a in tmp_dict:
                        dhcp_domain_name = tmp_dict['nic.%s.dhcp_domain_name.0' % a]
                    final_list.append({'system_hostname':system.hostname, 'ipv4_address':ipv4_address,  'adapter_name':adapter_name, 'mac_address':mac_address, 'dhcp_hostname':dhcp_hostname, 'dhcp_filename':dhcp_filename, 'dhcp_domain_name':dhcp_domain_name})
                #tmp_list.append(tmp_dict)
                return final_list
        elif 'key' in request.GET and request.GET['key'] > '':
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.system.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list
        elif 'value' in request.GET:
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.system.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list

Example 39

Project: inventory
Source File: keyvalue_handler.py
View license
    def read(self, request, key_value_id=None):
        #if keystore get var is set return the whole keystore
        if 'keystore' in request.GET:
            #if key get var is set return the keystore based on the existance of this key
            if 'key' in request.GET:
                base = KeyValue.objects.filter(key=request.GET['keystore']).filter(keyvalue_set__contains=request.GET['key'])
                tmp_list = []
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.obj.hostname, r.key)
                    tmp_list[key_name] = r.value
            if 'key' not in request.GET:
                tree = KeyValueTree(request.GET['keystore']).final
                return tree
        elif 'key_type' in request.GET:
            key_type = request.GET['key_type']
            tmp_list = []
            if key_type == 'dhcp_scopes':
                #Get keystores from truth that have dhcp.is_scope = True
                base = TruthKeyValue.objects.filter(key='dhcp.is_scope',value='True')
                #Iterate through the list and get all of the key/value pairs
                for row in base:
                    keyvalue = TruthKeyValue.objects.filter(truth=row.truth)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_list.append(tmp_dict)
                return tmp_list

            if key_type == 'system_by_reverse_dns_zone':

                #Get keystores from truth that have dhcp.is_scope = True
                keyvalue_pairs = KeyValue.objects.filter(key__contains='reverse_dns_zone',value=request.GET['zone']).filter(key__startswith='nic.')
                #Iterate through the list and get all of the key/value pairs
                tmp_list = []
                for row in keyvalue_pairs:
                    keyvalue = KeyValue.objects.filter(obj=row.obj)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_dict['hostname'] = row.obj.hostname
                    appendable = True
                    for the_items in tmp_list:
                        if 'hostname' not in the_items:
                            appendable = True
                        elif the_items['hostname'] == row.obj.hostname:
                            appendable = False
                    if appendable is True:
                        tmp_list.append(tmp_dict)
                    #tmp_list = list(set(tmp_list))
                return tmp_list
            if key_type == 'system_by_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                keyvalue_pairs = KeyValue.objects.filter(key__contains='dhcp_scope',value=request.GET['scope']).filter(key__startswith='nic.')
                #Iterate through the list and get all of the key/value pairs
                tmp_list = []
                for row in keyvalue_pairs:
                    keyvalue = KeyValue.objects.filter(obj=row.obj)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_dict['hostname'] = row.obj.hostname
                    appendable = True
                    for the_items in tmp_list:
                        if 'hostname' not in the_items:
                            appendable = True
                        elif the_items['hostname'] == row.obj.hostname:
                            appendable = False
                    if appendable is True:
                        tmp_list.append(tmp_dict)
                    #tmp_list = list(set(tmp_list))
                return tmp_list
            if key_type == 'adapters_by_system':
                #Get keystores from truth that have dhcp.is_scope = True
                system = None
                try:
                    system = System.objects.get(hostname=request.GET['system'])
                except:
                    system = None
                if not system:
                    try:
                        system = System.objects.get(id=request.GET['system'])
                    except:
                        system = None
                if not system:
                    resp = rc.NOT_FOUND
                    resp.write('json = {"error_message":"Unable to find system"}')
                    return resp

                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        if matches.group(1) not in adapter_ids:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_scope = ''
                    dhcp_filename = ''
                    ipv4_address = ''
                    dhcp_domain_name_servers = ''
                    option_hostname = ""
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.option_hostname.0' % a in tmp_dict:
                        option_hostname = tmp_dict['nic.%s.option_hostname.0' % a]
                    if 'nic.%s.dhcp_scope.0' % a in tmp_dict:
                        dhcp_scope = tmp_dict['nic.%s.dhcp_scope.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name_servers.0' % a in tmp_dict:
                        dhcp_domain_name_servers = tmp_dict['nic.%s.dhcp_domain_name_servers.0' % a]
                    try:
                        final_list.append({
                                'system_hostname':system.hostname,
                                'ipv4_address':ipv4_address,
                                'adapter_name':adapter_name,
                                'mac_address':mac_address,
                                'option_hostname':option_hostname,
                                'dhcp_scope':dhcp_scope,
                                'dhcp_filename':dhcp_filename,
                                'dhcp_domain_name_servers':dhcp_domain_name_servers,
                            }
                            )
                    except Exception, e:
                        pass
                #tmp_list.append(tmp_dict)
                return final_list
            if key_type == 'adapters_by_system_and_zone':
                #Get keystores from truth that have dhcp.is_scope = True
                zone = request.GET['zone']
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        dhcp_scope_match = 'nic.%s.reverse_dns_zone.0' % matches.group(1)
                        if matches.group(1) not in adapter_ids and dhcp_scope_match in tmp_dict and tmp_dict[dhcp_scope_match] == zone:
                        #if matches.group(1) not in adapter_ids and 'nic.%s.dhcp_scope.0' % matches.group(1) in tmp_dict and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] == dhcp_scope:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    dhcp_domain_name = ''
                    ipv4_address = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name.0' % a in tmp_dict:
                        dhcp_domain_name = tmp_dict['nic.%s.dhcp_domain_name.0' % a]
                    final_list.append({'system_hostname':system.hostname, 'ipv4_address':ipv4_address})
                #tmp_list.append(tmp_dict)
                return final_list
            if 'key_type' in request.GET and request.GET['key_type'] == 'key_by_system':
                try:
                    hostname = request.GET.get('hostname')
                    key = request.GET.get('key')
                    system = System.objects.get(hostname=hostname)
                    objects = KeyValue.objects.filter(key=key, obj=system)
                    tmp = []
                    for obj in objects:
                        tmp.append({'key': obj.key, 'value': obj.value})
                    resp = rc.ALL_OK
                    resp.write("json = {'data': %s}" % json.dumps(tmp))
                except:
                    resp = rc.NOT_FOUND
                    resp.write('json = {"error_message":"Unable to find Key or system"}')

                return resp
            if key_type == 'adapters_by_system_and_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                dhcp_scope = request.GET['dhcp_scope']
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        dhcp_scope_match = 'nic.%s.dhcp_scope.0' % matches.group(1)
                        ip_address_match = 'nic.%s.ipv4_address.0' % matches.group(1)
                        if matches.group(1) not in adapter_ids and ip_address_match in tmp_dict and dhcp_scope_match in tmp_dict and tmp_dict[dhcp_scope_match] == dhcp_scope:
                        #if matches.group(1) not in adapter_ids and 'nic.%s.dhcp_scope.0' % matches.group(1) in tmp_dict and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] == dhcp_scope:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    dhcp_domain_name = ''
                    ipv4_address = ''
                    dhcp_domain_name_servers = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict and 'nic.%s.option_hostname.0' % a not in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if  'nic.%s.option_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.option_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name.0' % a in tmp_dict:
                        dhcp_domain_name = tmp_dict['nic.%s.dhcp_domain_name.0' % a]
                    if 'nic.%s.dhcp_domain_name_servers.0' % a in tmp_dict:
                        dhcp_domain_name_servers = tmp_dict['nic.%s.dhcp_domain_name_servers.0' % a]
                    final_list.append({'system_hostname':system.hostname, 'ipv4_address':ipv4_address,  'adapter_name':adapter_name, 'mac_address':mac_address, 'option_hostname': dhcp_hostname, 'dhcp_hostname':dhcp_hostname, 'dhcp_filename':dhcp_filename, 'dhcp_domain_name':dhcp_domain_name, 'dhcp_domain_name_servers':dhcp_domain_name_servers})
                #tmp_list.append(tmp_dict)
                return final_list
        elif 'key' in request.GET and request.GET['key'] > '':
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.obj.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list
        elif 'value' in request.GET:
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.obj.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list

Example 40

Project: inventory
Source File: keyvalue_handler.py
View license
    def read(self, request, key_value_id=None):
        #if keystore get var is set return the whole keystore
        if 'keystore' in request.GET:
            #if key get var is set return the keystore based on the existance of this key
            if 'key' in request.GET:
                base = KeyValue.objects.filter(key=request.GET['keystore']).filter(keyvalue_set__contains=request.GET['key'])
                tmp_list = []
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.obj.hostname, r.key)
                    tmp_list[key_name] = r.value
            if 'key' not in request.GET:
                tree = KeyValueTree(request.GET['keystore']).final
                return tree
        elif 'key_type' in request.GET:
            key_type = request.GET['key_type']
            tmp_list = []
            if key_type == 'dhcp_scopes':
                #Get keystores from truth that have dhcp.is_scope = True
                base = TruthKeyValue.objects.filter(key='dhcp.is_scope',value='True')
                #Iterate through the list and get all of the key/value pairs
                for row in base:
                    keyvalue = TruthKeyValue.objects.filter(truth=row.truth)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_list.append(tmp_dict)
                return tmp_list

            if key_type == 'system_by_reverse_dns_zone':

                #Get keystores from truth that have dhcp.is_scope = True
                keyvalue_pairs = KeyValue.objects.filter(key__contains='reverse_dns_zone',value=request.GET['zone']).filter(key__startswith='nic.')
                #Iterate through the list and get all of the key/value pairs
                tmp_list = []
                for row in keyvalue_pairs:
                    keyvalue = KeyValue.objects.filter(obj=row.obj)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_dict['hostname'] = row.obj.hostname
                    appendable = True
                    for the_items in tmp_list:
                        if 'hostname' not in the_items:
                            appendable = True
                        elif the_items['hostname'] == row.obj.hostname:
                            appendable = False
                    if appendable is True:
                        tmp_list.append(tmp_dict)
                    #tmp_list = list(set(tmp_list))
                return tmp_list
            if key_type == 'system_by_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                keyvalue_pairs = KeyValue.objects.filter(key__contains='dhcp_scope',value=request.GET['scope']).filter(key__startswith='nic.')
                #Iterate through the list and get all of the key/value pairs
                tmp_list = []
                for row in keyvalue_pairs:
                    keyvalue = KeyValue.objects.filter(obj=row.obj)
                    tmp_dict = {}
                    for kv in keyvalue:
                        tmp_dict[kv.key] = kv.value
                    tmp_dict['hostname'] = row.obj.hostname
                    appendable = True
                    for the_items in tmp_list:
                        if 'hostname' not in the_items:
                            appendable = True
                        elif the_items['hostname'] == row.obj.hostname:
                            appendable = False
                    if appendable is True:
                        tmp_list.append(tmp_dict)
                    #tmp_list = list(set(tmp_list))
                return tmp_list
            if key_type == 'adapters_by_system':
                #Get keystores from truth that have dhcp.is_scope = True
                system = None
                try:
                    system = System.objects.get(hostname=request.GET['system'])
                except:
                    system = None
                if not system:
                    try:
                        system = System.objects.get(id=request.GET['system'])
                    except:
                        system = None
                if not system:
                    resp = rc.NOT_FOUND
                    resp.write('json = {"error_message":"Unable to find system"}')
                    return resp

                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        if matches.group(1) not in adapter_ids:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_scope = ''
                    dhcp_filename = ''
                    ipv4_address = ''
                    dhcp_domain_name_servers = ''
                    option_hostname = ""
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.option_hostname.0' % a in tmp_dict:
                        option_hostname = tmp_dict['nic.%s.option_hostname.0' % a]
                    if 'nic.%s.dhcp_scope.0' % a in tmp_dict:
                        dhcp_scope = tmp_dict['nic.%s.dhcp_scope.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name_servers.0' % a in tmp_dict:
                        dhcp_domain_name_servers = tmp_dict['nic.%s.dhcp_domain_name_servers.0' % a]
                    try:
                        final_list.append({
                                'system_hostname':system.hostname,
                                'ipv4_address':ipv4_address,
                                'adapter_name':adapter_name,
                                'mac_address':mac_address,
                                'option_hostname':option_hostname,
                                'dhcp_scope':dhcp_scope,
                                'dhcp_filename':dhcp_filename,
                                'dhcp_domain_name_servers':dhcp_domain_name_servers,
                            }
                            )
                    except Exception, e:
                        pass
                #tmp_list.append(tmp_dict)
                return final_list
            if key_type == 'adapters_by_system_and_zone':
                #Get keystores from truth that have dhcp.is_scope = True
                zone = request.GET['zone']
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        dhcp_scope_match = 'nic.%s.reverse_dns_zone.0' % matches.group(1)
                        if matches.group(1) not in adapter_ids and dhcp_scope_match in tmp_dict and tmp_dict[dhcp_scope_match] == zone:
                        #if matches.group(1) not in adapter_ids and 'nic.%s.dhcp_scope.0' % matches.group(1) in tmp_dict and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] == dhcp_scope:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    dhcp_domain_name = ''
                    ipv4_address = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name.0' % a in tmp_dict:
                        dhcp_domain_name = tmp_dict['nic.%s.dhcp_domain_name.0' % a]
                    final_list.append({'system_hostname':system.hostname, 'ipv4_address':ipv4_address})
                #tmp_list.append(tmp_dict)
                return final_list
            if 'key_type' in request.GET and request.GET['key_type'] == 'key_by_system':
                try:
                    hostname = request.GET.get('hostname')
                    key = request.GET.get('key')
                    system = System.objects.get(hostname=hostname)
                    objects = KeyValue.objects.filter(key=key, obj=system)
                    tmp = []
                    for obj in objects:
                        tmp.append({'key': obj.key, 'value': obj.value})
                    resp = rc.ALL_OK
                    resp.write("json = {'data': %s}" % json.dumps(tmp))
                except:
                    resp = rc.NOT_FOUND
                    resp.write('json = {"error_message":"Unable to find Key or system"}')

                return resp
            if key_type == 'adapters_by_system_and_scope':
                #Get keystores from truth that have dhcp.is_scope = True
                dhcp_scope = request.GET['dhcp_scope']
                system = System.objects.get(hostname=request.GET['system'])
                keyvalue_pairs = KeyValue.objects.filter(key__startswith='nic.').filter(obj=system).order_by('key')
                #Iterate through the list and get all of the key/value pairs
                tmp_dict = {}
                adapter_ids = []
                final_list = []
                for kv in keyvalue_pairs:
                    tmp_dict[kv.key] = kv.value
                for k in tmp_dict.iterkeys():
                    matches = re.match('nic\.(\d+).*',k)
                    if matches.group is not None:
                        dhcp_scope_match = 'nic.%s.dhcp_scope.0' % matches.group(1)
                        ip_address_match = 'nic.%s.ipv4_address.0' % matches.group(1)
                        if matches.group(1) not in adapter_ids and ip_address_match in tmp_dict and dhcp_scope_match in tmp_dict and tmp_dict[dhcp_scope_match] == dhcp_scope:
                        #if matches.group(1) not in adapter_ids and 'nic.%s.dhcp_scope.0' % matches.group(1) in tmp_dict and tmp_dict['nic.%s.dhcp_scope.0' % matches.group(1)] == dhcp_scope:
                            adapter_ids.append(matches.group(1))
                adapter_ids.sort()
                for a in adapter_ids:
                    adapter_name = ''
                    mac_address = ''
                    dhcp_hostname = ''
                    dhcp_filename = ''
                    dhcp_domain_name = ''
                    ipv4_address = ''
                    dhcp_domain_name_servers = ''
                    if 'nic.%s.ipv4_address.0' % a in tmp_dict:
                        ipv4_address = tmp_dict['nic.%s.ipv4_address.0' % a]
                    if 'nic.%s.name.0' % a in tmp_dict:
                        adapter_name = tmp_dict['nic.%s.name.0' % a]
                    if 'nic.%s.mac_address.0' % a in tmp_dict:
                        mac_address = tmp_dict['nic.%s.mac_address.0' % a]
                    if 'nic.%s.dhcp_hostname.0' % a in tmp_dict and 'nic.%s.option_hostname.0' % a not in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.dhcp_hostname.0' % a]
                    if  'nic.%s.option_hostname.0' % a in tmp_dict:
                        dhcp_hostname = tmp_dict['nic.%s.option_hostname.0' % a]
                    if 'nic.%s.dhcp_filename.0' % a in tmp_dict:
                        dhcp_filename = tmp_dict['nic.%s.dhcp_filename.0' % a]
                    if 'nic.%s.dhcp_domain_name.0' % a in tmp_dict:
                        dhcp_domain_name = tmp_dict['nic.%s.dhcp_domain_name.0' % a]
                    if 'nic.%s.dhcp_domain_name_servers.0' % a in tmp_dict:
                        dhcp_domain_name_servers = tmp_dict['nic.%s.dhcp_domain_name_servers.0' % a]
                    final_list.append({'system_hostname':system.hostname, 'ipv4_address':ipv4_address,  'adapter_name':adapter_name, 'mac_address':mac_address, 'option_hostname': dhcp_hostname, 'dhcp_hostname':dhcp_hostname, 'dhcp_filename':dhcp_filename, 'dhcp_domain_name':dhcp_domain_name, 'dhcp_domain_name_servers':dhcp_domain_name_servers})
                #tmp_list.append(tmp_dict)
                return final_list
        elif 'key' in request.GET and request.GET['key'] > '':
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.obj.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(key=request.GET['key'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list
        elif 'value' in request.GET:
            tmp_list = {}
            try:
                base = KeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'host:%s:%s' % (r.obj.hostname, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass
            try:
                base = TruthKeyValue.objects.filter(value=request.GET['value'])
                for row in base:
                    matches = re.match("\$\{(.*)\}", row.value)
                    if matches is not None:
                        m = MacroExpansion(matches.group(1))
                        row.value = m.output()
                for r in base:
                    key_name = 'truth:%s:%s' % (r.truth.name, r.key)
                    tmp_list[key_name] = r.value
            except Exception, e:
                pass

            return tmp_list

Example 41

View license
def process_file(ly, dryrun):
    filename = ly.rsplit(".", 2)[0]
    infile = os.path.join(inFolder, ly)
    inp = open(infile, 'rb')
    with open(os.path.join(FOLDER_TEMP, FILENAME_TEMP), 'wb') as outp:
        outp.write("""
    \header{{
        tagline = " " %remove the »Music engraving by LilyPond«
    }}

    \paper {{
      myStaffSize = #20
      %{{
         run
             lilypond -dshow-available-fonts blabla
         to show all fonts available in the process log.
      %}}

      #(define fonts
        (make-pango-font-tree "Linux Libertine"
                              "Linux Libertine"
                              "Linux Libertine Mono"
        (/ myStaffSize 20)))
        %system-system-spacing #'stretchability = #0
        %ragged-last-bottom = ##t
        %ragged-bottom = ##t
        %print-page-number = ##f
        #(set-paper-size "a4")
    }}
    """.format(margin=0))

        tw = inp.read()
        tw = tw.decode("utf-8")
        tw = tw.replace(u"\ufeff", "")
        inpaper = False
        name = None
        removed_lines = []
        markup = False
        markupc = 0
        song_text = []
        composer = poet = ""
        for line in tw.split("\n"):
            r = re.match(r'\W*title\W*=\W*"([^"]+)"', line)
            rcomposer = re.match(r'\W*composer\W*=\W*"([^"]+)"', line)
            rpoet = re.match(r'\W*poet\W*=\W*"([^"]+)"', line)
            komplizierter_poet = re.findall(r'"([^"]+)"', line)
            if inpaper or "\paper" in line:
                inpaper = True
                if "}" in line:
                    inpaper = False
                removed_lines.append(line)
            if markup or re.findall(r"^\s*\\markup", line):
                markup = True
                markupc += line.count("{")
                markupc -= line.count("}")
                if markupc == 0:
                    markup = False
                if r"\bold" in line:
                    num = re.findall(r'"\s*(\d+)\s*\.?\s*"', line)
                    if num:
                        song_text.append(num[0])
                    else:
                        line = re.sub(
                            '["{}]',
                            "",
                            re.sub(r"\\.*?[{ ]", "", line)
                        )
                        song_text.append(line)
                else:
                    m = re.match(r'\s*"([^"]*)"\s*', line)
                    if m:
                        song_text.append(m.groups()[0])
                    else:
                        removed_lines.append(line)
            elif r and len(r.groups()) == 1:
                name = r.groups()[0]
                removed_lines.append(line)
            elif rcomposer and len(rcomposer.groups()) == 1:
                composer = rcomposer.groups()[0]
            elif rpoet and len(rpoet.groups()) == 1:
                poet = rpoet.groups()[0]
            elif "set-global-staff-size" in line:
                removed_lines.append(line)
            elif "tagline" in line:
                removed_lines.append(line)
            elif "set-default-paper-size" in line:
                removed_lines.append(line)
            elif "version" in line:
                removed_lines.append(line)
            elif "opus" in line:
                removed_lines.append(line)
            elif r"\tempo" in line:
                removed_lines.append(line)
            elif any((x in line for x in ("top-margin", "bottom-margin", "left-margin", "right-margin", "line-width"))):
                removed_lines.append(line)
            elif "copyright" in line and "=" in line and "\"" in line:
                removed_lines.append(line)
            elif "subtitle" in line and "=" in line:
                removed_lines.append(line)
            elif "poet" in line and komplizierter_poet:
                #print komplizierter_poet
                komplizierter_poet = map(lambda x: x, komplizierter_poet)
                gr = u"\n".join(komplizierter_poet[1:])
                poet = u"{0} {1}".format(komplizierter_poet[0], gr)
                #print type(poet), poet
            else:
                outp.write(line.encode("utf-8"))
                outp.write("\n")

        inp.seek(0)
        file_content = inp.read()
        inp.close()

    to_return = {}

    ext = "eps"
    if (not dryrun):
        cl = [
            "lilypond",
            #"-V",
            "-I",
            os.path.abspath("../"),
            "--ps",
            "-d",
            "point-and-click=#f",
            "-dbackend=eps",
            "-o",
            "tmp",  # TODO: GLOBAL VARIABLE FOR THIS!
            FILENAME_TEMP
        ]
        sub = subprocess.Popen(
            cl,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=FOLDER_TEMP,
        )
        status = sub.wait()

        to_return.update({
            "lilypond": status,
            "lilypond_stdout": sub.stdout.read(),
            "lilypond_stderr": sub.stderr.read(),
        })

        shutil.copy(
            os.path.join(FOLDER_TEMP, "tmp.{}".format(ext)),
            os.path.join(outFolder, "{}.{}".format(filename, ext))
        )

    to_return.update({
        "file_content": file_content,
        "removed_lines": removed_lines,
        "data": {
            "filename": "{}.{}".format(filename, ext),
            "name": name,
            "poet": poet,
            "composer": composer,
            "text": song_text,
        },
        "lilypond": -1,
    })
    return to_return

Example 42

Project: parlparse
Source File: parse-written-answers.py
View license
    def parse(self,filename):

        m = re.match('(?ims)^.*(day-(wa-\d\d)_([a-z0-9]+\.htm))',filename)
        if not m:
            raise Exception, "Couldn't parse filename: "+filename
        self.original_url = "http://www.scottish.parliament.uk/business/pqa/%s/%s" % (m.group(2),m.group(3))

        filename_leaf = m.group(1)

        # We need to know what date this is, so deal with that first
        # of all in a brutish fashion, but cache the results:

        self.date = None

        if file_to_date.has_key(filename_leaf):
            if verbose: print "Found file to date mapping in cache."
            self.date = datetime.date(*strptime(file_to_date[filename_leaf],"%Y-%m-%d")[0:3])
        else:
            self.make_soup(filename)            
            page_as_text = tidy_string(non_tag_data_in(self.soup.find('body')))
            m = re.search('(?ims) (Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday) (\d+)\w* (\w+) (\d+)?',page_as_text)
            if m:
                day_of_week = m.group(1)
                day = m.group(2)
                month = month_name_to_int(m.group(3))
                year = m.group(4)
                # Sometimes the date string doesn't have the year:
                if not year:
                    m = re.search('day-wa-(\d\d)',filename)
                    if m.group(1) == '99':
                        year = '1999'
                    else:
                        year = '20' + m.group(1)
                self.date = datetime.date( int(year,10), month, int(day,10) )
                if not options.quiet: "Adding file to date mapping to cache."
                add_file_to_date_mapping(filename_leaf,str(self.date))
            else:
                raise Exception, "No date found in file: "+filename

        temp_output_filename = xml_output_directory + "tmp.xml"
        output_filename = xml_output_directory + "spwa" + str(self.date) + ".xml"

        if os.path.exists(output_filename):
            #error = "The output file "+output_filename+" already exists - skipping "+re.sub('^.*/','',filename)
            # raise Exception, error
            #if not options.quiet: print error
            return

        if not options.quiet: print "Parsing %s" % filename

        self.make_soup(filename)

        self.ofp = open(temp_output_filename,"w")

        self.ofp.write('''<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE publicwhip [

<!ENTITY pound   "&#163;">
<!ENTITY euro    "&#8364;">

<!ENTITY agrave  "&#224;">
<!ENTITY aacute  "&#225;">
<!ENTITY egrave  "&#232;">
<!ENTITY eacute  "&#233;">
<!ENTITY ecirc   "&#234;">
<!ENTITY iacute  "&#237;">
<!ENTITY ograve  "&#242;">
<!ENTITY oacute  "&#243;">
<!ENTITY uacute  "&#250;">
<!ENTITY Aacute  "&#193;">
<!ENTITY Eacute  "&#201;">
<!ENTITY Iacute  "&#205;">
<!ENTITY Oacute  "&#211;">
<!ENTITY Uacute  "&#218;">
<!ENTITY Uuml    "&#220;">
<!ENTITY auml    "&#228;">
<!ENTITY euml    "&#235;">
<!ENTITY iuml    "&#239;">
<!ENTITY ouml    "&#246;">
<!ENTITY uuml    "&#252;">
<!ENTITY fnof    "&#402;">
<!ENTITY aelig   "&#230;">
<!ENTITY dagger  "&#8224;">
<!ENTITY reg     "&#174;">
<!ENTITY nbsp    "&#160;">
<!ENTITY shy     "&#173;">
<!ENTITY deg     "&#176;">
<!ENTITY middot  "&#183;">
<!ENTITY ordm    "&#186;">
<!ENTITY ndash   "&#8211;">
<!ENTITY mdash   "&#8212;">
<!ENTITY lsquo   "&#8216;">
<!ENTITY rsquo   "&#8217;">
<!ENTITY ldquo   "&#8220;">
<!ENTITY rdquo   "&#8221;">
<!ENTITY hellip  "&#8230;">
<!ENTITY bull    "&#8226;">

<!ENTITY acirc   "&#226;">
<!ENTITY Agrave  "&#192;">
<!ENTITY Aring   "&#197;">
<!ENTITY aring   "&#229;">
<!ENTITY atilde  "&#227;">
<!ENTITY Ccedil  "&#199;">
<!ENTITY ccedil  "&#231;">
<!ENTITY Egrave  "&#200;">
<!ENTITY Icirc   "&#206;">
<!ENTITY icirc   "&#238;">
<!ENTITY Igrave  "&#204;">
<!ENTITY igrave  "&#236;">
<!ENTITY ntilde  "&#241;">
<!ENTITY ocirc   "&#244;">
<!ENTITY oelig   "&#339;">
<!ENTITY Ograve  "&#210;">
<!ENTITY Oslash  "&#216;">
<!ENTITY oslash  "&#248;">
<!ENTITY Scaron  "&#352;">
<!ENTITY scaron  "&#353;">
<!ENTITY sup1    "&#185;">
<!ENTITY sup2    "&#178;">
<!ENTITY sup3    "&#179;">
<!ENTITY ugrave  "&#249;">
<!ENTITY ucirc   "&#251;">
<!ENTITY Ugrave  "&#217;">
<!ENTITY yacute  "&#253;">
<!ENTITY frac12  "&#189;">
<!ENTITY micro   "&#181;">
<!ENTITY sbquo   "&#8218;">
<!ENTITY trade   "&#8482;">
<!ENTITY Dagger  "&#8225;">
<!ENTITY radic   "&#8730;">
]>

<publicwhip>

''')

        self.ofp.write("<source url=\"%s\"/>" % self.original_url )
        
        tag_with_most_paragraphs = None
        most_paragraphs_so_far = -1
        
        for t in self.soup.findAll(True):
            ps = paragraphs_in_tag(t)
            if ps > most_paragraphs_so_far:
                tag_with_most_paragraphs = t
                most_paragraphs_so_far = ps
        
        if verbose: print "Using element name: "+tag_with_most_paragraphs.name+" with "+str(most_paragraphs_so_far)+" paragraphs from "+filename
        
        if verbose: print tag_with_most_paragraphs.prettify()
        
        # When we're parsing we might have multiple questions in a
        # row.  We say that something's a question rather than an
        # answer if (a) it's followed by an ID or (b) it begins with
        # "To ask", otherwise it's an answer.  If we hit a new
        # heading, that suggests that the previous thing was an answer
        # as well.
        
        # The business of "Holding answers" is a bit confusing.  At
        # the bottom of each page there may be a list of question IDs
        # which were given holding answers, but the text of the
        # question is not in the page - you only find it when the
        # question is eventually answered.
        
        for t in tag_with_most_paragraphs:
            if t.__class__ == NavigableString:
                s = str(t)
                s = re.sub('(?ims)\s+',' ',s)
                if re.match('(?ims)^\s*$',s):
                    continue
                else:
                    self.add_to_paragraph(tidy_string(str(t)))
                if verbose: print "string: "+str(s)            
            elif t.__class__ == Tag:
                # Look for any <a name=""> tags in here:
                a = t.find( lambda p: p.name == 'a' and p.has_key('name') )
                if a:
                    self.sp_name = a['name']
                if t.has_key('align') and t['align'].lower() == 'right':
                    # Right aligned tags just have the question ID.
                    if self.find_id_and_possible_holding_date(t):
                        self.complete_question()
                    else:
                        if verbose: print "Couldn't parse top-level right aligned tag: "+str(t)
                elif t.has_key('class') and t['class'] == 'largeHeading':
                    self.add_large_heading(tidy_string(non_tag_data_in(t)))
                elif self.something_centered(t) or self.c1_heading(t):
                    # Centred tags are headings for questions...
                    s = tidy_string(non_tag_data_in(t))
                    if len(s) > 0:
                        self.complete_answer()
                        if verbose: print "center: "+s
                        self.add_heading(s)
                elif t.name == 'table':
                    # This is probably a table that's inserted just to
                    # right align the question ID.  The left cell may
                    # contain something to indicate that it's a
                    # holding answer.
                    if self.find_id_and_possible_holding_date(t):
                        # Then also look for the "Holding answer
                        # issued" details...
                        s = non_tag_data_in(t)
                        self.find_holding_answer_issued(s)
                        self.complete_question()
                    else:
                        # Then maybe it's a table as part of the
                        # answer, so add it as a paragraph.
                        self.add_paragraph(str(t))
                elif t.name == 'p':
                    if re.search("(The following questions were given holding answers|Questions given holding answers)",tidy_string(non_tag_data_in(t))):
                        if verbose: print "Found the trailing holding question list!"
                        # This indicates the end of the day's report
                        # for us (just ignore the following list of
                        # answers - it's not very interesting until we
                        # parse some later day and we can tell what
                        # the question was...)                        
                        break
                    if verbose: print "Didn't find the trailing holding question list in: "+non_tag_data_in(t)
                    non_empty_contents = filter( lambda x: x.__class__ != NavigableString or not re.match('^\s*$',x), t.contents )
                    if len(non_empty_contents) == 0:
                        continue
                    initial_strong_text = ''
                    while len(non_empty_contents) > 0 and non_empty_contents[0].__class__ == Tag and (non_empty_contents[0].name == 'strong' or non_empty_contents[0].name == 'b'):
                        initial_strong_text += " " + non_tag_data_in(non_empty_contents[0])
                        non_empty_contents = non_empty_contents[1:]
                    if len(initial_strong_text) > 0:
                        speaker_name = tidy_string(initial_strong_text)
                        # In some files this will be the ID (possibly
                        # plus holding indication), not right aligned
                        # as usual :(
                        if self.find_id_and_possible_holding_date(speaker_name):
                            self.complete_question()
                        else:
                            speaker_name = re.sub('(?ims)\s*:\s*$','',speaker_name)
                            person_id = self.valid_speaker(speaker_name)
                            if speaker_name and person_id:
                                self.complete_answer()
                                self.set_speaker(speaker_name,person_id)
                                for e in non_empty_contents:
                                    s = tidy_string(str(e))
                                    self.add_to_paragraph(s)
                            else:
                                self.add_paragraph_removing_enclosure(t)
                    else:
                        self.add_paragraph_removing_enclosure(t)
                elif t.name == 'div' or t.name == 'blockquote' or t.name == 'ol' or t.name == 'ul' or t.name == 'center':
                    # Just add them in a paragraph anyway, even though
                    # that wouldn't be valid HTML 4 strict in the case
                    # of the last three (IIRC)
                    self.add_paragraph(str(t))
                else:
                    # Well, if it's empty of text we don't care...
                    s = non_tag_data_in(t)
                    if not re.match('(?ims)^\s*$',s):
                        raise Exception, "Unknown tag found of name '"+t.name+"' with text: "+t.prettify()
        self.complete_answer()

        # Now output all the XML, working out IDs for each element.
        # IDs are of the form:
        # 
        #   uk.org.publicwhip/spwa/YYYY-MM-DD.X.T
        # 
        #     .... where:
        #            - YYYY-MM-DD is an ISO 8601 date
        # 
        #            - X is a integer starting at 0 on each day, which
        #              should be incremented for each new heading and
        #              be the same for a group of questions and their
        #              answer.
        #
        #            - T is "mh" or "h" for major and minor headings,
        #             "q0", "q1", "q2", etc. for each group of
        #             questions and "r0", "r1", etc. for the answers

        x = -1
        last_heading = None
        current_sp_id = None

        index = 0

        for i in range(0,len(self.all_stuff)):

            if i > 0:
                previous = self.all_stuff[i-1]
            else:
                previous = None

            if i < (len(self.all_stuff) - 1):
                next = self.all_stuff[i+1]
            else:
                next = None
                
            a = self.all_stuff[i]

            self.ofp.write('\n\n')

            if a.__class__ == Heading:
                last_was_answer = True
                if a.major:
                    subtype = "mh"
                else:
                    subtype = "h"
                if next and next.__class__ == QuestionOrReply and next.sp_id:
                    # Then use the question's sp_id:
                    self.ofp.write(a.to_xml(self.get_id(next.sp_id,subtype)))
                else:
                    x += 1
                    self.ofp.write(a.to_xml(self.get_id(str(x),subtype)))
                last_heading = a
            elif a.__class__ == QuestionOrReply:
                # Occasionally we think questions are actually
                # answers, so check the beginning of the first
                # paragraph:
                if not a.is_question and len(a.paragraphs) > 0 and re.search('^(?ims)\s*To\s+ask',a.paragraphs[0]):
                    a.is_question = True
                # If we're suddenly in an answer, reset index.
                if (not a.is_question) and previous and not (previous.__class__ == QuestionOrReply and not previous.is_question):
                    index = 0
                # If we're suddenly in a question, reset index and increment x unless the previous is a heading
                elif a.is_question:
                    if previous:
                        if previous.__class__ == QuestionOrReply:
                            if previous.is_question:
                                # If the one before is a question, that's fine.
                                current_sp_id = a.sp_id
                            else:
                                current_sp_id = a.sp_id
                                # If the previous one was an answer
                                # then we need to replay the last
                                # heading:
                                if not last_heading:
                                    raise Exception, "Somehow there's been no heading so far."
                                last_heading.sp_name = a.sp_name
                                if current_sp_id:
                                    self.ofp.write(last_heading.to_xml(self.get_id(current_sp_id,"h")))
                                else:
                                    x += 1
                                    self.ofp.write(last_heading.to_xml(self.get_id(str(x),"h")))
                                self.ofp.write("\n\n")
                                index = 0
                        else:
                            # i.e. this is the normal case, a question after a heading:
                            current_sp_id = a.sp_id
                            index = 0
                    else:
                        raise Exception, "Nothing before the first question (no heading)"
                if a.is_question:
                    subtype = "q" + str(index)
                else:
                    subtype = "r" + str(index)
                if current_sp_id:
                    self.ofp.write(a.to_xml(self.get_id(current_sp_id,subtype)))
                else:
                    self.ofp.write(a.to_xml(self.get_id(str(x),subtype)))
                index += 1

        self.ofp.write("</publicwhip>")
        self.ofp.close()

        retcode = call( [ "mv", temp_output_filename, output_filename ] )
        if retcode != 0:
            raise Exception, "Moving "+temp_output_filename+" to "+output_filename+" failed."

        xmlvalidate.parse(output_filename)
        #retcode = call( [ "xmlstarlet", "val", output_filename ] )
        #if retcode != 0:
        #    raise Exception, "Validating "+output_filename+" for well-formedness failed."

        fil = open('%schangedates.txt' % xml_output_directory, 'a+')
        fil.write('%d,spwa%s.xml\n' % (time.time(), self.date))
        fil.close()

Example 43

Project: parlparse
Source File: parse-written-answers.py
View license
    def parse(self,filename):

        m = re.match('(?ims)^.*(day-(wa-\d\d)_([a-z0-9]+\.htm))',filename)
        if not m:
            raise Exception, "Couldn't parse filename: "+filename
        self.original_url = "http://www.scottish.parliament.uk/business/pqa/%s/%s" % (m.group(2),m.group(3))

        filename_leaf = m.group(1)

        # We need to know what date this is, so deal with that first
        # of all in a brutish fashion, but cache the results:

        self.date = None

        if file_to_date.has_key(filename_leaf):
            if verbose: print "Found file to date mapping in cache."
            self.date = datetime.date(*strptime(file_to_date[filename_leaf],"%Y-%m-%d")[0:3])
        else:
            self.make_soup(filename)            
            page_as_text = tidy_string(non_tag_data_in(self.soup.find('body')))
            m = re.search('(?ims) (Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday) (\d+)\w* (\w+) (\d+)?',page_as_text)
            if m:
                day_of_week = m.group(1)
                day = m.group(2)
                month = month_name_to_int(m.group(3))
                year = m.group(4)
                # Sometimes the date string doesn't have the year:
                if not year:
                    m = re.search('day-wa-(\d\d)',filename)
                    if m.group(1) == '99':
                        year = '1999'
                    else:
                        year = '20' + m.group(1)
                self.date = datetime.date( int(year,10), month, int(day,10) )
                if not options.quiet: "Adding file to date mapping to cache."
                add_file_to_date_mapping(filename_leaf,str(self.date))
            else:
                raise Exception, "No date found in file: "+filename

        temp_output_filename = xml_output_directory + "tmp.xml"
        output_filename = xml_output_directory + "spwa" + str(self.date) + ".xml"

        if os.path.exists(output_filename):
            #error = "The output file "+output_filename+" already exists - skipping "+re.sub('^.*/','',filename)
            # raise Exception, error
            #if not options.quiet: print error
            return

        if not options.quiet: print "Parsing %s" % filename

        self.make_soup(filename)

        self.ofp = open(temp_output_filename,"w")

        self.ofp.write('''<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE publicwhip [

<!ENTITY pound   "&#163;">
<!ENTITY euro    "&#8364;">

<!ENTITY agrave  "&#224;">
<!ENTITY aacute  "&#225;">
<!ENTITY egrave  "&#232;">
<!ENTITY eacute  "&#233;">
<!ENTITY ecirc   "&#234;">
<!ENTITY iacute  "&#237;">
<!ENTITY ograve  "&#242;">
<!ENTITY oacute  "&#243;">
<!ENTITY uacute  "&#250;">
<!ENTITY Aacute  "&#193;">
<!ENTITY Eacute  "&#201;">
<!ENTITY Iacute  "&#205;">
<!ENTITY Oacute  "&#211;">
<!ENTITY Uacute  "&#218;">
<!ENTITY Uuml    "&#220;">
<!ENTITY auml    "&#228;">
<!ENTITY euml    "&#235;">
<!ENTITY iuml    "&#239;">
<!ENTITY ouml    "&#246;">
<!ENTITY uuml    "&#252;">
<!ENTITY fnof    "&#402;">
<!ENTITY aelig   "&#230;">
<!ENTITY dagger  "&#8224;">
<!ENTITY reg     "&#174;">
<!ENTITY nbsp    "&#160;">
<!ENTITY shy     "&#173;">
<!ENTITY deg     "&#176;">
<!ENTITY middot  "&#183;">
<!ENTITY ordm    "&#186;">
<!ENTITY ndash   "&#8211;">
<!ENTITY mdash   "&#8212;">
<!ENTITY lsquo   "&#8216;">
<!ENTITY rsquo   "&#8217;">
<!ENTITY ldquo   "&#8220;">
<!ENTITY rdquo   "&#8221;">
<!ENTITY hellip  "&#8230;">
<!ENTITY bull    "&#8226;">

<!ENTITY acirc   "&#226;">
<!ENTITY Agrave  "&#192;">
<!ENTITY Aring   "&#197;">
<!ENTITY aring   "&#229;">
<!ENTITY atilde  "&#227;">
<!ENTITY Ccedil  "&#199;">
<!ENTITY ccedil  "&#231;">
<!ENTITY Egrave  "&#200;">
<!ENTITY Icirc   "&#206;">
<!ENTITY icirc   "&#238;">
<!ENTITY Igrave  "&#204;">
<!ENTITY igrave  "&#236;">
<!ENTITY ntilde  "&#241;">
<!ENTITY ocirc   "&#244;">
<!ENTITY oelig   "&#339;">
<!ENTITY Ograve  "&#210;">
<!ENTITY Oslash  "&#216;">
<!ENTITY oslash  "&#248;">
<!ENTITY Scaron  "&#352;">
<!ENTITY scaron  "&#353;">
<!ENTITY sup1    "&#185;">
<!ENTITY sup2    "&#178;">
<!ENTITY sup3    "&#179;">
<!ENTITY ugrave  "&#249;">
<!ENTITY ucirc   "&#251;">
<!ENTITY Ugrave  "&#217;">
<!ENTITY yacute  "&#253;">
<!ENTITY frac12  "&#189;">
<!ENTITY micro   "&#181;">
<!ENTITY sbquo   "&#8218;">
<!ENTITY trade   "&#8482;">
<!ENTITY Dagger  "&#8225;">
<!ENTITY radic   "&#8730;">
]>

<publicwhip>

''')

        self.ofp.write("<source url=\"%s\"/>" % self.original_url )
        
        tag_with_most_paragraphs = None
        most_paragraphs_so_far = -1
        
        for t in self.soup.findAll(True):
            ps = paragraphs_in_tag(t)
            if ps > most_paragraphs_so_far:
                tag_with_most_paragraphs = t
                most_paragraphs_so_far = ps
        
        if verbose: print "Using element name: "+tag_with_most_paragraphs.name+" with "+str(most_paragraphs_so_far)+" paragraphs from "+filename
        
        if verbose: print tag_with_most_paragraphs.prettify()
        
        # When we're parsing we might have multiple questions in a
        # row.  We say that something's a question rather than an
        # answer if (a) it's followed by an ID or (b) it begins with
        # "To ask", otherwise it's an answer.  If we hit a new
        # heading, that suggests that the previous thing was an answer
        # as well.
        
        # The business of "Holding answers" is a bit confusing.  At
        # the bottom of each page there may be a list of question IDs
        # which were given holding answers, but the text of the
        # question is not in the page - you only find it when the
        # question is eventually answered.
        
        for t in tag_with_most_paragraphs:
            if t.__class__ == NavigableString:
                s = str(t)
                s = re.sub('(?ims)\s+',' ',s)
                if re.match('(?ims)^\s*$',s):
                    continue
                else:
                    self.add_to_paragraph(tidy_string(str(t)))
                if verbose: print "string: "+str(s)            
            elif t.__class__ == Tag:
                # Look for any <a name=""> tags in here:
                a = t.find( lambda p: p.name == 'a' and p.has_key('name') )
                if a:
                    self.sp_name = a['name']
                if t.has_key('align') and t['align'].lower() == 'right':
                    # Right aligned tags just have the question ID.
                    if self.find_id_and_possible_holding_date(t):
                        self.complete_question()
                    else:
                        if verbose: print "Couldn't parse top-level right aligned tag: "+str(t)
                elif t.has_key('class') and t['class'] == 'largeHeading':
                    self.add_large_heading(tidy_string(non_tag_data_in(t)))
                elif self.something_centered(t) or self.c1_heading(t):
                    # Centred tags are headings for questions...
                    s = tidy_string(non_tag_data_in(t))
                    if len(s) > 0:
                        self.complete_answer()
                        if verbose: print "center: "+s
                        self.add_heading(s)
                elif t.name == 'table':
                    # This is probably a table that's inserted just to
                    # right align the question ID.  The left cell may
                    # contain something to indicate that it's a
                    # holding answer.
                    if self.find_id_and_possible_holding_date(t):
                        # Then also look for the "Holding answer
                        # issued" details...
                        s = non_tag_data_in(t)
                        self.find_holding_answer_issued(s)
                        self.complete_question()
                    else:
                        # Then maybe it's a table as part of the
                        # answer, so add it as a paragraph.
                        self.add_paragraph(str(t))
                elif t.name == 'p':
                    if re.search("(The following questions were given holding answers|Questions given holding answers)",tidy_string(non_tag_data_in(t))):
                        if verbose: print "Found the trailing holding question list!"
                        # This indicates the end of the day's report
                        # for us (just ignore the following list of
                        # answers - it's not very interesting until we
                        # parse some later day and we can tell what
                        # the question was...)                        
                        break
                    if verbose: print "Didn't find the trailing holding question list in: "+non_tag_data_in(t)
                    non_empty_contents = filter( lambda x: x.__class__ != NavigableString or not re.match('^\s*$',x), t.contents )
                    if len(non_empty_contents) == 0:
                        continue
                    initial_strong_text = ''
                    while len(non_empty_contents) > 0 and non_empty_contents[0].__class__ == Tag and (non_empty_contents[0].name == 'strong' or non_empty_contents[0].name == 'b'):
                        initial_strong_text += " " + non_tag_data_in(non_empty_contents[0])
                        non_empty_contents = non_empty_contents[1:]
                    if len(initial_strong_text) > 0:
                        speaker_name = tidy_string(initial_strong_text)
                        # In some files this will be the ID (possibly
                        # plus holding indication), not right aligned
                        # as usual :(
                        if self.find_id_and_possible_holding_date(speaker_name):
                            self.complete_question()
                        else:
                            speaker_name = re.sub('(?ims)\s*:\s*$','',speaker_name)
                            person_id = self.valid_speaker(speaker_name)
                            if speaker_name and person_id:
                                self.complete_answer()
                                self.set_speaker(speaker_name,person_id)
                                for e in non_empty_contents:
                                    s = tidy_string(str(e))
                                    self.add_to_paragraph(s)
                            else:
                                self.add_paragraph_removing_enclosure(t)
                    else:
                        self.add_paragraph_removing_enclosure(t)
                elif t.name == 'div' or t.name == 'blockquote' or t.name == 'ol' or t.name == 'ul' or t.name == 'center':
                    # Just add them in a paragraph anyway, even though
                    # that wouldn't be valid HTML 4 strict in the case
                    # of the last three (IIRC)
                    self.add_paragraph(str(t))
                else:
                    # Well, if it's empty of text we don't care...
                    s = non_tag_data_in(t)
                    if not re.match('(?ims)^\s*$',s):
                        raise Exception, "Unknown tag found of name '"+t.name+"' with text: "+t.prettify()
        self.complete_answer()

        # Now output all the XML, working out IDs for each element.
        # IDs are of the form:
        # 
        #   uk.org.publicwhip/spwa/YYYY-MM-DD.X.T
        # 
        #     .... where:
        #            - YYYY-MM-DD is an ISO 8601 date
        # 
        #            - X is a integer starting at 0 on each day, which
        #              should be incremented for each new heading and
        #              be the same for a group of questions and their
        #              answer.
        #
        #            - T is "mh" or "h" for major and minor headings,
        #             "q0", "q1", "q2", etc. for each group of
        #             questions and "r0", "r1", etc. for the answers

        x = -1
        last_heading = None
        current_sp_id = None

        index = 0

        for i in range(0,len(self.all_stuff)):

            if i > 0:
                previous = self.all_stuff[i-1]
            else:
                previous = None

            if i < (len(self.all_stuff) - 1):
                next = self.all_stuff[i+1]
            else:
                next = None
                
            a = self.all_stuff[i]

            self.ofp.write('\n\n')

            if a.__class__ == Heading:
                last_was_answer = True
                if a.major:
                    subtype = "mh"
                else:
                    subtype = "h"
                if next and next.__class__ == QuestionOrReply and next.sp_id:
                    # Then use the question's sp_id:
                    self.ofp.write(a.to_xml(self.get_id(next.sp_id,subtype)))
                else:
                    x += 1
                    self.ofp.write(a.to_xml(self.get_id(str(x),subtype)))
                last_heading = a
            elif a.__class__ == QuestionOrReply:
                # Occasionally we think questions are actually
                # answers, so check the beginning of the first
                # paragraph:
                if not a.is_question and len(a.paragraphs) > 0 and re.search('^(?ims)\s*To\s+ask',a.paragraphs[0]):
                    a.is_question = True
                # If we're suddenly in an answer, reset index.
                if (not a.is_question) and previous and not (previous.__class__ == QuestionOrReply and not previous.is_question):
                    index = 0
                # If we're suddenly in a question, reset index and increment x unless the previous is a heading
                elif a.is_question:
                    if previous:
                        if previous.__class__ == QuestionOrReply:
                            if previous.is_question:
                                # If the one before is a question, that's fine.
                                current_sp_id = a.sp_id
                            else:
                                current_sp_id = a.sp_id
                                # If the previous one was an answer
                                # then we need to replay the last
                                # heading:
                                if not last_heading:
                                    raise Exception, "Somehow there's been no heading so far."
                                last_heading.sp_name = a.sp_name
                                if current_sp_id:
                                    self.ofp.write(last_heading.to_xml(self.get_id(current_sp_id,"h")))
                                else:
                                    x += 1
                                    self.ofp.write(last_heading.to_xml(self.get_id(str(x),"h")))
                                self.ofp.write("\n\n")
                                index = 0
                        else:
                            # i.e. this is the normal case, a question after a heading:
                            current_sp_id = a.sp_id
                            index = 0
                    else:
                        raise Exception, "Nothing before the first question (no heading)"
                if a.is_question:
                    subtype = "q" + str(index)
                else:
                    subtype = "r" + str(index)
                if current_sp_id:
                    self.ofp.write(a.to_xml(self.get_id(current_sp_id,subtype)))
                else:
                    self.ofp.write(a.to_xml(self.get_id(str(x),subtype)))
                index += 1

        self.ofp.write("</publicwhip>")
        self.ofp.close()

        retcode = call( [ "mv", temp_output_filename, output_filename ] )
        if retcode != 0:
            raise Exception, "Moving "+temp_output_filename+" to "+output_filename+" failed."

        xmlvalidate.parse(output_filename)
        #retcode = call( [ "xmlstarlet", "val", output_filename ] )
        #if retcode != 0:
        #    raise Exception, "Validating "+output_filename+" for well-formedness failed."

        fil = open('%schangedates.txt' % xml_output_directory, 'a+')
        fil.write('%d,spwa%s.xml\n' % (time.time(), self.date))
        fil.close()

Example 44

Project: tcollector
Source File: procstats.py
View license
def main():
    """procstats main loop"""

    f_uptime = open("/proc/uptime", "r")
    f_meminfo = open("/proc/meminfo", "r")
    f_vmstat = open("/proc/vmstat", "r")
    f_stat = open("/proc/stat", "r")
    f_loadavg = open("/proc/loadavg", "r")
    f_entropy_avail = open("/proc/sys/kernel/random/entropy_avail", "r")
    f_interrupts = open("/proc/interrupts", "r")

    f_scaling = "/sys/devices/system/cpu/cpu%s/cpufreq/%s_freq"
    f_scaling_min  = dict([])
    f_scaling_max  = dict([])
    f_scaling_cur  = dict([])
    f_softirqs = open("/proc/softirqs", "r")
    for cpu in glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_cur_freq"):
        m = re.match("/sys/devices/system/cpu/cpu([0-9]*)/cpufreq/scaling_cur_freq", cpu)
        if not m:
            continue
        cpu_no = m.group(1)
        sys.stderr.write(f_scaling % (cpu_no,"min"))
        f_scaling_min[cpu_no] = open(f_scaling % (cpu_no,"cpuinfo_min"), "r")
        f_scaling_max[cpu_no] = open(f_scaling % (cpu_no,"cpuinfo_max"), "r")
        f_scaling_cur[cpu_no] = open(f_scaling % (cpu_no,"scaling_cur"), "r")

    numastats = find_sysfs_numa_stats()
    utils.drop_privileges()

    while True:
        # proc.uptime
        f_uptime.seek(0)
        ts = int(time.time())
        for line in f_uptime:
            m = re.match("(\S+)\s+(\S+)", line)
            if m:
                print "proc.uptime.total %d %s" % (ts, m.group(1))
                print "proc.uptime.now %d %s" % (ts, m.group(2))

        # proc.meminfo
        f_meminfo.seek(0)
        ts = int(time.time())
        for line in f_meminfo:
            m = re.match("(\w+):\s+(\d+)\s+(\w+)", line)
            if m:
                if m.group(3).lower() == 'kb':
                    # convert from kB to B for easier graphing
                    value = str(int(m.group(2)) * 1024)
                else:
                    value = m.group(2)
                print ("proc.meminfo.%s %d %s"
                        % (m.group(1).lower(), ts, value))

        # proc.vmstat
        f_vmstat.seek(0)
        ts = int(time.time())
        for line in f_vmstat:
            m = re.match("(\w+)\s+(\d+)", line)
            if not m:
                continue
            if m.group(1) in ("pgpgin", "pgpgout", "pswpin",
                              "pswpout", "pgfault", "pgmajfault"):
                print "proc.vmstat.%s %d %s" % (m.group(1), ts, m.group(2))

        # proc.stat
        f_stat.seek(0)
        ts = int(time.time())
        for line in f_stat:
            m = re.match("(\w+)\s+(.*)", line)
            if not m:
                continue
            if m.group(1).startswith("cpu"):
                cpu_m = re.match("cpu(\d+)", m.group(1))
                if cpu_m:
                    metric_percpu = '.percpu'
                    tags = ' cpu=%s' % cpu_m.group(1)
                else:
                    metric_percpu = ''
                    tags = ''
                fields = m.group(2).split()
                cpu_types = ['user', 'nice', 'system', 'idle', 'iowait',
                    'irq', 'softirq', 'guest', 'guest_nice']

                # We use zip to ignore fields that don't exist.
                for value, field_name in zip(fields, cpu_types):
                    print "proc.stat.cpu%s %d %s type=%s%s" % (metric_percpu,
                        ts, value, field_name, tags)
            elif m.group(1) == "intr":
                print ("proc.stat.intr %d %s"
                        % (ts, m.group(2).split()[0]))
            elif m.group(1) == "ctxt":
                print "proc.stat.ctxt %d %s" % (ts, m.group(2))
            elif m.group(1) == "processes":
                print "proc.stat.processes %d %s" % (ts, m.group(2))
            elif m.group(1) == "procs_blocked":
                print "proc.stat.procs_blocked %d %s" % (ts, m.group(2))

        f_loadavg.seek(0)
        ts = int(time.time())
        for line in f_loadavg:
            m = re.match("(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)\s+", line)
            if not m:
                continue
            print "proc.loadavg.1min %d %s" % (ts, m.group(1))
            print "proc.loadavg.5min %d %s" % (ts, m.group(2))
            print "proc.loadavg.15min %d %s" % (ts, m.group(3))
            print "proc.loadavg.runnable %d %s" % (ts, m.group(4))
            print "proc.loadavg.total_threads %d %s" % (ts, m.group(5))

        f_entropy_avail.seek(0)
        ts = int(time.time())
        for line in f_entropy_avail:
            print "proc.kernel.entropy_avail %d %s" % (ts, line.strip())

        f_interrupts.seek(0)
        ts = int(time.time())
        # Get number of CPUs from description line.
        num_cpus = len(f_interrupts.readline().split())
        for line in f_interrupts:
            cols = line.split()

            irq_type = cols[0].rstrip(":")
            if irq_type.isalnum():
                if irq_type.isdigit():
                    if cols[-2] == "PCI-MSI-edge" and "eth" in cols[-1]:
                        irq_type = cols[-1]
                    else:
                        continue  # Interrupt type is just a number, ignore.
                for i, val in enumerate(cols[1:]):
                    if i >= num_cpus:
                        # All values read, remaining cols contain textual
                        # description
                        break
                    if not val.isdigit():
                        # something is weird, there should only be digit values
                        sys.stderr.write("Unexpected interrupts value %r in"
                                         " %r: " % (val, cols))
                        break
                    print ("proc.interrupts %s %s type=%s cpu=%s"
                           % (ts, val, irq_type, i))

        f_softirqs.seek(0)
        ts = int(time.time())
        # Get number of CPUs from description line.
        num_cpus = len(f_softirqs.readline().split())
        for line in f_softirqs:
            cols = line.split()

            irq_type = cols[0].rstrip(":")
            for i, val in enumerate(cols[1:]):
                if i >= num_cpus:
                    # All values read, remaining cols contain textual
                    # description
                    break
                if not val.isdigit():
                    # something is weird, there should only be digit values
                    sys.stderr.write("Unexpected softirq value %r in"
                                     " %r: " % (val, cols))
                    break
                print ("proc.softirqs %s %s type=%s cpu=%s"
                       % (ts, val, irq_type, i))

        print_numa_stats(numastats)

        # Print scaling stats
        ts = int(time.time())
        for cpu_no in f_scaling_min.keys():
            f = f_scaling_min[cpu_no]
            f.seek(0)
            for line in f:
                print "proc.scaling.min %d %s cpu=%s" % (ts, line.rstrip('\n'), cpu_no)
        ts = int(time.time())
        for cpu_no in f_scaling_max.keys():
            f = f_scaling_max[cpu_no]
            f.seek(0)
            for line in f:
                print "proc.scaling.max %d %s cpu=%s" % (ts, line.rstrip('\n'), cpu_no)
        ts = int(time.time())
        for cpu_no in f_scaling_cur.keys():
            f = f_scaling_cur[cpu_no]
            f.seek(0)
            for line in f:
                print "proc.scaling.cur %d %s cpu=%s" % (ts, line.rstrip('\n'), cpu_no)

        sys.stdout.flush()
        time.sleep(COLLECTION_INTERVAL)

Example 45

Project: snakemakelib
Source File: scrnaseq.py
View license
def scrnaseq_pca_plots(pca_results_file=None, metadata=None, pcacomp=(1,2), pcaobjfile=None):
    """Make PCA QC plots for scrnaseq workflow

    Args:
      pca_results_file (str): pca results file
      metadata (str): metadata file name
      pcacomp (int): tuple of ints corresponding to components to draw
      pcaobjfile (str): file name containing pickled pca object

    Returns: 
      dict: dictionary with keys 'fig' pointing to a (:py:class:`~bokeh.models.GridPlot`) Bokeh GridPlot object and key 'table' pointing to a (:py:class:`~bokeh.widgets.DataTable`) DataTable

    """
    if not metadata is None:
        md = pd.read_csv(metadata, index_col=0)
    if not pcaobjfile is None:
        with open(pcaobjfile, 'rb') as fh:
            pcaobj = pickle.load(fh)
    df_pca = pd.read_csv(pca_results_file, index_col="sample")
    df_pca['color'] = ['red'] * df_pca.shape[0]
    df_pca['x'] = df_pca['0']
    df_pca['y'] = df_pca['1']

    source = ColumnDataSource(df_pca)
    TOOLS = "pan,wheel_zoom,box_zoom,box_select,resize,reset,save,hover"

    # Add radio button group
    cmap = colorbrewer(datalen = df_pca.shape[0])
    callback_rbg = CustomJS(args=dict(source=source), code="""
        var data = source.get('data');
        var active = cb_obj.get('active')
        var label = cb_obj.get('label')[active]
    var Paired = {
    3	: [ "#A6CEE3","#1F78B4","#B2DF8A"],
    4	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C"],
    5	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99"],
    6	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C"],
    7	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F"],
    8	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00"] ,
    9	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6"],
    10	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6","#6A3D9A"],
    11	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6","#6A3D9A","#FFFF99"],
    12	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6","#6A3D9A","#FFFF99", "#B15928"]};
        var colormap = {};

        var j = 0;
        for (i = 0; i < data['sample'].length; i++) {
            if (data[label][i] in colormap) {
            } else {
                colormap[data[label][i]] = j;
                j++;
            }
        }
        var nfac = Object.keys(colormap).length;
        if (nfac > 12) {
            nfac = 12;
        } 
        if (nfac < 3) {
           nfac = 3;
        }
        var colors = Paired[nfac];
        for (i = 0; i < data[label].length; i++) {
              data['color'][i] = colors[colormap[data[label][i]]]
        }
        source.trigger('change');
    """)
    callback  = CustomJS(args=dict(source=source), code="""
        var data = source.get('data');
        var active = cb_obj.get('active');
        var label = cb_obj.get('label');
    var Paired = {
    3	: [ "#A6CEE3","#1F78B4","#B2DF8A"],
    4	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C"],
    5	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99"],
    6	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C"],
    7	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F"],
    8	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00"] ,
    9	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6"],
    10	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6","#6A3D9A"],
    11	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6","#6A3D9A","#FFFF99"],
    12	: [ "#A6CEE3","#1F78B4","#B2DF8A","#33A02C","#FB9A99","#E31A1C","#FDBF6F","#FF7F00","#CAB2D6","#6A3D9A","#FFFF99", "#B15928"]};
        var colormap = {};
    if (!active) {
        var j = 0;
        for (i = 0; i < data['sample'].length; i++) {
            if (data[label][i] in colormap) {
            } else {
                colormap[data[label][i]] = j;
                j++;
            }
        }
        var nfac = Object.keys(colormap).length;
        if (nfac > 12) {
            nfac = 12;
        } 
        if (nfac < 3) {
           nfac = 3;
        }
        var colors = Paired[nfac];
        for (i = 0; i < data[label].length; i++) {
              data['color'][i] = colors[colormap[data[label][i]]]
        }
        source.trigger('change');
    }
    """)
    if not md is None:
        # Waiting for callbacks to be implemented upstream in bokeh
        # rbg = RadioButtonGroup(labels=list(md.columns),
        #                        callback=callback)
        toggle_buttons = [Toggle(label=x, callback=callback) for x in list(md.columns)]
    else:
        toggle_buttons = []
        # rbg = RadioButtonGroup()
    # PC components
    xcallback = CustomJS(args=dict(source=source), code="""
        var data = source.get('data');
        var active = cb_obj.get('active')
        var value = cb_obj.get('value')
        x = data['x']
        for (i = 0; i < x.length; i++) {
              x[i] = data[value][i]
              data['sample'][i] = value
              data['FileID'][i] = active
        }

        source.trigger('change');
    """)
    ycallback = CustomJS(args=dict(source=source), code="""
        var data = source.get('data');
        var value = cb_obj.get('value')
        y = data['y']
        for (i = 0; i < y.length; i++) {
             y[i] = data[value][i]
        }
        source.trigger('change');
    """)

    pca_components = sorted([int(x) + 1 for x in source.column_names if re.match("\d+", x)])
    menulist = [(str(x), str(x)) for x in pca_components]
    component_x = Dropdown(label = "PCA component x", menu = menulist, default_value="1",
                           callback=xcallback)
    component_y = Dropdown(label = "PCA component y", menu = menulist, default_value="2",
                           callback=ycallback)

    # Make the pca plot
    kwfig = {'plot_width': 400, 'plot_height': 400,
             'title_text_font_size': "12pt"}


    p1 = figure(title="Principal component analysis",
                tools=TOOLS, **kwfig)

    points(p1, 'x', 'y', source=source, color='color', size=10,
           alpha=.7)
    kwxaxis = {'axis_label': "Component {} ({:.2f}%)".format(
        pcacomp[0], 100.0 * pcaobj.explained_variance_ratio_[pcacomp[0] - 1]),
               'axis_label_text_font_size': '10pt',
               'major_label_orientation': np.pi/3}
    kwyaxis = {'axis_label': "Component {} ({:.2f}%)".format(
        pcacomp[1], 100.0 * pcaobj.explained_variance_ratio_[pcacomp[1] - 1]),
               'axis_label_text_font_size': '10pt',
               'major_label_orientation': np.pi/3}
    xaxis(p1, **kwxaxis)
    yaxis(p1, **kwyaxis)
    tooltiplist = [("sample", "@sample")] if "sample" in source.column_names else []
    if not md is None:
        tooltiplist = tooltiplist + [(str(x), "@{}".format(x)) for x
                                     in md.columns]
    tooltips(p1, HoverTool, tooltiplist)

    # Detected genes, FPKM and TPM
    p2 = figure(title="Number of detected genes",
                x_range=list(df_pca.index), tools=TOOLS,
                **kwfig)
    kwxaxis.update({'axis_label': "Sample"})
    kwyaxis.update({'axis_label': "Detected genes"})
    dotplot(p2, "sample", "FPKM", source=source)
    xaxis(p2, **kwxaxis)
    yaxis(p2, **kwyaxis)
    tooltips(p2, HoverTool, [('sample', '@sample'),
                             ('# genes (FPKM)', '@FPKM')])
    return {'fig':vform(*(toggle_buttons + [gridplot([[p1, p2]])]))}

Example 46

Project: raspberry_pwn
Source File: hash.py
View license
def dictionaryAttack(attack_dict):
    suffix_list = [""]
    custom_wordlist = []
    hash_regexes = []
    results = []
    resumes = []
    processException = False
    user_hash = []

    for (_, hashes) in attack_dict.items():
        for hash_ in hashes:
            if not hash_:
                continue

            hash_ = hash_.split()[0] if hash_ and hash_.strip() else hash_
            regex = hashRecognition(hash_)

            if regex and regex not in hash_regexes:
                hash_regexes.append(regex)
                infoMsg = "using hash method '%s'" % __functions__[regex].func_name
                logger.info(infoMsg)

    for hash_regex in hash_regexes:
        keys = set()
        attack_info = []

        for (user, hashes) in attack_dict.items():
            for hash_ in hashes:
                if not hash_:
                    continue

                hash_ = hash_.split()[0] if hash_ and hash_.strip() else hash_

                if re.match(hash_regex, hash_):
                    item = None

                    if hash_regex not in (HASH.CRYPT_GENERIC, HASH.WORDPRESS):
                        hash_ = hash_.lower()

                    if hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC):
                        item = [(user, hash_), {}]
                    elif hash_regex in (HASH.ORACLE_OLD, HASH.POSTGRES):
                        item = [(user, hash_), {'username': user}]
                    elif hash_regex in (HASH.ORACLE):
                        item = [(user, hash_), {'salt': hash_[-20:]}]
                    elif hash_regex in (HASH.MSSQL, HASH.MSSQL_OLD, HASH.MSSQL_NEW):
                        item = [(user, hash_), {'salt': hash_[6:14]}]
                    elif hash_regex in (HASH.CRYPT_GENERIC):
                        item = [(user, hash_), {'salt': hash_[0:2]}]
                    elif hash_regex in (HASH.WORDPRESS):
                        item = [(user, hash_), {'salt': hash_[4:12], 'count': 1 << ITOA64.index(hash_[3]), 'prefix': hash_[:12]}]

                    if item and hash_ not in keys:
                        resumed = hashDBRetrieve(hash_)
                        if not resumed:
                            attack_info.append(item)
                            user_hash.append(item[0])
                        else:
                            infoMsg = "resuming password '%s' for hash '%s'" % (resumed, hash_)
                            if user and not user.startswith(DUMMY_USER_PREFIX):
                                infoMsg += " for user '%s'" % user
                            logger.info(infoMsg)
                            resumes.append((user, hash_, resumed))
                        keys.add(hash_)

        if not attack_info:
            continue

        if not kb.wordlists:
            while not kb.wordlists:

                # the slowest of all methods hence smaller default dict
                if hash_regex in (HASH.ORACLE_OLD, HASH.WORDPRESS):
                    dictPaths = [paths.SMALL_DICT]
                else:
                    dictPaths = [paths.WORDLIST]

                message = "what dictionary do you want to use?\n"
                message += "[1] default dictionary file '%s' (press Enter)\n" % dictPaths[0]
                message += "[2] custom dictionary file\n"
                message += "[3] file with list of dictionary files"
                choice = readInput(message, default="1")

                try:
                    if choice == "2":
                        message = "what's the custom dictionary's location?\n"
                        dictPaths = [readInput(message)]

                        logger.info("using custom dictionary")
                    elif choice == "3":
                        message = "what's the list file location?\n"
                        listPath = readInput(message)
                        checkFile(listPath)
                        dictPaths = getFileItems(listPath)

                        logger.info("using custom list of dictionaries")
                    else:
                        logger.info("using default dictionary")

                    dictPaths = filter(None, dictPaths)

                    for dictPath in dictPaths:
                        checkFile(dictPath)

                    kb.wordlists = dictPaths

                except SqlmapFilePathException, msg:
                    warnMsg = "there was a problem while loading dictionaries"
                    warnMsg += " ('%s')" % msg
                    logger.critical(warnMsg)

            message = "do you want to use common password suffixes? (slow!) [y/N] "
            test = readInput(message, default="N")

            if test[0] in ("y", "Y"):
                suffix_list += COMMON_PASSWORD_SUFFIXES

        infoMsg = "starting dictionary-based cracking (%s)" % __functions__[hash_regex].func_name
        logger.info(infoMsg)

        for item in attack_info:
            ((user, _), _) = item
            if user and not user.startswith(DUMMY_USER_PREFIX):
                custom_wordlist.append(normalizeUnicode(user))

        if hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC):
            for suffix in suffix_list:
                if not attack_info or processException:
                    break

                if suffix:
                    clearConsoleLine()
                    infoMsg = "using suffix '%s'" % suffix
                    logger.info(infoMsg)

                retVal = None
                processes = []

                try:
                    if _multiprocessing:
                        if _multiprocessing.cpu_count() > 1:
                            infoMsg = "starting %d processes " % _multiprocessing.cpu_count()
                            singleTimeLogMessage(infoMsg)

                        gc.disable()

                        retVal = _multiprocessing.Queue()
                        count = _multiprocessing.Value('i', _multiprocessing.cpu_count())

                        for i in xrange(_multiprocessing.cpu_count()):
                            p = _multiprocessing.Process(target=_bruteProcessVariantA, args=(attack_info, hash_regex, suffix, retVal, i, count, kb.wordlists, custom_wordlist))
                            processes.append(p)

                        for p in processes:
                            p.daemon = True
                            p.start()

                        while count.value > 0:
                            time.sleep(0.5)

                    else:
                        warnMsg = "multiprocessing hash cracking is currently "
                        warnMsg += "not supported on this platform"
                        singleTimeWarnMessage(warnMsg)

                        retVal = Queue()
                        _bruteProcessVariantA(attack_info, hash_regex, suffix, retVal, 0, 1, kb.wordlists, custom_wordlist)

                except KeyboardInterrupt:
                    print
                    processException = True
                    warnMsg = "user aborted during dictionary-based attack phase (Ctrl+C was pressed)"
                    logger.warn(warnMsg)

                    for process in processes:
                        try:
                            process.terminate()
                            process.join()
                        except OSError:
                            pass

                finally:
                    if _multiprocessing:
                        gc.enable()

                    if retVal:
                        conf.hashDB.beginTransaction()

                        while not retVal.empty():
                            user, hash_, word = item = retVal.get(block=False)
                            attack_info = filter(lambda _: _[0][0] != user or _[0][1] != hash_, attack_info)
                            hashDBWrite(hash_, word)
                            results.append(item)

                        conf.hashDB.endTransaction()

            clearConsoleLine()

        else:
            for ((user, hash_), kwargs) in attack_info:
                if processException:
                    break

                if any(_[0] == user and _[1] == hash_ for _ in results):
                    continue

                count = 0
                found = False

                for suffix in suffix_list:
                    if found or processException:
                        break

                    if suffix:
                        clearConsoleLine()
                        infoMsg = "using suffix '%s'" % suffix
                        logger.info(infoMsg)

                    retVal = None
                    processes = []

                    try:
                        if _multiprocessing:
                            if _multiprocessing.cpu_count() > 1:
                                infoMsg = "starting %d processes " % _multiprocessing.cpu_count()
                                singleTimeLogMessage(infoMsg)

                            gc.disable()

                            retVal = _multiprocessing.Queue()
                            found_ = _multiprocessing.Value('i', False)
                            count = _multiprocessing.Value('i', _multiprocessing.cpu_count())

                            for i in xrange(_multiprocessing.cpu_count()):
                                p = _multiprocessing.Process(target=_bruteProcessVariantB, args=(user, hash_, kwargs, hash_regex, suffix, retVal, found_, i, count, kb.wordlists, custom_wordlist))
                                processes.append(p)

                            for p in processes:
                                p.daemon = True
                                p.start()

                            while count.value > 0:
                                time.sleep(0.5)

                            found = found_.value != 0

                        else:
                            warnMsg = "multiprocessing hash cracking is currently "
                            warnMsg += "not supported on this platform"
                            singleTimeWarnMessage(warnMsg)

                            class Value():
                                pass

                            retVal = Queue()
                            found_ = Value()
                            found_.value = False

                            _bruteProcessVariantB(user, hash_, kwargs, hash_regex, suffix, retVal, found_, 0, 1, kb.wordlists, custom_wordlist)

                            found = found_.value

                    except KeyboardInterrupt:
                        print
                        processException = True
                        warnMsg = "user aborted during dictionary-based attack phase (Ctrl+C was pressed)"
                        logger.warn(warnMsg)

                        for process in processes:
                            try:
                                process.terminate()
                                process.join()
                            except OSError:
                                pass

                    finally:
                        if _multiprocessing:
                            gc.enable()

                        if retVal:
                            conf.hashDB.beginTransaction()

                            while not retVal.empty():
                                user, hash_, word = item = retVal.get(block=False)
                                hashDBWrite(hash_, word)
                                results.append(item)

                            conf.hashDB.endTransaction()

                clearConsoleLine()

    results.extend(resumes)

    if len(hash_regexes) == 0:
        warnMsg = "unknown hash format. "
        warnMsg += "Please report by e-mail to %s" % ML
        logger.warn(warnMsg)

    if len(results) == 0:
        warnMsg = "no clear password(s) found"
        logger.warn(warnMsg)

    return results

Example 47

View license
def check_snapslist(vm_name, options, option_dict, output,
                    snaps_before, snaps_list):
    no_metadata = options.find("--no-metadata")
    fdisks = "disks"

    # command with print-xml will not really create snapshot
    if options.find("print-xml") >= 0:
        xtf = xml_utils.XMLTreeFile(output)

        # With --print-xml there isn't new snapshot created
        if len(snaps_before) != len(snaps_list):
            raise error.TestFail("--print-xml create new snapshot")

    else:
        # The following does not check with print-xml
        get_sname = output.split()[2]

        # check domain/snapshot xml depends on if have metadata
        if no_metadata < 0:
            output_dump = virsh.snapshot_dumpxml(vm_name,
                                                 get_sname).stdout.strip()
        else:
            output_dump = virsh.dumpxml(vm_name).stdout.strip()
            fdisks = "devices"

        xtf = xml_utils.XMLTreeFile(output_dump)

        find = 0
        for snap in snaps_list:
            if snap == get_sname:
                find = 1
                break

        # Should find snap in snaplist without --no-metadata
        if (find == 0 and no_metadata < 0):
            raise error.TestFail("Can not find snapshot %s!"
                                 % get_sname)
        # Should not find snap in list without metadata
        elif (find == 1 and no_metadata >= 0):
            raise error.TestFail("Can find snapshot metadata even "
                                 "if have --no-metadata")
        elif (find == 0 and no_metadata >= 0):
            logging.info("Can not find snapshot %s as no-metadata "
                         "is given" % get_sname)

            # Check snapshot only in qemu-img
            if (options.find("--disk-only") < 0 and
                    options.find("--memspec") < 0):
                ret = check_snap_in_image(vm_name, get_sname)

                if ret is False:
                    raise error.TestFail("No snap info in image")

        else:
            logging.info("Find snapshot %s in snapshot list."
                         % get_sname)

        # Check if the disk file exist when disk-only is given
        if options.find("disk-only") >= 0:
            for disk in xtf.find(fdisks).findall('disk'):
                if disk.get('snapshot') == 'no':
                    continue
                diskpath = disk.find('source').get('file')
                if os.path.isfile(diskpath):
                    logging.info("disk file %s exist" % diskpath)
                    os.remove(diskpath)
                else:
                    # Didn't find <source file="path to disk"/>
                    # in output - this could leave a file around
                    # wherever the main OS image file is found
                    logging.debug("output_dump=%s", output_dump)
                    raise error.TestFail("Can not find disk %s"
                                         % diskpath)

        # Check if the guest is halted when 'halt' is given
        if options.find("halt") >= 0:
            domstate = virsh.domstate(vm_name)
            if re.match("shut off", domstate.stdout):
                logging.info("Domain is halted after create "
                             "snapshot")
            else:
                raise error.TestFail("Domain is not halted after "
                                     "snapshot created")

    # Check the snapshot xml regardless of having print-xml or not
    if (options.find("name") >= 0 and no_metadata < 0):
        if xtf.findtext('name') == option_dict["name"]:
            logging.info("get snapshot name same as set")
        else:
            raise error.TestFail("Get wrong snapshot name %s" %
                                 xtf.findtext('name'))

    if (options.find("description") >= 0 and no_metadata < 0):
        desc = xtf.findtext('description')
        if desc == option_dict["description"]:
            logging.info("get snapshot description same as set")
        else:
            raise error.TestFail("Get wrong description on xml")

    if options.find("diskspec") >= 0:
        if isinstance(option_dict['diskspec'], list):
            index = len(option_dict['diskspec'])
        else:
            index = 1

        disks = xtf.find(fdisks).findall('disk')

        for num in range(index):
            if isinstance(option_dict['diskspec'], list):
                option_disk = option_dict['diskspec'][num]
            else:
                option_disk = option_dict['diskspec']

            option_disk = "name=" + option_disk
            disk_dict = utils_misc.valued_option_dict(option_disk,
                                                      ",", 0, "=")
            logging.debug("disk_dict is %s", disk_dict)

            # For no metadata snapshot do not check name and
            # snapshot
            if no_metadata < 0:
                dname = disks[num].get('name')
                logging.debug("dname is %s", dname)
                if dname == disk_dict['name']:
                    logging.info("get disk%d name same as set in "
                                 "diskspec", num)
                else:
                    raise error.TestFail("Get wrong disk%d name %s"
                                         % num, dname)

                if option_disk.find('snapshot=') >= 0:
                    dsnap = disks[num].get('snapshot')
                    logging.debug("dsnap is %s", dsnap)
                    if dsnap == disk_dict['snapshot']:
                        logging.info("get disk%d snapshot type same"
                                     " as set in diskspec", num)
                    else:
                        raise error.TestFail("Get wrong disk%d "
                                             "snapshot type %s" %
                                             num, dsnap)

            if option_disk.find('driver=') >= 0:
                dtype = disks[num].find('driver').get('type')
                if dtype == disk_dict['driver']:
                    logging.info("get disk%d driver type same as "
                                 "set in diskspec", num)
                else:
                    raise error.TestFail("Get wrong disk%d driver "
                                         "type %s" % num, dtype)

            if option_disk.find('file=') >= 0:
                sfile = disks[num].find('source').get('file')
                if sfile == disk_dict['file']:
                    logging.info("get disk%d source file same as "
                                 "set in diskspec", num)
                    if os.path.exists(sfile):
                        os.unlink(sfile)
                else:
                    raise error.TestFail("Get wrong disk%d source "
                                         "file %s" % num, sfile)

    # For memspec check if the xml is same as setting
    # Also check if the mem file exists
    if options.find("memspec") >= 0:
        memspec = option_dict['memspec']
        if re.search('file=', option_dict['memspec']) < 0:
            memspec = 'file=' + option_dict['memspec']

        mem_dict = utils_misc.valued_option_dict(memspec, ",", 0,
                                                 "=")
        logging.debug("mem_dict is %s", mem_dict)

        if no_metadata < 0:
            if memspec.find('snapshot=') >= 0:
                snap = xtf.find('memory').get('snapshot')
                if snap == mem_dict['snapshot']:
                    logging.info("get memory snapshot type same as"
                                 " set in diskspec")
                else:
                    raise error.TestFail("Get wrong memory snapshot"
                                         " type on print xml")

            memfile = xtf.find('memory').get('file')
            if memfile == mem_dict['file']:
                logging.info("get memory file same as set in "
                             "diskspec")
            else:
                raise error.TestFail("Get wrong memory file on "
                                     "print xml %s", memfile)

        if options.find("print-xml") < 0:
            if os.path.isfile(mem_dict['file']):
                logging.info("memory file generated")
                os.remove(mem_dict['file'])
            else:
                raise error.TestFail("Fail to generate memory file"
                                     " %s", mem_dict['file'])

Example 48

Project: tp-qemu
Source File: virtio_scsi_mq.py
View license
@error.context_aware
def run(test, params, env):
    """
    Qemu multiqueue test for virtio-scsi controller:

    1) Boot up a guest with virtio-scsi device which support multi-queue and
       the vcpu and images number of guest should match the multi-queue number
    2) Check the multi queue option from monitor
    3) Check device init status in guest
    4) Load I/O in all targets
    5) Check the interrupt queues in guest

    :param test: QEMU test object
    :param params: Dictionary with the test parameters
    :param env: Dictionary with test environment.
    """
    def proc_interrupts_results(results):
        results_dict = {}
        cpu_count = 0
        cpu_list = []
        for line in results.splitlines():
            line = line.strip()
            if re.match("CPU0", line):
                cpu_list = re.findall("CPU\d+", line)
                cpu_count = len(cpu_list)
                continue
            if cpu_count > 0:
                irq_key = re.split(":", line)[0]
                results_dict[irq_key] = {}
                content = line[len(irq_key) + 1:].strip()
                if len(re.split("\s+", content)) < cpu_count:
                    continue
                count = 0
                irq_des = ""
                for irq_item in re.split("\s+", content):
                    if count < cpu_count:
                        if count == 0:
                            results_dict[irq_key]["count"] = []
                        results_dict[irq_key]["count"].append(irq_item)
                    else:
                        irq_des += " %s" % irq_item
                    count += 1
                results_dict[irq_key]["irq_des"] = irq_des.strip()
        return results_dict, cpu_list

    timeout = float(params.get("login_timeout", 240))
    host_cpu_num = local_host.LocalHost().get_num_cpu()
    while host_cpu_num:
        num_queues = str(host_cpu_num)
        host_cpu_num &= host_cpu_num - 1
    params['smp'] = num_queues
    params['num_queues'] = num_queues
    images_num = int(num_queues)
    extra_image_size = params.get("image_size_extra_images", "512M")
    system_image = params.get("images")
    system_image_drive_format = params.get("system_image_drive_format", "ide")
    params["drive_format_%s" % system_image] = system_image_drive_format

    error.context("Boot up guest with block devcie with num_queues"
                  " is %s and smp is %s" % (num_queues, params['smp']),
                  logging.info)
    for vm in env.get_all_vms():
        if vm.is_alive():
            vm.destroy()
    for extra_image in range(images_num):
        image_tag = "stg%s" % extra_image
        params["images"] += " %s" % image_tag
        params["image_name_%s" % image_tag] = "images/%s" % image_tag
        params["image_size_%s" % image_tag] = extra_image_size
        params["force_create_image_%s" % image_tag] = "yes"
        image_params = params.object_params(image_tag)
        env_process.preprocess_image(test, image_params, image_tag)

    params["start_vm"] = "yes"
    vm = env.get_vm(params["main_vm"])
    env_process.preprocess_vm(test, params, env, vm.name)
    session = vm.wait_for_login(timeout=timeout)

    error.context("Check irqbalance service status", logging.info)
    output = session.cmd_output("systemctl status irqbalance")
    if not re.findall("Active: active", output):
        session.cmd("systemctl start irqbalance")
        output = session.cmd_output("systemctl status irqbalance")
        output = utils_misc.strip_console_codes(output)
        if not re.findall("Active: active", output):
            raise error.TestNAError("Can not start irqbalance inside guest. "
                                    "Skip this test.")

    error.context("Pin vcpus to host cpus", logging.info)
    host_numa_nodes = utils_misc.NumaInfo()
    vcpu_num = 0
    for numa_node_id in host_numa_nodes.nodes:
        numa_node = host_numa_nodes.nodes[numa_node_id]
        for _ in range(len(numa_node.cpus)):
            if vcpu_num >= len(vm.vcpu_threads):
                break
            vcpu_tid = vm.vcpu_threads[vcpu_num]
            logging.debug("pin vcpu thread(%s) to cpu"
                          "(%s)" % (vcpu_tid,
                                    numa_node.pin_cpu(vcpu_tid)))
            vcpu_num += 1

    error.context("Verify num_queues from monitor", logging.info)
    qtree = qemu_qtree.QtreeContainer()
    try:
        qtree.parse_info_qtree(vm.monitor.info('qtree'))
    except AttributeError:
        raise error.TestNAError("Monitor deson't supoort qtree "
                                "skip this test")
    error_msg = "Number of queues mismatch: expect %s"
    error_msg += " report from monitor: %s(%s)"
    scsi_bus_addr = ""
    qtree_num_queues_full = ""
    qtree_num_queues = ""
    for node in qtree.get_nodes():
        if isinstance(node, qemu_qtree.QtreeDev) and (
                node.qtree['type'] == "virtio-scsi-device"):
            qtree_num_queues_full = node.qtree["num_queues"]
            qtree_num_queues = re.search(
                "[0-9]+",
                qtree_num_queues_full).group()
        elif isinstance(node, qemu_qtree.QtreeDev) and node.qtree['type'] == "virtio-scsi-pci":
            scsi_bus_addr = node.qtree['addr']

    if qtree_num_queues != num_queues:
        error_msg = error_msg % (num_queues,
                                 qtree_num_queues,
                                 qtree_num_queues_full)
        raise error.TestFail(error_msg)
    if not scsi_bus_addr:
        raise error.TestError("Didn't find addr from qtree. Please check "
                              "the log.")
    error.context("Check device init status in guest", logging.info)
    init_check_cmd = params.get("init_check_cmd", "dmesg | grep irq")
    output = session.cmd_output(init_check_cmd)
    irqs_pattern = params.get("irqs_pattern", "%s:\s+irq\s+(\d+)")
    irqs_pattern = irqs_pattern % scsi_bus_addr
    irqs_watch = re.findall(irqs_pattern, output)
    # As there are several interrupts count for virtio device:
    # config, control, event and request. And the each queue have
    # a request count. So the totally count for virtio device should
    # equal to queus number plus three.
    if len(irqs_watch) != 3 + int(num_queues):
        raise error.TestFail("Failed to check the interrupt ids from dmesg")
    irq_check_cmd = params.get("irq_check_cmd", "cat /proc/interrupts")
    output = session.cmd_output(irq_check_cmd)
    irq_results, _ = proc_interrupts_results(output)
    for irq_watch in irqs_watch:
        if irq_watch not in irq_results:
            raise error.TestFail("Can't find irq %s from procfs" % irq_watch)

    error.context("Load I/O in all targets", logging.info)
    get_dev_cmd = params.get("get_dev_cmd", "ls /dev/[svh]d*")
    output = session.cmd_output(get_dev_cmd)
    system_dev = re.findall("[svh]d(\w+)\d+", output)[0]
    dd_timeout = int(re.findall("\d+", extra_image_size)[0])
    fill_cmd = ""
    count = 0
    for dev in re.split("\s+", output):
        if not dev:
            continue
        if not re.findall("[svh]d%s" % system_dev, dev):
            fill_cmd += " dd of=%s if=/dev/urandom bs=1M " % dev
            fill_cmd += "count=%s &" % dd_timeout
            count += 1
    if count != images_num:
        raise error.TestError("Disks are not all show up in system. Output "
                              "from the check command: %s" % output)
    # As Bug 1177332 exists, mq is not supported completely.
    # So don't considering performance currently, dd_timeout is longer.
    dd_timeout = dd_timeout * images_num * 2
    session.cmd(fill_cmd, timeout=dd_timeout)

    dd_thread_num = count
    while dd_thread_num:
        time.sleep(5)
        dd_thread_num = session.cmd_output("pgrep -x dd", timeout=dd_timeout)

    error.context("Check the interrupt queues in guest", logging.info)
    output = session.cmd_output(irq_check_cmd)
    irq_results, cpu_list = proc_interrupts_results(output)
    irq_bit_map = 0
    for irq_watch in irqs_watch:
        if "request" in irq_results[irq_watch]["irq_des"]:
            for index, count in enumerate(irq_results[irq_watch]["count"]):
                if int(count) > 0:
                    irq_bit_map |= 2 ** index

    error_msg = ""
    cpu_not_used = []
    for index, cpu in enumerate(cpu_list):
        if 2 ** index & irq_bit_map != 2 ** index:
            cpu_not_used.append(cpu)

    if cpu_not_used:
        logging.debug("Interrupt info from procfs:\n%s" % output)
        error_msg = " ".join(cpu_not_used)
        if len(cpu_not_used) > 1:
            error_msg += " are"
        else:
            error_msg += " is"
        error_msg += " not used during test. Please check debug log for"
        error_msg += " more information."
        raise error.TestFail(error_msg)

Example 49

Project: pychess
Source File: pgn.py
View license
    def loadToModel(self, gameno, position=-1, model=None):
        if not model:
            model = GameModel()

        # the seven mandatory PGN headers
        model.tags['Event'] = self._getTag(gameno, 'Event')
        model.tags['Site'] = self._getTag(gameno, 'Site')
        model.tags['Date'] = self._getTag(gameno, 'Date')
        model.tags['Round'] = self.get_round(gameno)
        model.tags['White'], model.tags['Black'] = self.get_player_names(
            gameno)
        model.tags['Result'] = reprResult[self.get_result(gameno)]

        if model.tags['Date']:
            date_match = re.match(".*(\d{4}).(\d{2}).(\d{2}).*",
                                  model.tags['Date'])
            if date_match:
                year, month, day = date_match.groups()
                model.tags['Year'] = year
                model.tags['Month'] = month
                model.tags['Day'] = day

                # non-mandatory headers
        for tag in ('Annotator', 'ECO', 'WhiteElo', 'BlackElo', 'TimeControl'):
            if self._getTag(gameno, tag):
                model.tags[tag] = self._getTag(gameno, tag)
            else:
                model.tags[tag] = ""

        model.info = self.get_info(gameno)

        # TODO: enable this when NewGameDialog is altered to give user option of
        # whether to use PGN's clock time, or their own custom time. Also,
        # dialog should set+insensitize variant based on the variant of the
        # game selected in the dialog
        if model.tags['TimeControl']:
            secs, gain = parseTimeControlTag(model.tags['TimeControl'])
            model.timed = True
            model.timemodel.secs = secs
            model.timemodel.gain = gain
            model.timemodel.minutes = secs / 60
            for tag, color in (('WhiteClock', WHITE), ('BlackClock', BLACK)):
                if self._getTag(gameno, tag):
                    try:
                        millisec = parseClockTimeTag(self._getTag(gameno, tag))
                        # We need to fix when FICS reports negative clock time like this
                        # [TimeControl "180+0"]
                        # [WhiteClock "0:00:15.867"]
                        # [BlackClock "23:59:58.820"]
                        start_sec = (
                            millisec - 24 * 60 * 60 * 1000
                        ) / 1000. if millisec > 23 * 60 * 60 * 1000 else millisec / 1000.
                        model.timemodel.intervals[color][0] = start_sec
                    except ValueError:
                        raise LoadingError(
                            "Error parsing '%s' Header for gameno %s" % (tag, gameno))

        fenstr = self._getTag(gameno, "FEN")
        variant = self.get_variant(gameno)

        if variant:
            if variant not in name2variant:
                raise LoadingError("Unknown variant %s for gameno %s" % (variant, gameno))

            model.tags["Variant"] = variant
            # Fixes for some non statndard Chess960 .pgn
            if (fenstr is not None) and variant == "Fischerandom":
                parts = fenstr.split()
                parts[0] = parts[0].replace(".", "/").replace("0", "")
                if len(parts) == 1:
                    parts.append("w")
                    parts.append("-")
                    parts.append("-")
                fenstr = " ".join(parts)

            model.variant = name2variant[variant]
            board = LBoard(model.variant.variant)
        else:
            model.variant = NormalBoard
            board = LBoard()

        if fenstr:
            try:
                board.applyFen(fenstr)
            except SyntaxError as err:
                board.applyFen(FEN_EMPTY)
                raise LoadingError(
                    _("The game can't be loaded, because of an error parsing FEN"),
                    err.args[0])
        else:
            board.applyFen(FEN_START)

        boards = [board]

        del model.moves[:]
        del model.variations[:]

        self.error = None
        movetext = self.get_movetext(gameno)

        boards = self.parse_movetext(movetext, boards[0], position)

        # The parser built a tree of lboard objects, now we have to
        # create the high level Board and Move lists...

        for board in boards:
            if board.lastMove is not None:
                model.moves.append(Move(board.lastMove))

        self.has_emt = False
        self.has_eval = False

        def walk(model, node, path):
            if node.prev is None:
                # initial game board
                board = model.variant(setup=node.asFen(), lboard=node)
            else:
                move = Move(node.lastMove)
                try:
                    board = node.prev.pieceBoard.move(move, lboard=node)
                except:
                    raise LoadingError(
                        _("Invalid move."),
                        "%s%s" % (move_count(node, black_periods=True), move))

            if node.next is None:
                model.variations.append(path + [board])
            else:
                walk(model, node.next, path + [board])

            for child in node.children:
                if isinstance(child, list):
                    if len(child) > 1:
                        # non empty variation, go walk
                        walk(model, child[1], list(path))
                else:
                    if not self.has_emt:
                        self.has_emt = child.find("%emt") >= 0
                    if not self.has_eval:
                        self.has_eval = child.find("%eval") >= 0

        # Collect all variation paths into a list of board lists
        # where the first one will be the boards of mainline game.
        # model.boards will allways point to the current shown variation
        # which will be model.variations[0] when we are in the mainline.
        walk(model, boards[0], [])
        model.boards = model.variations[0]
        self.has_emt = self.has_emt and "TimeControl" in model.tags
        if self.has_emt or self.has_eval:
            if self.has_emt:
                blacks = len(model.moves) // 2
                whites = len(model.moves) - blacks

                model.timemodel.intervals = [
                    [model.timemodel.intervals[0][0]] * (whites + 1),
                    [model.timemodel.intervals[1][0]] * (blacks + 1),
                ]
                secs, gain = parseTimeControlTag(model.tags['TimeControl'])
                model.timemodel.intervals[0][0] = secs
                model.timemodel.intervals[1][0] = secs
            for ply, board in enumerate(boards):
                for child in board.children:
                    if isinstance(child, basestring):
                        if self.has_emt:
                            match = movetime.search(child)
                            if match:
                                movecount, color = divmod(ply + 1, 2)
                                hour, minute, sec, msec = match.groups()
                                prev = model.timemodel.intervals[color][
                                    movecount - 1]
                                hour = 0 if hour is None else int(hour[:-1])
                                minute = 0 if minute is None else int(minute[:-1])
                                msec = 0 if msec is None else int(msec)
                                msec += int(sec) * 1000 + int(
                                    minute) * 60 * 1000 + int(
                                        hour) * 60 * 60 * 1000
                                model.timemodel.intervals[color][
                                    movecount] = prev - msec / 1000. + gain

                        if self.has_eval:
                            match = moveeval.search(child)
                            if match:
                                sign, num, fraction, depth = match.groups()
                                sign = 1 if sign is None or sign == "+" else -1
                                num = int(num) if int(
                                    num) == MATE_VALUE else int(num)
                                fraction = 0 if fraction is None else int(
                                    fraction)
                                value = sign * (num * 100 + fraction)
                                depth = "" if depth is None else depth
                                if board.color == BLACK:
                                    value = -value
                                model.scores[ply] = ("", value, depth)
            log.debug("pgn.loadToModel: intervals %s" %
                      model.timemodel.intervals)
        # Find the physical status of the game
        model.status, model.reason = getStatus(model.boards[-1])

        # Apply result from .pgn if the last position was loaded
        if position == -1 or len(model.moves) == position - model.lowply:
            status = self.get_result(gameno)
            if status in (WHITEWON, BLACKWON) and status != model.status:
                model.status = status
                model.reason = WON_RESIGN
            elif status == DRAW and status != model.status:
                model.status = DRAW
                model.reason = DRAW_AGREE

        # If parsing gave an error we throw it now, to enlarge our possibility
        # of being able to continue the game from where it failed.
        if self.error:
            raise self.error

        return model

Example 50

Project: PyClassLessons
Source File: wheel.py
View license
def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None,
                     pycompile=True, scheme=None):
    """Install a wheel"""

    if not scheme:
        scheme = distutils_scheme(name, user=user, home=home, root=root)

    if root_is_purelib(name, wheeldir):
        lib_dir = scheme['purelib']
    else:
        lib_dir = scheme['platlib']

    info_dir = []
    data_dirs = []
    source = wheeldir.rstrip(os.path.sep) + os.path.sep

    # Record details of the files moved
    #   installed = files copied from the wheel to the destination
    #   changed = files changed while installing (scripts #! line typically)
    #   generated = files newly generated during the install (script wrappers)
    installed = {}
    changed = set()
    generated = []

    # Compile all of the pyc files that we're going to be installing
    if pycompile:
        compileall.compile_dir(source, force=True, quiet=True)

    def normpath(src, p):
        return make_path_relative(src, p).replace(os.path.sep, '/')

    def record_installed(srcfile, destfile, modified=False):
        """Map archive RECORD paths to installation RECORD paths."""
        oldpath = normpath(srcfile, wheeldir)
        newpath = normpath(destfile, lib_dir)
        installed[oldpath] = newpath
        if modified:
            changed.add(destfile)

    def clobber(source, dest, is_base, fixer=None, filter=None):
        if not os.path.exists(dest): # common for the 'include' path
            os.makedirs(dest)

        for dir, subdirs, files in os.walk(source):
            basedir = dir[len(source):].lstrip(os.path.sep)
            destdir = os.path.join(dest, basedir)
            if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
                continue
            for s in subdirs:
                destsubdir = os.path.join(dest, basedir, s)
                if is_base and basedir == '' and destsubdir.endswith('.data'):
                    data_dirs.append(s)
                    continue
                elif (is_base
                    and s.endswith('.dist-info')
                    # is self.req.project_name case preserving?
                    and s.lower().startswith(req.project_name.replace('-', '_').lower())):
                    assert not info_dir, 'Multiple .dist-info directories'
                    info_dir.append(destsubdir)
            for f in files:
                # Skip unwanted files
                if filter and filter(f):
                    continue
                srcfile = os.path.join(dir, f)
                destfile = os.path.join(dest, basedir, f)
                # directory creation is lazy and after the file filtering above
                # to ensure we don't install empty dirs; empty dirs can't be
                # uninstalled.
                if not os.path.exists(destdir):
                    os.makedirs(destdir)
                # use copy2 (not move) to be extra sure we're not moving
                # directories over; copy2 fails for directories.  this would
                # fail tests (not during released/user execution)
                shutil.copy2(srcfile, destfile)
                changed = False
                if fixer:
                    changed = fixer(destfile)
                record_installed(srcfile, destfile, changed)

    clobber(source, lib_dir, True)

    assert info_dir, "%s .dist-info directory not found" % req

    # Get the defined entry points
    ep_file = os.path.join(info_dir[0], 'entry_points.txt')
    console, gui = get_entrypoints(ep_file)

    def is_entrypoint_wrapper(name):
        # EP, EP.exe and EP-script.py are scripts generated for
        # entry point EP by setuptools
        if name.lower().endswith('.exe'):
            matchname = name[:-4]
        elif name.lower().endswith('-script.py'):
            matchname = name[:-10]
        elif name.lower().endswith(".pya"):
            matchname = name[:-4]
        else:
            matchname = name
        # Ignore setuptools-generated scripts
        return (matchname in console or matchname in gui)

    for datadir in data_dirs:
        fixer = None
        filter = None
        for subdir in os.listdir(os.path.join(wheeldir, datadir)):
            fixer = None
            if subdir == 'scripts':
                fixer = fix_script
                filter = is_entrypoint_wrapper
            source = os.path.join(wheeldir, datadir, subdir)
            dest = scheme[subdir]
            clobber(source, dest, False, fixer=fixer, filter=filter)

    maker = ScriptMaker(None, scheme['scripts'])

    # Ensure we don't generate any variants for scripts because this is almost
    # never what somebody wants.
    # See https://bitbucket.org/pypa/distlib/issue/35/
    maker.variants = set(('', ))

    # This is required because otherwise distlib creates scripts that are not
    # executable.
    # See https://bitbucket.org/pypa/distlib/issue/32/
    maker.set_mode = True

    # Simplify the script and fix the fact that the default script swallows
    # every single stack trace.
    # See https://bitbucket.org/pypa/distlib/issue/34/
    # See https://bitbucket.org/pypa/distlib/issue/33/
    def _get_script_text(entry):
        return maker.script_template % {
            "module": entry.prefix,
            "import_name": entry.suffix.split(".")[0],
            "func": entry.suffix,
        }

    maker._get_script_text = _get_script_text
    maker.script_template = """# -*- coding: utf-8 -*-
import re
import sys

from %(module)s import %(import_name)s

if __name__ == '__main__':
    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
    sys.exit(%(func)s())
"""

    # Special case pip and setuptools to generate versioned wrappers
    #
    # The issue is that some projects (specifically, pip and setuptools) use
    # code in setup.py to create "versioned" entry points - pip2.7 on Python
    # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into
    # the wheel metadata at build time, and so if the wheel is installed with
    # a *different* version of Python the entry points will be wrong. The
    # correct fix for this is to enhance the metadata to be able to describe
    # such versioned entry points, but that won't happen till Metadata 2.0 is
    # available.
    # In the meantime, projects using versioned entry points will either have
    # incorrect versioned entry points, or they will not be able to distribute
    # "universal" wheels (i.e., they will need a wheel per Python version).
    #
    # Because setuptools and pip are bundled with _ensurepip and virtualenv,
    # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we
    # override the versioned entry points in the wheel and generate the
    # correct ones. This code is purely a short-term measure until Metadat 2.0
    # is available.
    #
    # To add the level of hack in this section of code, in order to support
    # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment
    # variable which will control which version scripts get installed.
    #
    # ENSUREPIP_OPTIONS=altinstall
    #   - Only pipX.Y and easy_install-X.Y will be generated and installed
    # ENSUREPIP_OPTIONS=install
    #   - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note
    #     that this option is technically if ENSUREPIP_OPTIONS is set and is
    #     not altinstall
    # DEFAULT
    #   - The default behavior is to install pip, pipX, pipX.Y, easy_install
    #     and easy_install-X.Y.
    pip_script = console.pop('pip', None)
    if pip_script:
        if "ENSUREPIP_OPTIONS" not in os.environ:
            spec = 'pip = ' + pip_script
            generated.extend(maker.make(spec))

        if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
            spec = 'pip%s = %s' % (sys.version[:1], pip_script)
            generated.extend(maker.make(spec))

        spec = 'pip%s = %s' % (sys.version[:3], pip_script)
        generated.extend(maker.make(spec))
        # Delete any other versioned pip entry points
        pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)]
        for k in pip_ep:
            del console[k]
    easy_install_script = console.pop('easy_install', None)
    if easy_install_script:
        if "ENSUREPIP_OPTIONS" not in os.environ:
            spec = 'easy_install = ' + easy_install_script
            generated.extend(maker.make(spec))

        spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script)
        generated.extend(maker.make(spec))
        # Delete any other versioned easy_install entry points
        easy_install_ep = [k for k in console
                if re.match(r'easy_install(-\d\.\d)?$', k)]
        for k in easy_install_ep:
            del console[k]

    # Generate the console and GUI entry points specified in the wheel
    if len(console) > 0:
        generated.extend(maker.make_multiple(['%s = %s' % kv for kv in console.items()]))
    if len(gui) > 0:
        generated.extend(maker.make_multiple(['%s = %s' % kv for kv in gui.items()], {'gui': True}))

    record = os.path.join(info_dir[0], 'RECORD')
    temp_record = os.path.join(info_dir[0], 'RECORD.pip')
    with open_for_csv(record, 'r') as record_in:
        with open_for_csv(temp_record, 'w+') as record_out:
            reader = csv.reader(record_in)
            writer = csv.writer(record_out)
            for row in reader:
                row[0] = installed.pop(row[0], row[0])
                if row[0] in changed:
                    row[1], row[2] = rehash(row[0])
                writer.writerow(row)
            for f in generated:
                h, l = rehash(f)
                writer.writerow((f, h, l))
            for f in installed:
                writer.writerow((installed[f], '', ''))
    shutil.move(temp_record, record)