re.sub

Here are the examples of the python api re.sub taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 151

Project: papercut
Source File: phpbb_mysql.py
View license
    def do_POST(self, group_name, lines, ip_address, username=''):
        forum_id = self.get_forum(group_name)
        prefix = settings.phpbb_table_prefix
        # patch by Andreas Wegmann <[email protected]> to fix the handling of unusual encodings of messages
        lines = mime_decode_header(re.sub(q_quote_multiline, "=?\\1?Q?\\2\\3?=", lines))
        body = self.get_message_body(lines)
        author, email = from_regexp.search(lines, 0).groups()
        subject = subject_regexp.search(lines, 0).groups()[0].strip()
        # get the authentication information now
        if username != '':
            stmt = """
                    SELECT
                        user_id
                    FROM
                        %susers
                    WHERE
                        username='%s'""" % (prefix, username)
            num_rows = self.cursor.execute(stmt)
            if num_rows == 0:
                poster_id = -1
            else:
                poster_id = self.cursor.fetchone()[0]
            post_username = ''
        else:
            poster_id = -1
            post_username = author
        if lines.find('References') != -1:
            # get the 'modifystamp' value from the parent (if any)
            references = references_regexp.search(lines, 0).groups()
            parent_id, void = references[-1].strip().split('@')
            stmt = """
                    SELECT
                        topic_id
                    FROM
                        %sposts
                    WHERE
                        post_id=%s
                    GROUP BY
                        post_id""" % (prefix, parent_id)
            num_rows = self.cursor.execute(stmt)
            if num_rows == 0:
                return None
            thread_id = self.cursor.fetchone()[0]
        else:
            # create a new topic
            stmt = """
                    INSERT INTO
                        %stopics
                    (
                        forum_id,
                        topic_title,
                        topic_poster,
                        topic_time,
                        topic_status,
                        topic_vote,
                        topic_type
                    ) VALUES (
                        %s,
                        '%s',
                        %s,
                        UNIX_TIMESTAMP(),
                        0,
                        0,
                        0
                    )""" % (prefix, forum_id, self.quote_string(subject), poster_id)
            self.cursor.execute(stmt)
            thread_id = self.cursor.insert_id()
        stmt = """
                INSERT INTO
                    %sposts
                (
                    topic_id,
                    forum_id,
                    poster_id,
                    post_time,
                    poster_ip,
                    post_username,
                    enable_bbcode,
                    enable_html,
                    enable_smilies,
                    enable_sig
                ) VALUES (
                    %s,
                    %s,
                    %s,
                    UNIX_TIMESTAMP(),
                    '%s',
                    '%s',
                    1,
                    0,
                    1,
                    0
                )""" % (prefix, thread_id, forum_id, poster_id, self.encode_ip(ip_address), post_username)
        self.cursor.execute(stmt)
        new_id = self.cursor.insert_id()
        if not new_id:
            return None
        else:
            # insert into the '*posts_text' table
            stmt = """
                    INSERT INTO
                        %sposts_text
                    (
                        post_id,
                        bbcode_uid,
                        post_subject,
                        post_text
                    ) VALUES (
                        %s,
                        '%s',
                        '%s',
                        '%s'
                    )""" % (prefix, new_id, self.make_bbcode_uid(), self.quote_string(subject), self.quote_string(body))
            if not self.cursor.execute(stmt):
                # delete from 'topics' and 'posts' tables before returning...
                stmt = """
                        DELETE FROM
                            %stopics
                        WHERE
                            topic_id=%s""" % (prefix, thread_id)
                self.cursor.execute(stmt)
                stmt = """
                        DELETE FROM
                            %sposts
                        WHERE
                            post_id=%s""" % (prefix, new_id)
                self.cursor.execute(stmt)
                return None
            else:
                if lines.find('References') != -1:
                    # update the total number of posts in the forum
                    stmt = """
                            UPDATE
                                %sforums
                            SET
                                forum_posts=forum_posts+1,
                                forum_last_post_id=%s
                            WHERE
                                forum_id=%s
                            """ % (settings.phpbb_table_prefix, new_id, forum_id)
                    self.cursor.execute(stmt)
                else:
                    # update the total number of topics and posts in the forum
                    stmt = """
                            UPDATE
                                %sforums
                            SET
                                forum_topics=forum_topics+1,
                                forum_posts=forum_posts+1,
                                forum_last_post_id=%s
                            WHERE
                                forum_id=%s
                            """ % (settings.phpbb_table_prefix, new_id, forum_id)
                    self.cursor.execute(stmt)
                # update the user's post count, if this is indeed a real user
                if poster_id != -1:
                    stmt = """
                            UPDATE
                                %susers
                            SET
                                user_posts=user_posts+1
                            WHERE
                                user_id=%s""" % (prefix, poster_id)
                    self.cursor.execute(stmt)
                # setup last post on the topic thread (Patricio Anguita <[email protected]>)
                stmt = """
                        UPDATE
                            %stopics
                        SET
                            topic_replies=topic_replies+1,
                            topic_last_post_id=%s
                        WHERE
                            topic_id=%s""" % (prefix, new_id, thread_id)
                self.cursor.execute(stmt)
                # if this is the first post on the thread.. (Patricio Anguita <[email protected]>)
                if lines.find('References') == -1:
                    stmt = """
                            UPDATE
                                %stopics
                            SET
                                topic_first_post_id=%s
                            WHERE
                                topic_id=%s AND
                                topic_first_post_id=0""" % (prefix, new_id, thread_id)
                    self.cursor.execute(stmt)
                return 1

Example 152

Project: papercut
Source File: phpnuke_phpbb_mysql.py
View license
    def do_POST(self, group_name, lines, ip_address, username=''):
        forum_id = self.get_forum(group_name)
        prefix = settings.phpbb_table_prefix
        nuke_prefix = settings.nuke_table_prefix
        # patch by Andreas Wegmann <[email protected]> to fix the handling of unusual encodings of messages
        lines = mime_decode_header(re.sub(q_quote_multiline, "=?\\1?Q?\\2\\3?=", lines))
        body = self.get_message_body(lines)
        author, email = from_regexp.search(lines, 0).groups()
        subject = subject_regexp.search(lines, 0).groups()[0].strip()
        # get the authentication information now
        if username != '':
            stmt = """
                    SELECT
                        user_id
                    FROM
                        %susers
                    WHERE
                        username='%s'""" % (nuke_prefix, username)
            num_rows = self.cursor.execute(stmt)
            if num_rows == 0:
                poster_id = -1
            else:
                poster_id = self.cursor.fetchone()[0]
            post_username = ''
        else:
            poster_id = -1
            post_username = author
        if lines.find('References') != -1:
            # get the 'modifystamp' value from the parent (if any)
            references = references_regexp.search(lines, 0).groups()
            parent_id, void = references[-1].strip().split('@')
            stmt = """
                    SELECT
                        topic_id
                    FROM
                        %sposts
                    WHERE
                        post_id=%s
                    GROUP BY
                        post_id""" % (prefix, parent_id)
            num_rows = self.cursor.execute(stmt)
            if num_rows == 0:
                return None
            thread_id = self.cursor.fetchone()[0]
        else:
            # create a new topic
            stmt = """
                    INSERT INTO
                        %stopics
                    (
                        forum_id,
                        topic_title,
                        topic_poster,
                        topic_time,
                        topic_status,
                        topic_vote,
                        topic_type
                    ) VALUES (
                        %s,
                        '%s',
                        %s,
                        UNIX_TIMESTAMP(),
                        0,
                        0,
                        0
                    )""" % (prefix, forum_id, self.quote_string(subject), poster_id)
            self.cursor.execute(stmt)
            thread_id = self.cursor.insert_id()
        stmt = """
                INSERT INTO
                    %sposts
                (
                    topic_id,
                    forum_id,
                    poster_id,
                    post_time,
                    poster_ip,
                    post_username,
                    enable_bbcode,
                    enable_html,
                    enable_smilies,
                    enable_sig
                ) VALUES (
                    %s,
                    %s,
                    %s,
                    UNIX_TIMESTAMP(),
                    '%s',
                    '%s',
                    1,
                    0,
                    1,
                    0
                )""" % (prefix, thread_id, forum_id, poster_id, self.encode_ip(ip_address), post_username)
        self.cursor.execute(stmt)
        new_id = self.cursor.insert_id()
        if not new_id:
            return None
        else:
            # insert into the '*posts_text' table
            stmt = """
                    INSERT INTO
                        %sposts_text
                    (
                        post_id,
                        bbcode_uid,
                        post_subject,
                        post_text
                    ) VALUES (
                        %s,
                        '%s',
                        '%s',
                        '%s'
                    )""" % (prefix, new_id, self.make_bbcode_uid(), self.quote_string(subject), self.quote_string(body))
            if not self.cursor.execute(stmt):
                # delete from 'topics' and 'posts' tables before returning...
                stmt = """
                        DELETE FROM
                            %stopics
                        WHERE
                            topic_id=%s""" % (prefix, thread_id)
                self.cursor.execute(stmt)
                stmt = """
                        DELETE FROM
                            %sposts
                        WHERE
                            post_id=%s""" % (prefix, new_id)
                self.cursor.execute(stmt)
                return None
            else:
                if lines.find('References') != -1:
                    # update the total number of posts in the forum
                    stmt = """
                            UPDATE
                                %sforums
                            SET
                                forum_posts=forum_posts+1,
                                forum_last_post_id=%s
                            WHERE
                                forum_id=%s
                            """ % (settings.phpbb_table_prefix, new_id, forum_id)
                    self.cursor.execute(stmt)
                else:
                    # update the total number of topics and posts in the forum
                    stmt = """
                            UPDATE
                                %sforums
                            SET
                                forum_topics=forum_topics+1,
                                forum_posts=forum_posts+1,
                                forum_last_post_id=%s
                            WHERE
                                forum_id=%s
                            """ % (settings.phpbb_table_prefix, new_id, forum_id)
                    self.cursor.execute(stmt)
                # update the user's post count, if this is indeed a real user
                if poster_id != -1:
                    stmt = """
                            UPDATE
                                %susers
                            SET
                                user_posts=user_posts+1
                            WHERE
                                user_id=%s""" % (nuke_prefix, poster_id)
                    self.cursor.execute(stmt)
                # setup last post on the topic thread (Patricio Anguita <[email protected]>)
                stmt = """
                        UPDATE
                            %stopics
                        SET
                            topic_replies=topic_replies+1,
                            topic_last_post_id=%s
                        WHERE
                            topic_id=%s""" % (prefix, new_id, thread_id)
                self.cursor.execute(stmt)
                # if this is the first post on the thread.. (Patricio Anguita <[email protected]>)
                if lines.find('References') == -1:
                    stmt = """
                            UPDATE
                                %stopics
                            SET
                                topic_first_post_id=%s
                            WHERE
                                topic_id=%s AND
                                topic_first_post_id=0""" % (prefix, new_id, thread_id)
                    self.cursor.execute(stmt)
                return 1

Example 153

Project: ANALYSE
Source File: formula.py
View license
    def preprocess_pmathml(self, xml):
        r"""
        Pre-process presentation MathML from ASCIIMathML to make it more
        acceptable for SnuggleTeX, and also to accomodate some sympy
        conventions (eg hat(i) for \hat{i}).

        This method would be a good spot to look for an integral and convert
        it, if possible...
        """

        if type(xml) == str or type(xml) == unicode:
            xml = etree.fromstring(xml)		# TODO: wrap in try

        xml = self.fix_greek_in_mathml(xml)	 # convert greek utf letters to greek spelled out in ascii

        def gettag(expr):
            return re.sub('{http://[^}]+}', '', expr.tag)

        def fix_pmathml(xml):
            """
            f and g are processed as functions by asciimathml, eg "f-2" turns
            into "<mrow><mi>f</mi><mo>-</mo></mrow><mn>2</mn>" this is
            really terrible for turning into cmathml.  undo this here.
            """
            for k in xml:
                tag = gettag(k)
                if tag == 'mrow':
                    if len(k) == 2:
                        if gettag(k[0]) == 'mi' and k[0].text in ['f', 'g'] and gettag(k[1]) == 'mo':
                            idx = xml.index(k)
                            xml.insert(idx, deepcopy(k[0]))	 # drop the <mrow> container
                            xml.insert(idx + 1, deepcopy(k[1]))
                            xml.remove(k)
                fix_pmathml(k)

        fix_pmathml(xml)

        def fix_hat(xml):
            """
            hat i is turned into <mover><mi>i</mi><mo>^</mo></mover> ; mangle
            this into <mi>hat(f)</mi> hat i also somtimes turned into
            <mover><mrow> <mi>j</mi> </mrow><mo>^</mo></mover>
            """
            for k in xml:
                tag = gettag(k)
                if tag == 'mover':
                    if len(k) == 2:
                        if gettag(k[0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
                            newk = etree.Element('mi')
                            newk.text = 'hat(%s)' % k[0].text
                            xml.replace(k, newk)
                        if gettag(k[0]) == 'mrow' and gettag(k[0][0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
                            newk = etree.Element('mi')
                            newk.text = 'hat(%s)' % k[0][0].text
                            xml.replace(k, newk)
                fix_hat(k)
        fix_hat(xml)

        def flatten_pmathml(xml):
            """
            Give the text version of certain PMathML elements

            Sometimes MathML will be given with each letter separated (it
            doesn't know if its implicit multiplication or what). From an xml
            node, find the (text only) variable name it represents. So it takes
            <mrow>
              <mi>m</mi>
              <mi>a</mi>
              <mi>x</mi>
            </mrow>
            and returns 'max', for easier use later on.
            """
            tag = gettag(xml)
            if tag == 'mn':
                return xml.text
            elif tag == 'mi':
                return xml.text
            elif tag == 'mrow':
                return ''.join([flatten_pmathml(y) for y in xml])
            raise Exception('[flatten_pmathml] unknown tag %s' % tag)

        def fix_mathvariant(parent):
            """
            Fix certain kinds of math variants

            Literally replace <mstyle mathvariant="script"><mi>N</mi></mstyle>
            with 'scriptN'. There have been problems using script_N or script(N)
            """
            for child in parent:
                if (gettag(child) == 'mstyle' and child.get('mathvariant') == 'script'):
                    newchild = etree.Element('mi')
                    newchild.text = 'script%s' % flatten_pmathml(child[0])
                    parent.replace(child, newchild)
                fix_mathvariant(child)
        fix_mathvariant(xml)

        # find "tagged" superscripts
        # they have the character \u200b in the superscript
        # replace them with a__b so snuggle doesn't get confused
        def fix_superscripts(xml):
            """ Look for and replace sup elements with 'X__Y' or 'X_Y__Z'

            In the javascript, variables with '__X' in them had an invisible
            character inserted into the sup (to distinguish from powers)
            E.g. normal:
            <msubsup>
              <mi>a</mi>
              <mi>b</mi>
              <mi>c</mi>
            </msubsup>
            to be interpreted '(a_b)^c' (nothing done by this method)

            And modified:
            <msubsup>
              <mi>b</mi>
              <mi>x</mi>
              <mrow>
                <mo>&#x200B;</mo>
                <mi>d</mi>
              </mrow>
            </msubsup>
            to be interpreted 'a_b__c'

            also:
            <msup>
              <mi>x</mi>
              <mrow>
                <mo>&#x200B;</mo>
                <mi>B</mi>
              </mrow>
            </msup>
            to be 'x__B'
            """
            for k in xml:
                tag = gettag(k)

                # match things like the last example--
                # the second item in msub is an mrow with the first
                # character equal to \u200b
                if (
                        tag == 'msup' and
                        len(k) == 2 and gettag(k[1]) == 'mrow' and
                        gettag(k[1][0]) == 'mo' and k[1][0].text == u'\u200b'  # whew
                ):

                    # replace the msup with 'X__Y'
                    k[1].remove(k[1][0])
                    newk = etree.Element('mi')
                    newk.text = '%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]))
                    xml.replace(k, newk)

                # match things like the middle example-
                # the third item in msubsup is an mrow with the first
                # character equal to \u200b
                if (
                        tag == 'msubsup' and
                        len(k) == 3 and gettag(k[2]) == 'mrow' and
                        gettag(k[2][0]) == 'mo' and k[2][0].text == u'\u200b'    # whew
                ):

                    # replace the msubsup with 'X_Y__Z'
                    k[2].remove(k[2][0])
                    newk = etree.Element('mi')
                    newk.text = '%s_%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]), flatten_pmathml(k[2]))
                    xml.replace(k, newk)

                fix_superscripts(k)
        fix_superscripts(xml)

        def fix_msubsup(parent):
            """
            Snuggle returns an error when it sees an <msubsup> replace such
            elements with an <msup>, except the first element is of
            the form a_b. I.e. map a_b^c => (a_b)^c
            """
            for child in parent:
                # fix msubsup
                if (gettag(child) == 'msubsup' and len(child) == 3):
                    newchild = etree.Element('msup')
                    newbase = etree.Element('mi')
                    newbase.text = '%s_%s' % (flatten_pmathml(child[0]), flatten_pmathml(child[1]))
                    newexp = child[2]
                    newchild.append(newbase)
                    newchild.append(newexp)
                    parent.replace(child, newchild)

                fix_msubsup(child)
        fix_msubsup(xml)

        self.xml = xml  # pylint: disable=attribute-defined-outside-init
        return self.xml

Example 154

Project: ANALYSE
Source File: formula.py
View license
    def preprocess_pmathml(self, xml):
        r"""
        Pre-process presentation MathML from ASCIIMathML to make it more
        acceptable for SnuggleTeX, and also to accomodate some sympy
        conventions (eg hat(i) for \hat{i}).

        This method would be a good spot to look for an integral and convert
        it, if possible...
        """

        if type(xml) == str or type(xml) == unicode:
            xml = etree.fromstring(xml)		# TODO: wrap in try

        xml = self.fix_greek_in_mathml(xml)	 # convert greek utf letters to greek spelled out in ascii

        def gettag(expr):
            return re.sub('{http://[^}]+}', '', expr.tag)

        def fix_pmathml(xml):
            """
            f and g are processed as functions by asciimathml, eg "f-2" turns
            into "<mrow><mi>f</mi><mo>-</mo></mrow><mn>2</mn>" this is
            really terrible for turning into cmathml.  undo this here.
            """
            for k in xml:
                tag = gettag(k)
                if tag == 'mrow':
                    if len(k) == 2:
                        if gettag(k[0]) == 'mi' and k[0].text in ['f', 'g'] and gettag(k[1]) == 'mo':
                            idx = xml.index(k)
                            xml.insert(idx, deepcopy(k[0]))	 # drop the <mrow> container
                            xml.insert(idx + 1, deepcopy(k[1]))
                            xml.remove(k)
                fix_pmathml(k)

        fix_pmathml(xml)

        def fix_hat(xml):
            """
            hat i is turned into <mover><mi>i</mi><mo>^</mo></mover> ; mangle
            this into <mi>hat(f)</mi> hat i also somtimes turned into
            <mover><mrow> <mi>j</mi> </mrow><mo>^</mo></mover>
            """
            for k in xml:
                tag = gettag(k)
                if tag == 'mover':
                    if len(k) == 2:
                        if gettag(k[0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
                            newk = etree.Element('mi')
                            newk.text = 'hat(%s)' % k[0].text
                            xml.replace(k, newk)
                        if gettag(k[0]) == 'mrow' and gettag(k[0][0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
                            newk = etree.Element('mi')
                            newk.text = 'hat(%s)' % k[0][0].text
                            xml.replace(k, newk)
                fix_hat(k)
        fix_hat(xml)

        def flatten_pmathml(xml):
            """
            Give the text version of certain PMathML elements

            Sometimes MathML will be given with each letter separated (it
            doesn't know if its implicit multiplication or what). From an xml
            node, find the (text only) variable name it represents. So it takes
            <mrow>
              <mi>m</mi>
              <mi>a</mi>
              <mi>x</mi>
            </mrow>
            and returns 'max', for easier use later on.
            """
            tag = gettag(xml)
            if tag == 'mn':
                return xml.text
            elif tag == 'mi':
                return xml.text
            elif tag == 'mrow':
                return ''.join([flatten_pmathml(y) for y in xml])
            raise Exception('[flatten_pmathml] unknown tag %s' % tag)

        def fix_mathvariant(parent):
            """
            Fix certain kinds of math variants

            Literally replace <mstyle mathvariant="script"><mi>N</mi></mstyle>
            with 'scriptN'. There have been problems using script_N or script(N)
            """
            for child in parent:
                if (gettag(child) == 'mstyle' and child.get('mathvariant') == 'script'):
                    newchild = etree.Element('mi')
                    newchild.text = 'script%s' % flatten_pmathml(child[0])
                    parent.replace(child, newchild)
                fix_mathvariant(child)
        fix_mathvariant(xml)

        # find "tagged" superscripts
        # they have the character \u200b in the superscript
        # replace them with a__b so snuggle doesn't get confused
        def fix_superscripts(xml):
            """ Look for and replace sup elements with 'X__Y' or 'X_Y__Z'

            In the javascript, variables with '__X' in them had an invisible
            character inserted into the sup (to distinguish from powers)
            E.g. normal:
            <msubsup>
              <mi>a</mi>
              <mi>b</mi>
              <mi>c</mi>
            </msubsup>
            to be interpreted '(a_b)^c' (nothing done by this method)

            And modified:
            <msubsup>
              <mi>b</mi>
              <mi>x</mi>
              <mrow>
                <mo>&#x200B;</mo>
                <mi>d</mi>
              </mrow>
            </msubsup>
            to be interpreted 'a_b__c'

            also:
            <msup>
              <mi>x</mi>
              <mrow>
                <mo>&#x200B;</mo>
                <mi>B</mi>
              </mrow>
            </msup>
            to be 'x__B'
            """
            for k in xml:
                tag = gettag(k)

                # match things like the last example--
                # the second item in msub is an mrow with the first
                # character equal to \u200b
                if (
                        tag == 'msup' and
                        len(k) == 2 and gettag(k[1]) == 'mrow' and
                        gettag(k[1][0]) == 'mo' and k[1][0].text == u'\u200b'  # whew
                ):

                    # replace the msup with 'X__Y'
                    k[1].remove(k[1][0])
                    newk = etree.Element('mi')
                    newk.text = '%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]))
                    xml.replace(k, newk)

                # match things like the middle example-
                # the third item in msubsup is an mrow with the first
                # character equal to \u200b
                if (
                        tag == 'msubsup' and
                        len(k) == 3 and gettag(k[2]) == 'mrow' and
                        gettag(k[2][0]) == 'mo' and k[2][0].text == u'\u200b'    # whew
                ):

                    # replace the msubsup with 'X_Y__Z'
                    k[2].remove(k[2][0])
                    newk = etree.Element('mi')
                    newk.text = '%s_%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]), flatten_pmathml(k[2]))
                    xml.replace(k, newk)

                fix_superscripts(k)
        fix_superscripts(xml)

        def fix_msubsup(parent):
            """
            Snuggle returns an error when it sees an <msubsup> replace such
            elements with an <msup>, except the first element is of
            the form a_b. I.e. map a_b^c => (a_b)^c
            """
            for child in parent:
                # fix msubsup
                if (gettag(child) == 'msubsup' and len(child) == 3):
                    newchild = etree.Element('msup')
                    newbase = etree.Element('mi')
                    newbase.text = '%s_%s' % (flatten_pmathml(child[0]), flatten_pmathml(child[1]))
                    newexp = child[2]
                    newchild.append(newbase)
                    newchild.append(newexp)
                    parent.replace(child, newchild)

                fix_msubsup(child)
        fix_msubsup(xml)

        self.xml = xml  # pylint: disable=attribute-defined-outside-init
        return self.xml

Example 155

Project: riker
Source File: api.py
View license
def deploy_latest_app_ami(app_name, env_name):

    global aws
    aws = AWS.from_config(config)

    aws.connect()

    lb_group_ids=[aws.get_security_group_id('riker-load-balancer')]
    inst_group_ids=[aws.get_security_group_id('riker-instance')]

    app = App(env_name, app_name)

    health_check_target = app.config.get('health_check', 'TCP:80')

    name = re.sub('[^A-Za-z0-9\-]', '-', app.name)

    app_image = LatestAppImage(app).get()

    print '-----> Connecting to ELB'
    elb_conn = boto.connect_elb()

    log('info', 'Load balancer', show_header=True)
    load_balancer_name = name
    try:
        elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
        lb = elb_result[0]
        log('info', 'Found {}'.format(load_balancer_name))
    except boto.exception.BotoServerError:
        log('info', 'Not found, creating load balancer')
        listeners = [(80, 80, 'HTTP', 'HTTP')]
        lb = elb_conn.create_load_balancer(name=load_balancer_name,
                                           zones=None,
                                           complex_listeners=listeners,
                                           security_groups=lb_group_ids,
                                           subnets=[aws.subnet_id])
    hc = HealthCheck(target=health_check_target)
    lb.configure_health_check(hc)
    cda = ConnectionDrainingAttribute()
    cda.enabled = True
    cda.timeout = 300
    elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name,
                                  attribute='connectionDraining',
                                  value=cda)

    print '-----> Connecting to AutoScale'
    as_conn = boto.connect_autoscale()

    log('info', 'Launch configuration', show_header=True)
    launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id'])
    lc_result = as_conn.get_all_launch_configurations(names=[launch_config_name])
    if len(lc_result) == 0:
        log('info', 'Not found, creating LaunchConfiguration')
        lc = LaunchConfiguration(name=launch_config_name,
                                 image_id=app_image.id,
                                 key_name=aws.key_pair_name,
                                 security_groups=inst_group_ids,
                                 instance_type=aws.instance_type)
        as_conn.create_launch_configuration(lc)
    else:
        log('info', 'Found {}'.format(launch_config_name))
        lc = lc_result[0]

    existing_group = None
    deploy_id = int(app_image.tags['deploy-id'] or 0)
    log('info', 'Getting previous auto-scaling group', show_header=True)
    for did in xrange(deploy_id-1, 0, -1):
        existing_group_name = "{}-{}".format(name, did)
        log('info', '{} ?'.format(existing_group_name))
        ag_result = as_conn.get_all_groups(names=[existing_group_name])
        if len(ag_result) > 0:
            existing_group = ag_result[0]
            log('info', 'Found {}'.format(existing_group.name))
            break
        else:
            log('info', 'No')

    if existing_group is not None:
        existing_healthy_instances = [inst for inst in existing_group.instances if inst.lifecycle_state == 'InService' and inst.health_status == 'Healthy']
        existing_healthy_instance_count = len(existing_healthy_instances)
        desired_capacity = existing_group.desired_capacity
        min_size = existing_group.min_size
        max_size = existing_group.max_size
        if existing_healthy_instance_count == 0 and desired_capacity == 0:
            print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format(existing_group)
            desired_capacity = 1
            min_size = 1
            max_size = max_size if max_size > 0 else 1
    else:
        existing_healthy_instance_count = 0
        desired_capacity = 1
        min_size = 1
        max_size = 1

    log('info', '{} existing instance(s) found'.format(existing_healthy_instance_count), show_header=True)

    log('info', 'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}'.format(desired_capacity, min_size, max_size))

    log('info', 'Auto-scaling group', show_header=True)
    group_name = "{}-{}".format(name, app_image.tags['deploy-id'])
    ag_result = as_conn.get_all_groups(names=[group_name])
    if len(ag_result) == 0:
        log('info', 'Not found, creating autoscale group')
        ag = AutoScalingGroup(name=group_name,
                              load_balancers=[load_balancer_name], launch_config=lc,
                              desired_capacity=desired_capacity, min_size=min_size, max_size=max_size,
                              health_check_type='ELB', health_check_period='300',
                              vpc_zone_identifier=aws.subnet_id)
        as_conn.create_auto_scaling_group(ag)
    else:
        log('info', 'Found {}'.format(group_name))
        ag = ag_result[0]
        ag.desired_capacity = desired_capacity
        ag.max_size = max_size
        ag.min_size = min_size
        ag.launch_config_name = launch_config_name
        ag.update()

    log('info', 'Waiting for new instances to become healthy', show_header=True)
    all_healthy = False
    for i in xrange(60):
        if i > 0:
            print '       ---'
            time.sleep(10)
        elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
        lb = elb_result[0]
        lb_insts = lb.get_instance_health()
        print '       Load-balancer instances: {}'.format(lb_insts)
        # NOTE: re-get auto-scaling group to get updated instance info.
        ag = as_conn.get_all_groups(names=[group_name])[0]
        ag_insts = [inst for inst in ag.instances]
        log('info', 'Auto-scaling group Instances: {}'.format(ag_insts))
        if len(ag_insts) < desired_capacity:
            not_yet_launched_count = desired_capacity - len(ag_insts)
            log('info', '{} new instance(s) not yet launched'.format(not_yet_launched_count))
            continue
        ag_inst_ids = set(inst.instance_id for inst in ag_insts)
        lb_inst_ids = set(inst.instance_id for inst in lb_insts)
        asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids)
        if len(asg_insts_not_in_lb) > 0:
            log('info', '{} new instance(s) not yet in load balancer'.format(len(asg_insts_not_in_lb)))
            continue
        new_lb_insts = [inst for inst in lb_insts if inst.instance_id in ag_inst_ids]
        healthy_new_lb_insts = [inst for inst in new_lb_insts if inst.state == 'InService']
        all_healthy = len(healthy_new_lb_insts) == len(ag_insts)
        log('info', '{} new instance(s) are healthy'.format(len(healthy_new_lb_insts)))
        diff = existing_healthy_instance_count - len(healthy_new_lb_insts)
        if existing_group is not None and diff >= 0:
            change = False
            if existing_group.desired_capacity != diff:
                existing_group.desired_capacity = diff
                change = True
            if existing_group.max_size != diff:
                existing_group.max_size = diff
                change = True
            if diff < existing_group.min_size:
                existing_group.min_size = diff
                change = True
            if change:
                existing_group.update()
                log('info', 'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}'.format(existing_group, existing_group.desired_capacity, existing_group.min_size, existing_group.max_size))
        if all_healthy:
            log('info', 'All new instances healthy!', show_header=True)
            healthy_lb_inst_ids = [inst.instance_id for inst in lb_insts if inst.state == 'InService']
            previous_healthy_inst_ids = [inst.instance_id for inst in existing_healthy_instances] if existing_group else []
            not_yet_out_of_service = set(previous_healthy_inst_ids).intersection(healthy_lb_inst_ids)
            if len(not_yet_out_of_service) > 0:
                log('info', 'Waiting to remove previous instances ({}) from load balancer'.format(not_yet_out_of_service))
            else:
                log('info', 'All previous instances ({}) have been removed from load balancer'.format(previous_healthy_inst_ids), show_header=True)
        if all_healthy and len(not_yet_out_of_service) == 0:
            break
    else:
        raise Exception("Timeout")

    elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
    lb = elb_result[0]
    lb_insts = [inst for inst in lb.get_instance_health() if inst.state == 'InService']
    print '-----> Deployed {} instance(s) of {} to {}'.format(lb_insts, app.name, lb.dns_name)

    print '-----> DONE!'

Example 156

Project: karesansui
Source File: guestimport.py
View license
    @auth
    def _POST(self, *param, **params):
        host_id = self.chk_hostby1(param)
        if host_id is None: return web.notfound()

        model = findbyhost1(self.orm, host_id)

        if not validates_guest_import(self):
            return web.badrequest(self.view.alert)

        uuid = self.input.uuid
        kvc = KaresansuiVirtConnection()
        try:
            # Storage Pool
            #inactive_pool = kvc.list_inactive_storage_pool()
            inactive_pool = []
            active_pool = kvc.list_active_storage_pool()
            pools = inactive_pool + active_pool
            pools.sort()

            export = []
            for pool_name in pools:
                pool = kvc.search_kvn_storage_pools(pool_name)
                path = pool[0].get_info()["target"]["path"]
                if os.path.exists(path):
                    for _afile in glob.glob("%s/*/info.dat" % (path,)):
                        e_param = ExportConfigParam()
                        e_param.load_xml_config(_afile)

                        if e_param.get_uuid() != uuid:
                            continue

                        e_name = e_param.get_domain()
                        _dir = os.path.dirname(_afile)

                        param = ConfigParam(e_name)

                        path = "%s/%s.xml" % (_dir, e_name)
                        if os.path.isfile(path) is False:
                            self.logger.error('Export corrupt data.(file not found) - path=%s' % path)
                            return web.internalerror()

                        param.load_xml_config(path)

                        if e_name != param.get_domain_name():
                            self.logger.error('Export corrupt data.(The name does not match) - info=%s, xml=%s' \
                                              % (e_name, param.get_name()))
                            return web.internalerror()

                        _dir = os.path.dirname(_afile)

                        title = e_param.get_title()
                        if title != "":
                            title = re.sub("[\r\n]","",title)
                        if title == "":
                            title = _('untitled')

                        created = e_param.get_created()
                        if created != "":
                            created_str = time.strftime("%Y/%m/%d %H:%M:%S", \
                                                        time.localtime(float(created)))
                        else:
                            created_str = _("N/A")

                        export.append({"info" : {"dir" : _dir,
                                                 "pool" : pool_name,
                                                 "uuid" : e_param.get_uuid(),
                                                 "name" : e_param.get_domain(),
                                                 "created" : int(created),
                                                 "created_str" : created_str,
                                                 "title" : title,
                                                 "database" : {"name" : e_param.database["name"],
                                                               "tags" : e_param.database["tags"],
                                                               "attribute" : e_param.database["attribute"],
                                                               "notebook" : {"title" : e_param.database["notebook"]["title"],
                                                                             "value" : e_param.database["notebook"]["value"],
                                                                             },
                                                               "uniq_key" : e_param.database["uniq_key"],
                                                               "hypervisor" : e_param.database["hypervisor"],
                                                               "icon" : e_param.database["icon"],
                                                               },
                                                  },
                                       "xml" : {"on_reboot" : param.get_behavior('on_reboot'),
                                                "on_poweroff" : param.get_behavior('on_poweroff'),
                                                "on_crash" : param.get_behavior('on_crash'),
                                                "boot_dev" : param.get_boot_dev(),
                                                #"bootloader" : param.get_bootloader(),
                                                #"commandline" : param.get_commandline(),
                                                #"current_snapshot" : param.get_current_snapshot(),
                                                'disk' : param.get_disk(),
                                                "domain_name" : param.get_domain_name(),
                                                "domain_type" : param.get_domain_type(),
                                                "features_acpi" : param.get_features_acpi(),
                                                "features_apic" : param.get_features_apic(),
                                                "features_pae" : param.get_features_pae(),
                                                #"initrd" : param.get_initrd(),
                                                "interface" : param.get_interface(),
                                                #"kernel" : param.get_kernel(),
                                                "max_memory" : param.get_max_memory(),
                                                'max_vcpus' : param.get_max_vcpus(),
                                                "max_vcpus_limit" : param.get_max_vcpus_limit(),
                                                "memory" : param.get_memory(),
                                                "uuid" : param.get_uuid(),
                                                "vcpus" : param.get_vcpus(),
                                                "vcpus_limit" : param.get_vcpus_limit(),
                                                "graphics_autoport" : param.get_graphics_autoport(),
                                                "keymap" : param.get_graphics_keymap(),
                                                "graphics_listen" : param.get_graphics_listen(),
                                                "graphics_passwd" : param.get_graphics_passwd(),
                                                "graphics_port" : param.get_graphics_port(),
                                                },
                                       "pool" : pool[0].get_info(),
                                       })
            if len(export) != 1:
                self.logger.info("Export does not exist. - uuid=%s" % self.view.uuid)
                return web.badrequest()
            else:
                export = export[0]
        finally:
            kvc.close()

        # Pool running?
        if export['pool']['is_active'] is False:
            return web.badrequest("The destination, the storage pool is not running.")

        dest_domname = export['xml']['domain_name']
        dest_uniqkey = export['info']['database']['uniq_key']
        # Same guest OS is already running.
        if m_findby1uniquekey(self.orm, dest_uniqkey) is not None:
            self.logger.info(_("guest '%s' already exists. (DB) - %s") % (dest_domname, dest_uniqkey))
            return web.badrequest(_("guest '%s' already exists.") % dest_domname)

        dest_dir = "%s/%s" % (export['pool']['target']['path'], export['xml']['domain_name'])
        if os.path.exists(dest_dir) is True:
            self.logger.info(_("guest '%s' already exists. (FS) - %s") % (dest_domname, dest_dir))
            return  web.badrequest(_("guest '%s' already exists.") % dest_domname)

        # disk check
        try:
            src_disk = "%s/%s/images/%s.img" \
                       % (export["info"]["dir"], export["info"]["name"], export["info"]["name"])

            if os.path.exists(src_disk):
                s_size = os.path.getsize(src_disk) / (1024 * 1024) # a unit 'MB'
                if chk_create_disk(export["info"]["dir"], s_size) is False:
                    partition = get_partition_info(export["info"]["dir"], header=False)
                    return web.badrequest(
                        _("No space available to create disk image in '%s' partition.") \
                        % partition[5][0])
        except:
            pass

        extra_uniq_key = string_from_uuid(generate_uuid())
        options = {}
        options["exportuuid"] = export["info"]["uuid"]
        options["destuuid"] = extra_uniq_key
        options["quiet"] = None

        # Database Notebook
        try:
            _notebook = n_new(
                export["info"]["database"]["notebook"]["title"],
                export["info"]["database"]["notebook"]["value"],
                )
        except:
            _notebook = None

        # Database Tag
        _tags = []
        try:
            tag_array = comma_split(export["info"]["database"]["tags"])
            tag_array = uniq_sort(tag_array)
            for x in tag_array:
                if t_count(self.orm, x) == 0:
                    _tags.append(t_new(x))
                else:
                    _tags.append(t_name(self.orm, x))
        except:
            _tags.append(t_new(""))

        parent = m_findby1(self.orm, host_id)
        dest_guest = m_new(created_user=self.me,
                           modified_user=self.me,
                           uniq_key=extra_uniq_key,
                           name=export["info"]["database"]["name"],
                           attribute=int(export["info"]["database"]["attribute"]),
                           hypervisor=int(export["info"]["database"]["hypervisor"]),
                           notebook=_notebook,
                           tags=_tags,
                           icon=export["info"]["database"]["icon"],
                           is_deleted=False,
                           parent=parent,
                           )

        ret = regist_guest(self,
                           _guest=dest_guest,
                           icon_filename=export["info"]["database"]["icon"],
                           cmd=VIRT_COMMAND_IMPORT_GUEST,
                           options=options,
                           cmdname=['Import Guest', 'Import Guest'],
                           rollback_options={"name" : export["xml"]["domain_name"]},
                           is_create=False
                           )

        if ret is True:
            return web.accepted()
        else:
            return False

Example 157

Project: karesansui
Source File: guestimport.py
View license
    @auth
    def _POST(self, *param, **params):
        host_id = self.chk_hostby1(param)
        if host_id is None: return web.notfound()

        model = findbyhost1(self.orm, host_id)

        if not validates_guest_import(self):
            return web.badrequest(self.view.alert)

        uuid = self.input.uuid
        kvc = KaresansuiVirtConnection()
        try:
            # Storage Pool
            #inactive_pool = kvc.list_inactive_storage_pool()
            inactive_pool = []
            active_pool = kvc.list_active_storage_pool()
            pools = inactive_pool + active_pool
            pools.sort()

            export = []
            for pool_name in pools:
                pool = kvc.search_kvn_storage_pools(pool_name)
                path = pool[0].get_info()["target"]["path"]
                if os.path.exists(path):
                    for _afile in glob.glob("%s/*/info.dat" % (path,)):
                        e_param = ExportConfigParam()
                        e_param.load_xml_config(_afile)

                        if e_param.get_uuid() != uuid:
                            continue

                        e_name = e_param.get_domain()
                        _dir = os.path.dirname(_afile)

                        param = ConfigParam(e_name)

                        path = "%s/%s.xml" % (_dir, e_name)
                        if os.path.isfile(path) is False:
                            self.logger.error('Export corrupt data.(file not found) - path=%s' % path)
                            return web.internalerror()

                        param.load_xml_config(path)

                        if e_name != param.get_domain_name():
                            self.logger.error('Export corrupt data.(The name does not match) - info=%s, xml=%s' \
                                              % (e_name, param.get_name()))
                            return web.internalerror()

                        _dir = os.path.dirname(_afile)

                        title = e_param.get_title()
                        if title != "":
                            title = re.sub("[\r\n]","",title)
                        if title == "":
                            title = _('untitled')

                        created = e_param.get_created()
                        if created != "":
                            created_str = time.strftime("%Y/%m/%d %H:%M:%S", \
                                                        time.localtime(float(created)))
                        else:
                            created_str = _("N/A")

                        export.append({"info" : {"dir" : _dir,
                                                 "pool" : pool_name,
                                                 "uuid" : e_param.get_uuid(),
                                                 "name" : e_param.get_domain(),
                                                 "created" : int(created),
                                                 "created_str" : created_str,
                                                 "title" : title,
                                                 "database" : {"name" : e_param.database["name"],
                                                               "tags" : e_param.database["tags"],
                                                               "attribute" : e_param.database["attribute"],
                                                               "notebook" : {"title" : e_param.database["notebook"]["title"],
                                                                             "value" : e_param.database["notebook"]["value"],
                                                                             },
                                                               "uniq_key" : e_param.database["uniq_key"],
                                                               "hypervisor" : e_param.database["hypervisor"],
                                                               "icon" : e_param.database["icon"],
                                                               },
                                                  },
                                       "xml" : {"on_reboot" : param.get_behavior('on_reboot'),
                                                "on_poweroff" : param.get_behavior('on_poweroff'),
                                                "on_crash" : param.get_behavior('on_crash'),
                                                "boot_dev" : param.get_boot_dev(),
                                                #"bootloader" : param.get_bootloader(),
                                                #"commandline" : param.get_commandline(),
                                                #"current_snapshot" : param.get_current_snapshot(),
                                                'disk' : param.get_disk(),
                                                "domain_name" : param.get_domain_name(),
                                                "domain_type" : param.get_domain_type(),
                                                "features_acpi" : param.get_features_acpi(),
                                                "features_apic" : param.get_features_apic(),
                                                "features_pae" : param.get_features_pae(),
                                                #"initrd" : param.get_initrd(),
                                                "interface" : param.get_interface(),
                                                #"kernel" : param.get_kernel(),
                                                "max_memory" : param.get_max_memory(),
                                                'max_vcpus' : param.get_max_vcpus(),
                                                "max_vcpus_limit" : param.get_max_vcpus_limit(),
                                                "memory" : param.get_memory(),
                                                "uuid" : param.get_uuid(),
                                                "vcpus" : param.get_vcpus(),
                                                "vcpus_limit" : param.get_vcpus_limit(),
                                                "graphics_autoport" : param.get_graphics_autoport(),
                                                "keymap" : param.get_graphics_keymap(),
                                                "graphics_listen" : param.get_graphics_listen(),
                                                "graphics_passwd" : param.get_graphics_passwd(),
                                                "graphics_port" : param.get_graphics_port(),
                                                },
                                       "pool" : pool[0].get_info(),
                                       })
            if len(export) != 1:
                self.logger.info("Export does not exist. - uuid=%s" % self.view.uuid)
                return web.badrequest()
            else:
                export = export[0]
        finally:
            kvc.close()

        # Pool running?
        if export['pool']['is_active'] is False:
            return web.badrequest("The destination, the storage pool is not running.")

        dest_domname = export['xml']['domain_name']
        dest_uniqkey = export['info']['database']['uniq_key']
        # Same guest OS is already running.
        if m_findby1uniquekey(self.orm, dest_uniqkey) is not None:
            self.logger.info(_("guest '%s' already exists. (DB) - %s") % (dest_domname, dest_uniqkey))
            return web.badrequest(_("guest '%s' already exists.") % dest_domname)

        dest_dir = "%s/%s" % (export['pool']['target']['path'], export['xml']['domain_name'])
        if os.path.exists(dest_dir) is True:
            self.logger.info(_("guest '%s' already exists. (FS) - %s") % (dest_domname, dest_dir))
            return  web.badrequest(_("guest '%s' already exists.") % dest_domname)

        # disk check
        try:
            src_disk = "%s/%s/images/%s.img" \
                       % (export["info"]["dir"], export["info"]["name"], export["info"]["name"])

            if os.path.exists(src_disk):
                s_size = os.path.getsize(src_disk) / (1024 * 1024) # a unit 'MB'
                if chk_create_disk(export["info"]["dir"], s_size) is False:
                    partition = get_partition_info(export["info"]["dir"], header=False)
                    return web.badrequest(
                        _("No space available to create disk image in '%s' partition.") \
                        % partition[5][0])
        except:
            pass

        extra_uniq_key = string_from_uuid(generate_uuid())
        options = {}
        options["exportuuid"] = export["info"]["uuid"]
        options["destuuid"] = extra_uniq_key
        options["quiet"] = None

        # Database Notebook
        try:
            _notebook = n_new(
                export["info"]["database"]["notebook"]["title"],
                export["info"]["database"]["notebook"]["value"],
                )
        except:
            _notebook = None

        # Database Tag
        _tags = []
        try:
            tag_array = comma_split(export["info"]["database"]["tags"])
            tag_array = uniq_sort(tag_array)
            for x in tag_array:
                if t_count(self.orm, x) == 0:
                    _tags.append(t_new(x))
                else:
                    _tags.append(t_name(self.orm, x))
        except:
            _tags.append(t_new(""))

        parent = m_findby1(self.orm, host_id)
        dest_guest = m_new(created_user=self.me,
                           modified_user=self.me,
                           uniq_key=extra_uniq_key,
                           name=export["info"]["database"]["name"],
                           attribute=int(export["info"]["database"]["attribute"]),
                           hypervisor=int(export["info"]["database"]["hypervisor"]),
                           notebook=_notebook,
                           tags=_tags,
                           icon=export["info"]["database"]["icon"],
                           is_deleted=False,
                           parent=parent,
                           )

        ret = regist_guest(self,
                           _guest=dest_guest,
                           icon_filename=export["info"]["database"]["icon"],
                           cmd=VIRT_COMMAND_IMPORT_GUEST,
                           options=options,
                           cmdname=['Import Guest', 'Import Guest'],
                           rollback_options={"name" : export["xml"]["domain_name"]},
                           is_create=False
                           )

        if ret is True:
            return web.accepted()
        else:
            return False

Example 158

Project: karesansui
Source File: hostby1watch.py
View license
    @auth
    def _POST(self, *param, **params):
        host_id = self.chk_hostby1(param)
        if host_id is None: return web.notfound()

        if not validates_watch(self):
            self.logger.debug("Set watch failed. Did not validate.")
            return web.badrequest(self.view.alert)

        plugin = self.input.watch_target
        plugin_instance = None
        type = None
        type_instance = None
        plugin_ds = None
        libvirt_host = None

        if plugin == COLLECTD_PLUGIN_CPU:
            #cpu method
            plugin_instance = string.atoi(self.input.logical_cpu_number) - 1
            type_instance = self.input.cpu_status
            type = COLLECTD_CPU_TYPE
            plugin_ds = COLLECTD_CPU_DS

        elif plugin == COLLECTD_PLUGIN_MEMORY:
            #memory method
            type_instance = self.input.memory_status
            type = COLLECTD_MEMORY_TYPE
            plugin_ds = COLLECTD_MEMORY_DS

        elif plugin == COLLECTD_PLUGIN_DF:
            #df method
            type = COLLECTD_DF_TYPE
            type_instance = self.input.df_target_fs
            type_instance = re.sub(r'^/dev/', '', type_instance)
            type_instance = re.sub(r'/', '_', type_instance)
            plugin_ds = self.input.df_disk_status

        elif plugin == COLLECTD_PLUGIN_INTERFACE:
            #interface method
            type = self.input.network_status
            type_instance = self.input.network_target_interface
            plugin_ds = self.input.network_direction

        elif plugin == COLLECTD_PLUGIN_LIBVIRT:
            #libvirt method
            libvirt_host = self.input.libvirt_target_machine
            if self.input.libvirt_target == "cpu":
                if self.input.libvirt_vcpu_target == "total":
                    type = COLLECTD_LIBVIRT_TYPE['CPU_TOTAL']
                else:
                    type = COLLECTD_LIBVIRT_TYPE['VCPU']
                    type_instance = self.input.libvirt_vcpu_target

                plugin_ds = COLLECTD_CPU_DS

            elif self.input.libvirt_target == "disk":
                type = COLLECTD_LIBVIRT_TYPE['DISK_OCTETS']
                type_instance = self.input.libvirt_disk_target
                plugin_ds = self.input.libvirt_disk_value_type

            elif self.input.libvirt_target == "network":
                type = "if_" + self.input.libvirt_network_status
                type_instance = self.input.libvirt_target_interface
                plugin_ds = self.input.libvirt_network_direction

        elif plugin == COLLECTD_PLUGIN_LOAD:
            #load method
            type = COLLECTD_LOAD_TYPE
            plugin_ds = self.input.load_term

        else:
            self.logger.debug("Set watch failed. Unknown plugin type.")
            return web.badrequest()

        plugin_selector = create_plugin_selector(plugin_instance, type, type_instance, plugin_ds, libvirt_host)

        ## text
        continuation_count = self.input.continuation_count
        prohibition_period = self.input.prohibition_period
        threshold_val1     = self.input.threshold_val1
        threshold_val2     = self.input.threshold_val2
        threshold_type     = self.input.threshold_type
        if is_param(self.input, 'warning_script'):
            warning_script = self.input.warning_script
        else:
            warning_script = ""
        if is_param(self.input, 'warning_mail_body'):
            warning_mail_body = self.input.warning_mail_body
        else:
            warning_mail_body = ""
        if is_param(self.input, 'failure_script'):
            failure_script = self.input.failure_script
        else:
            failure_script = ""
        if is_param(self.input, 'failure_mail_body'):
            failure_mail_body = self.input.failure_mail_body
        else:
            failure_mail_body = ""
        if is_param(self.input, 'okay_script'):
            okay_script = self.input.okay_script
        else:
            okay_script = ""
        if is_param(self.input, 'okay_mail_body'):
            okay_mail_body = self.input.okay_mail_body
        else:
            okay_mail_body = ""
        if is_param(self.input, 'notify_mail_to'):
            notify_mail_to = self.input.notify_mail_to
        else:
            notify_mail_to = ""
        if is_param(self.input, 'notify_mail_from'):
            notify_mail_from = self.input.notify_mail_from
        else:
            notify_mail_from = ""

        ## bool
        bool_input_key = ["use_percentage", "enable_warning_mail",
                          "enable_failure_mail", "enable_okay_mail",
                          "enable_warning_script", "enable_failure_script",
                          "enable_okay_script"]
        bool_values = {}
        for key in bool_input_key:
            if self.input.has_key(key):
                bool_values.update({key:True})
            else:
                bool_values.update({key:False})

        if threshold_type == "max":
            warning_value = create_threshold_value(min_value=None, max_value=threshold_val1)
            failure_value = create_threshold_value(min_value=None, max_value=threshold_val2)
        elif threshold_type == "min":
            warning_value = create_threshold_value(min_value=threshold_val2, max_value=None)
            failure_value = create_threshold_value(min_value=threshold_val1, max_value=None)
        else:
            self.logger.debug("Set watch failed. Unknown threshold type.")
            return web.badrequest()

        machine = m_findby1(self.orm, host_id)

        if w_is_uniq_duplication(self.orm, machine, plugin, plugin_selector) is True:
            self.logger.debug("Set watch failed. Duplicate watch DB.")
            return web.badrequest("Set watch failed. Duplication watch")

        _watch = w_new(created_user          = self.me,
                       modified_user         = self.me,
                       name                  = self.input.watch_name,
                       plugin                = plugin,
                       plugin_selector       = plugin_selector,
                       karesansui_version    = get_karesansui_version(),
                       collectd_version      = get_collectd_version(),
                       machine               = machine,
                       continuation_count    = continuation_count,
                       prohibition_period    = prohibition_period,
                       warning_value         = warning_value,
                       is_warning_percentage = bool_values.get("use_percentage"),
                       is_warning_script     = bool_values.get("enable_warning_script"),
                       warning_script        = warning_script,
                       is_warning_mail       = bool_values.get("enable_warning_mail"),
                       warning_mail_body     = warning_mail_body,
                       failure_value         = failure_value,
                       is_failure_percentage = bool_values.get("use_percentage"),
                       is_failure_script     = bool_values.get("enable_failure_script"),
                       failure_script        = failure_script,
                       is_failure_mail       = bool_values.get("enable_failure_mail"),
                       failure_mail_body     = failure_mail_body,
                       is_okay_script        = bool_values.get("enable_okay_script"),
                       okay_script           = okay_script,
                       is_okay_mail          = bool_values.get("enable_okay_mail"),
                       okay_mail_body        = okay_mail_body,
                       notify_mail_to        = notify_mail_to,
                       notify_mail_from      = notify_mail_from,
                       is_deleted            = False,
                       )
        w_save(self.orm, _watch)

        modules = ["collectdplugin"]

        host = m_findbyhost1(self.orm, host_id)
        extra_args = {'include':'^threshold_'}
        #extra_args = {}
        dop = read_conf(modules, webobj=self, machine=host, extra_args=extra_args)
        if dop is False:
            self.logger.debug("Set watch failed. Failed read conf.")
            return web.internalerror('Internal Server Error. (Read Conf)')

        params = {}
        if threshold_type == "max":
            params['WarningMax'] = str(threshold_val1)
            params['FailureMax'] = str(threshold_val2)
        elif threshold_type == "min":
            params['WarningMin'] = str(threshold_val2)
            params['FailureMin'] = str(threshold_val1)

        params['Percentage'] = str(bool_values.get("use_percentage")).lower()
        params['Persist']    = "true"
        set_threshold(plugin,plugin_selector,params,dop=dop,webobj=self, host=host)

        extra_args = {}
        command = "/etc/init.d/collectd condrestart"
        extra_args = {"post-command": command}
        retval = write_conf(dop,  webobj=self, machine=host, extra_args=extra_args)
        if retval is False:
            self.logger.debug("Set watch failed. Failed write conf.")
            return web.internalerror('Internal Server Error. (Write Conf)')

        return web.created(None)

Example 159

Project: karesansui
Source File: hostby1watch.py
View license
    @auth
    def _POST(self, *param, **params):
        host_id = self.chk_hostby1(param)
        if host_id is None: return web.notfound()

        if not validates_watch(self):
            self.logger.debug("Set watch failed. Did not validate.")
            return web.badrequest(self.view.alert)

        plugin = self.input.watch_target
        plugin_instance = None
        type = None
        type_instance = None
        plugin_ds = None
        libvirt_host = None

        if plugin == COLLECTD_PLUGIN_CPU:
            #cpu method
            plugin_instance = string.atoi(self.input.logical_cpu_number) - 1
            type_instance = self.input.cpu_status
            type = COLLECTD_CPU_TYPE
            plugin_ds = COLLECTD_CPU_DS

        elif plugin == COLLECTD_PLUGIN_MEMORY:
            #memory method
            type_instance = self.input.memory_status
            type = COLLECTD_MEMORY_TYPE
            plugin_ds = COLLECTD_MEMORY_DS

        elif plugin == COLLECTD_PLUGIN_DF:
            #df method
            type = COLLECTD_DF_TYPE
            type_instance = self.input.df_target_fs
            type_instance = re.sub(r'^/dev/', '', type_instance)
            type_instance = re.sub(r'/', '_', type_instance)
            plugin_ds = self.input.df_disk_status

        elif plugin == COLLECTD_PLUGIN_INTERFACE:
            #interface method
            type = self.input.network_status
            type_instance = self.input.network_target_interface
            plugin_ds = self.input.network_direction

        elif plugin == COLLECTD_PLUGIN_LIBVIRT:
            #libvirt method
            libvirt_host = self.input.libvirt_target_machine
            if self.input.libvirt_target == "cpu":
                if self.input.libvirt_vcpu_target == "total":
                    type = COLLECTD_LIBVIRT_TYPE['CPU_TOTAL']
                else:
                    type = COLLECTD_LIBVIRT_TYPE['VCPU']
                    type_instance = self.input.libvirt_vcpu_target

                plugin_ds = COLLECTD_CPU_DS

            elif self.input.libvirt_target == "disk":
                type = COLLECTD_LIBVIRT_TYPE['DISK_OCTETS']
                type_instance = self.input.libvirt_disk_target
                plugin_ds = self.input.libvirt_disk_value_type

            elif self.input.libvirt_target == "network":
                type = "if_" + self.input.libvirt_network_status
                type_instance = self.input.libvirt_target_interface
                plugin_ds = self.input.libvirt_network_direction

        elif plugin == COLLECTD_PLUGIN_LOAD:
            #load method
            type = COLLECTD_LOAD_TYPE
            plugin_ds = self.input.load_term

        else:
            self.logger.debug("Set watch failed. Unknown plugin type.")
            return web.badrequest()

        plugin_selector = create_plugin_selector(plugin_instance, type, type_instance, plugin_ds, libvirt_host)

        ## text
        continuation_count = self.input.continuation_count
        prohibition_period = self.input.prohibition_period
        threshold_val1     = self.input.threshold_val1
        threshold_val2     = self.input.threshold_val2
        threshold_type     = self.input.threshold_type
        if is_param(self.input, 'warning_script'):
            warning_script = self.input.warning_script
        else:
            warning_script = ""
        if is_param(self.input, 'warning_mail_body'):
            warning_mail_body = self.input.warning_mail_body
        else:
            warning_mail_body = ""
        if is_param(self.input, 'failure_script'):
            failure_script = self.input.failure_script
        else:
            failure_script = ""
        if is_param(self.input, 'failure_mail_body'):
            failure_mail_body = self.input.failure_mail_body
        else:
            failure_mail_body = ""
        if is_param(self.input, 'okay_script'):
            okay_script = self.input.okay_script
        else:
            okay_script = ""
        if is_param(self.input, 'okay_mail_body'):
            okay_mail_body = self.input.okay_mail_body
        else:
            okay_mail_body = ""
        if is_param(self.input, 'notify_mail_to'):
            notify_mail_to = self.input.notify_mail_to
        else:
            notify_mail_to = ""
        if is_param(self.input, 'notify_mail_from'):
            notify_mail_from = self.input.notify_mail_from
        else:
            notify_mail_from = ""

        ## bool
        bool_input_key = ["use_percentage", "enable_warning_mail",
                          "enable_failure_mail", "enable_okay_mail",
                          "enable_warning_script", "enable_failure_script",
                          "enable_okay_script"]
        bool_values = {}
        for key in bool_input_key:
            if self.input.has_key(key):
                bool_values.update({key:True})
            else:
                bool_values.update({key:False})

        if threshold_type == "max":
            warning_value = create_threshold_value(min_value=None, max_value=threshold_val1)
            failure_value = create_threshold_value(min_value=None, max_value=threshold_val2)
        elif threshold_type == "min":
            warning_value = create_threshold_value(min_value=threshold_val2, max_value=None)
            failure_value = create_threshold_value(min_value=threshold_val1, max_value=None)
        else:
            self.logger.debug("Set watch failed. Unknown threshold type.")
            return web.badrequest()

        machine = m_findby1(self.orm, host_id)

        if w_is_uniq_duplication(self.orm, machine, plugin, plugin_selector) is True:
            self.logger.debug("Set watch failed. Duplicate watch DB.")
            return web.badrequest("Set watch failed. Duplication watch")

        _watch = w_new(created_user          = self.me,
                       modified_user         = self.me,
                       name                  = self.input.watch_name,
                       plugin                = plugin,
                       plugin_selector       = plugin_selector,
                       karesansui_version    = get_karesansui_version(),
                       collectd_version      = get_collectd_version(),
                       machine               = machine,
                       continuation_count    = continuation_count,
                       prohibition_period    = prohibition_period,
                       warning_value         = warning_value,
                       is_warning_percentage = bool_values.get("use_percentage"),
                       is_warning_script     = bool_values.get("enable_warning_script"),
                       warning_script        = warning_script,
                       is_warning_mail       = bool_values.get("enable_warning_mail"),
                       warning_mail_body     = warning_mail_body,
                       failure_value         = failure_value,
                       is_failure_percentage = bool_values.get("use_percentage"),
                       is_failure_script     = bool_values.get("enable_failure_script"),
                       failure_script        = failure_script,
                       is_failure_mail       = bool_values.get("enable_failure_mail"),
                       failure_mail_body     = failure_mail_body,
                       is_okay_script        = bool_values.get("enable_okay_script"),
                       okay_script           = okay_script,
                       is_okay_mail          = bool_values.get("enable_okay_mail"),
                       okay_mail_body        = okay_mail_body,
                       notify_mail_to        = notify_mail_to,
                       notify_mail_from      = notify_mail_from,
                       is_deleted            = False,
                       )
        w_save(self.orm, _watch)

        modules = ["collectdplugin"]

        host = m_findbyhost1(self.orm, host_id)
        extra_args = {'include':'^threshold_'}
        #extra_args = {}
        dop = read_conf(modules, webobj=self, machine=host, extra_args=extra_args)
        if dop is False:
            self.logger.debug("Set watch failed. Failed read conf.")
            return web.internalerror('Internal Server Error. (Read Conf)')

        params = {}
        if threshold_type == "max":
            params['WarningMax'] = str(threshold_val1)
            params['FailureMax'] = str(threshold_val2)
        elif threshold_type == "min":
            params['WarningMin'] = str(threshold_val2)
            params['FailureMin'] = str(threshold_val1)

        params['Percentage'] = str(bool_values.get("use_percentage")).lower()
        params['Persist']    = "true"
        set_threshold(plugin,plugin_selector,params,dop=dop,webobj=self, host=host)

        extra_args = {}
        command = "/etc/init.d/collectd condrestart"
        extra_args = {"post-command": command}
        retval = write_conf(dop,  webobj=self, machine=host, extra_args=extra_args)
        if retval is False:
            self.logger.debug("Set watch failed. Failed write conf.")
            return web.internalerror('Internal Server Error. (Write Conf)')

        return web.created(None)

Example 160

Project: pypdflite
Source File: ttfonts.py
View license
    def extractInfo(self):
        #################/
        # name - Naming table
        #################/
        self.sFamilyClass = 0
        self.sFamilySubClass = 0

        name_offset = self.seek_table("name")
        format = self.read_ushort()
        if format != 0:
            raise Exception("Unknown name table format " + format)
        numRecords = self.read_ushort()
        string_data_offset = name_offset + self.read_ushort()
        names = {1: '', 2: '', 3: '', 4: '', 6: ''}
        K = names.keys()
        nameCount = len(names)
        for i in range(numRecords):
            platformId = self.read_ushort()
            encodingId = self.read_ushort()
            languageId = self.read_ushort()
            nameId = self.read_ushort()
            length = self.read_ushort()
            offset = self.read_ushort()
            if nameId not in K:
                continue
            N = ''
            if platformId == 3 and encodingId == 1 and languageId == 0x409:  # Microsoft, Unicode, US English, PS Name
                opos = self._pos
                self.seek(string_data_offset + offset)
                if length % 2 != 0:
                    raise Exception("PostScript name is UTF-16BE string of odd length")
                length /= 2
                N = ''
                while length > 0:
                    char = self.read_ushort()
                    N += (chr(char))
                    length -= 1
                self._pos = opos
                self.seek(opos)

            elif platformId == 1 and encodingId == 0 and languageId == 0:  # Macintosh, Roman, English, PS Name
                opos = self._pos
                N = self.get_chunk(string_data_offset + offset, length)
                self._pos = opos
                self.seek(opos)

            if N and names[nameId] == '':
                names[nameId] = N
                nameCount -= 1
                if nameCount == 0:
                    break

        if names[6]:
            psName = names[6]
        elif names[4]:
            psName = re.sub(' ', '-', names[4])
        elif names[1]:
            psName = re.sub(' ', '-', names[1])
        else:
            psName = ''
        if not psName:
            raise Exception("Could not find PostScript font name")
        self.name = psName
        if names[1]:
            self.familyName = names[1]
        else:
            self.familyName = psName
        if names[2]:
            self.styleName = names[2]
        else:
            self.styleName = 'Regular'
        if names[4]:
            self.fullName = names[4]
        else:
            self.fullName = psName
        if names[3]:
            self.uniqueFontID = names[3]
        else:
            self.uniqueFontID = psName
        if names[6]:
            self.fullName = names[6]

        #################/
        # head - Font header table
        #################/
        self.seek_table("head")
        self.skip(18)
        self.unitsPerEm = unitsPerEm = self.read_ushort()
        scale = 1000 / float(unitsPerEm)
        self.skip(16)
        xMin = self.read_short()
        yMin = self.read_short()
        xMax = self.read_short()
        yMax = self.read_short()
        self.bbox = [(xMin * scale), (yMin * scale), (xMax * scale), (yMax * scale)]
        self.skip(3 * 2)
        indexToLocFormat = self.read_ushort()
        glyphDataFormat = self.read_ushort()
        if glyphDataFormat != 0:
            raise Exception('Unknown glyph data format ' + glyphDataFormat)

        #################/
        # hhea metrics table
        #################/
        # ttf2t1 seems to use this value rather than the one in OS/2 - so put in for compatibility
        if "hhea" in self.tables:
            self.seek_table("hhea")
            self.skip(4)
            hheaAscender = self.read_short()
            hheaDescender = self.read_short()
            self.ascent = (hheaAscender * scale)
            self.descent = (hheaDescender * scale)

        #################/
        # OS/2 - OS/2 and Windows metrics table
        #################/
        if "OS/2" in self.tables:
            self.seek_table("OS/2")
            version = self.read_ushort()
            self.skip(2)
            usWeightClass = self.read_ushort()
            self.skip(2)
            fsType = self.read_ushort()
            if fsType == 0x0002 or (fsType & 0x0300) != 0:
                self.restrictedUse = True
                raise Exception('ERROR - Font file ' + self.filename + ' cannot be embedded due to copyright restrictions.')

            self.skip(20)
            sF = self.read_short()
            self.sFamilyClass = (sF >> 8)
            self.sFamilySubClass = (sF & 0xFF)
            self._pos += 10  # PANOSE = 10 byte length
            panose = self.fh.read(10)
            self.skip(26)
            sTypoAscender = self.read_short()
            sTypoDescender = self.read_short()
            if not self.ascent:
                self.ascent = (sTypoAscender * scale)
            if not self.descent:
                self.descent = (sTypoDescender * scale)
            if version > 1:
                self.skip(16)
                sCapHeight = self.read_short()
                self.capHeight = (sCapHeight * scale)
            else:
                self.capHeight = self.ascent
        else:
            usWeightClass = 500
            if not self.ascent:
                self.ascent = (yMax * scale)
            if not self.descent:
                self.descent = (yMin * scale)
            self.capHeight = self.ascent
        self.stemV = 50 + int(pow((usWeightClass / 65.0), 2))

        #################/
        # post - PostScript table
        #################/
        self.seek_table("post")
        self.skip(4)
        self.italicAngle = self.read_short() + self.read_ushort() / 65536.0
        self.underlinePosition = self.read_short() * scale
        self.underlineThickness = self.read_short() * scale
        isFixedPitch = self.read_ulong()

        self.flags = 4

        if self.italicAngle != 0:
            self.flags = self.flags | 64
        if usWeightClass >= 600:
            self.flags = self.flags | 262144
        if isFixedPitch:
            self.flags = self.flags | 1

        #################/
        # hhea - Horizontal header table
        #################/
        self.seek_table("hhea")
        self.skip(32)
        metricDataFormat = self.read_ushort()
        if metricDataFormat != 0:
            raise Exception('Unknown horizontal metric data format ', metricDataFormat)
        numberOfHMetrics = self.read_ushort()
        if numberOfHMetrics == 0:
            raise Exception('Number of horizontal metrics is 0')

        #################/
        # maxp - Maximum profile table
        #################/
        self.seek_table("maxp")
        self.skip(4)
        numGlyphs = self.read_ushort()

        #################/
        # cmap - Character to glyph index mapping table
        #################/
        cmap_offset = self.seek_table("cmap")
        self.skip(2)
        cmapTableCount = self.read_ushort()
        unicode_cmap_offset = 0
        for i in range(cmapTableCount):
            platformID = self.read_ushort()
            encodingID = self.read_ushort()
            offset = self.read_ulong()
            save_pos = self._pos
            if (platformID == 3 and encodingID == 1) or platformID == 0:  # Microsoft, Unicode
                format = self.get_ushort(cmap_offset + offset)
                if format == 4:
                    if not unicode_cmap_offset:
                        unicode_cmap_offset = cmap_offset + offset
                    break
            self.seek(save_pos)

        if not unicode_cmap_offset:
            raise Exception('Font (' + self.filename + ') does not have cmap for Unicode (platform 3, encoding 1, format 4, or platform 0, any encoding, format 4)')

        glyphToChar = {}
        charToGlyph = {}
        self.getCMAP4(unicode_cmap_offset, glyphToChar, charToGlyph)

        #################/
        # hmtx - Horizontal metrics table
        #################/
        self.getHMTX(numberOfHMetrics, numGlyphs, glyphToChar, scale)

Example 161

Project: pypdflite
Source File: ttfonts.py
View license
    def extractInfo(self):
        #################/
        # name - Naming table
        #################/
        self.sFamilyClass = 0
        self.sFamilySubClass = 0

        name_offset = self.seek_table("name")
        format = self.read_ushort()
        if format != 0:
            raise Exception("Unknown name table format " + format)
        numRecords = self.read_ushort()
        string_data_offset = name_offset + self.read_ushort()
        names = {1: '', 2: '', 3: '', 4: '', 6: ''}
        K = names.keys()
        nameCount = len(names)
        for i in range(numRecords):
            platformId = self.read_ushort()
            encodingId = self.read_ushort()
            languageId = self.read_ushort()
            nameId = self.read_ushort()
            length = self.read_ushort()
            offset = self.read_ushort()
            if nameId not in K:
                continue
            N = ''
            if platformId == 3 and encodingId == 1 and languageId == 0x409:  # Microsoft, Unicode, US English, PS Name
                opos = self._pos
                self.seek(string_data_offset + offset)
                if length % 2 != 0:
                    raise Exception("PostScript name is UTF-16BE string of odd length")
                length /= 2
                N = ''
                while length > 0:
                    char = self.read_ushort()
                    N += (chr(char))
                    length -= 1
                self._pos = opos
                self.seek(opos)

            elif platformId == 1 and encodingId == 0 and languageId == 0:  # Macintosh, Roman, English, PS Name
                opos = self._pos
                N = self.get_chunk(string_data_offset + offset, length)
                self._pos = opos
                self.seek(opos)

            if N and names[nameId] == '':
                names[nameId] = N
                nameCount -= 1
                if nameCount == 0:
                    break

        if names[6]:
            psName = names[6]
        elif names[4]:
            psName = re.sub(' ', '-', names[4])
        elif names[1]:
            psName = re.sub(' ', '-', names[1])
        else:
            psName = ''
        if not psName:
            raise Exception("Could not find PostScript font name")
        self.name = psName
        if names[1]:
            self.familyName = names[1]
        else:
            self.familyName = psName
        if names[2]:
            self.styleName = names[2]
        else:
            self.styleName = 'Regular'
        if names[4]:
            self.fullName = names[4]
        else:
            self.fullName = psName
        if names[3]:
            self.uniqueFontID = names[3]
        else:
            self.uniqueFontID = psName
        if names[6]:
            self.fullName = names[6]

        #################/
        # head - Font header table
        #################/
        self.seek_table("head")
        self.skip(18)
        self.unitsPerEm = unitsPerEm = self.read_ushort()
        scale = 1000 / float(unitsPerEm)
        self.skip(16)
        xMin = self.read_short()
        yMin = self.read_short()
        xMax = self.read_short()
        yMax = self.read_short()
        self.bbox = [(xMin * scale), (yMin * scale), (xMax * scale), (yMax * scale)]
        self.skip(3 * 2)
        indexToLocFormat = self.read_ushort()
        glyphDataFormat = self.read_ushort()
        if glyphDataFormat != 0:
            raise Exception('Unknown glyph data format ' + glyphDataFormat)

        #################/
        # hhea metrics table
        #################/
        # ttf2t1 seems to use this value rather than the one in OS/2 - so put in for compatibility
        if "hhea" in self.tables:
            self.seek_table("hhea")
            self.skip(4)
            hheaAscender = self.read_short()
            hheaDescender = self.read_short()
            self.ascent = (hheaAscender * scale)
            self.descent = (hheaDescender * scale)

        #################/
        # OS/2 - OS/2 and Windows metrics table
        #################/
        if "OS/2" in self.tables:
            self.seek_table("OS/2")
            version = self.read_ushort()
            self.skip(2)
            usWeightClass = self.read_ushort()
            self.skip(2)
            fsType = self.read_ushort()
            if fsType == 0x0002 or (fsType & 0x0300) != 0:
                self.restrictedUse = True
                raise Exception('ERROR - Font file ' + self.filename + ' cannot be embedded due to copyright restrictions.')

            self.skip(20)
            sF = self.read_short()
            self.sFamilyClass = (sF >> 8)
            self.sFamilySubClass = (sF & 0xFF)
            self._pos += 10  # PANOSE = 10 byte length
            panose = self.fh.read(10)
            self.skip(26)
            sTypoAscender = self.read_short()
            sTypoDescender = self.read_short()
            if not self.ascent:
                self.ascent = (sTypoAscender * scale)
            if not self.descent:
                self.descent = (sTypoDescender * scale)
            if version > 1:
                self.skip(16)
                sCapHeight = self.read_short()
                self.capHeight = (sCapHeight * scale)
            else:
                self.capHeight = self.ascent
        else:
            usWeightClass = 500
            if not self.ascent:
                self.ascent = (yMax * scale)
            if not self.descent:
                self.descent = (yMin * scale)
            self.capHeight = self.ascent
        self.stemV = 50 + int(pow((usWeightClass / 65.0), 2))

        #################/
        # post - PostScript table
        #################/
        self.seek_table("post")
        self.skip(4)
        self.italicAngle = self.read_short() + self.read_ushort() / 65536.0
        self.underlinePosition = self.read_short() * scale
        self.underlineThickness = self.read_short() * scale
        isFixedPitch = self.read_ulong()

        self.flags = 4

        if self.italicAngle != 0:
            self.flags = self.flags | 64
        if usWeightClass >= 600:
            self.flags = self.flags | 262144
        if isFixedPitch:
            self.flags = self.flags | 1

        #################/
        # hhea - Horizontal header table
        #################/
        self.seek_table("hhea")
        self.skip(32)
        metricDataFormat = self.read_ushort()
        if metricDataFormat != 0:
            raise Exception('Unknown horizontal metric data format ', metricDataFormat)
        numberOfHMetrics = self.read_ushort()
        if numberOfHMetrics == 0:
            raise Exception('Number of horizontal metrics is 0')

        #################/
        # maxp - Maximum profile table
        #################/
        self.seek_table("maxp")
        self.skip(4)
        numGlyphs = self.read_ushort()

        #################/
        # cmap - Character to glyph index mapping table
        #################/
        cmap_offset = self.seek_table("cmap")
        self.skip(2)
        cmapTableCount = self.read_ushort()
        unicode_cmap_offset = 0
        for i in range(cmapTableCount):
            platformID = self.read_ushort()
            encodingID = self.read_ushort()
            offset = self.read_ulong()
            save_pos = self._pos
            if (platformID == 3 and encodingID == 1) or platformID == 0:  # Microsoft, Unicode
                format = self.get_ushort(cmap_offset + offset)
                if format == 4:
                    if not unicode_cmap_offset:
                        unicode_cmap_offset = cmap_offset + offset
                    break
            self.seek(save_pos)

        if not unicode_cmap_offset:
            raise Exception('Font (' + self.filename + ') does not have cmap for Unicode (platform 3, encoding 1, format 4, or platform 0, any encoding, format 4)')

        glyphToChar = {}
        charToGlyph = {}
        self.getCMAP4(unicode_cmap_offset, glyphToChar, charToGlyph)

        #################/
        # hmtx - Horizontal metrics table
        #################/
        self.getHMTX(numberOfHMetrics, numGlyphs, glyphToChar, scale)

Example 162

Project: kivy-designer
Source File: kivy_console.py
View license
    def on_enter(self, *args):
        '''When the user press enter and wants to run a command
        '''
        self.unbind(on_subprocess_done=self.on_enter)
        # if there is a command running, it's necessary to stop it first
        if self.command_status == 'started':
            self.kill_process()
            # restart this call after killing the running process
            self.bind(on_subprocess_done=self.on_enter)
            return

        txtinput_command_line = self.txtinput_command_line
        add_to_cache = self.add_to_cache
        command_history = self.command_history

        def remove_command_interaction_widgets(*args):
            '''command finished: remove widget responsible for interaction
            '''
            parent.remove_widget(self.interact_layout)
            self.interact_layout = None
            # enable running a new command
            try:
                parent.add_widget(self.txtinput_command_line)
            except:
                self._initialize(0)

            self._focus(txtinput_command_line, True)
            self.command_status = 'closed'
            self.dispatch('on_subprocess_done')

        def run_cmd(*args):
            '''Run the command
            '''
            # this is run inside a thread so take care, avoid gui ops
            try:
                _posix = True
                if sys.platform[0] == 'w':
                    _posix = False
                cmd = command
                if PY2 and isinstance(cmd, unicode):
                    cmd = command.encode(get_fs_encoding())
                if not self.shell:
                    cmd = shlex.split(cmd, posix=_posix)
                    for i in range(len(cmd)):
                        cmd[i] = cmd[i].replace('\x01', ' ')
                    map(lambda s: s.decode(get_fs_encoding()), cmd)
            except Exception as err:
                cmd = ''
                self.add_to_cache(''.join((str(err), ' <', command, ' >\n')))
            if len(cmd) > 0:
                prev_stdout = sys.stdout
                sys.stdout = self.stdout
                try:
                    # execute command
                    self.popen_obj = popen = subprocess.Popen(
                        cmd,
                        bufsize=0,
                        stdout=subprocess.PIPE,
                        stdin=subprocess.PIPE,
                        stderr=subprocess.STDOUT,
                        preexec_fn=None,
                        close_fds=False,
                        shell=self.shell,
                        cwd=self.cur_dir.encode(get_fs_encoding()),
                        universal_newlines=False,
                        startupinfo=None,
                        creationflags=0)
                    popen_stdout_r = popen.stdout.readline
                    popen_stdout_flush = popen.stdout.flush
                    txt = popen_stdout_r()
                    plat = platform()
                    while txt:
                        # skip flush on android
                        if plat[0] != 'a':
                            popen_stdout_flush()

                        if isinstance(txt, bytes):
                            txt = txt.decode(get_fs_encoding())
                        add_to_cache(txt)
                        txt = popen_stdout_r()
                except (OSError, ValueError) as err:
                    add_to_cache(''.join(
                            (str(err),
                             ' < ', command, ' >\n')))
                    self.command_status = 'closed'
                    self.dispatch('on_subprocess_done')
                sys.stdout = prev_stdout
            self.popen_obj = None
            Clock.schedule_once(remove_command_interaction_widgets, 0)

        # append text to textcache
        add_to_cache(self.txtinput_command_line.text + '\n')
        command = txtinput_command_line.text[len(self.prompt()):]

        if command == '':
            self.txtinput_command_line_refocus = True
            return

        # store command in command_history
        if self.command_history_pos > 0:
            self.command_history_pos = len(command_history)
            if command_history[self.command_history_pos - 1] != command:
                command_history.append(command)
        else:
            command_history.append(command)

        len_command_history = len(command_history)
        self.command_history_pos = len(command_history)

        # on reaching limit(cached_lines) pop first command
        if len_command_history >= self.cached_commands:
            self.command_history = command_history[1:]

        # replace $PATH with
        command = os.path.expandvars(command)

        if command == 'clear' or command == 'cls':
            self.clear()
            txtinput_command_line.text = self.prompt()
            self.txtinput_command_line_refocus = True
            self.command_status = 'closed'
            self.dispatch('on_subprocess_done')
            return
        # if command = cd change directory
        if command.startswith('cd ') or command.startswith('export '):
            if command[0] == 'e':
                e_q = command[7:].find('=')
                _exprt = command[7:]
                if e_q:
                    os.environ[_exprt[:e_q]] = _exprt[e_q + 1:]
                    self.environment = os.environ.copy()
            else:
                try:
                    command = re.sub('[ ]+', ' ', command)
                    if command[3] == os.sep:
                        os.chdir(command[3:])
                    else:
                        os.chdir(os.path.join(self.cur_dir, command[3:]))
                    if sys.version_info >= (3, 0):
                        self.cur_dir = os.getcwd()
                    else:
                        self.cur_dir = os.getcwdu()
                except OSError as err:
                    Logger.debug('Shell Console: err:' + err.strerror +
                                 ' directory:' + str(command[3:]))
                    add_to_cache(''.join((err.strerror, '\n')))
            txtinput_command_line.text = self.prompt()
            self.txtinput_command_line_refocus = True
            self.command_status = 'closed'
            self.dispatch('on_subprocess_done')
            return

        txtinput_command_line.text = self.prompt()
        # store output in textcache
        parent = txtinput_command_line.parent
        # disable running a new command while and old one is running
        parent.remove_widget(txtinput_command_line)
        # add widget for interaction with the running command
        txtinput_run_command = TextInput(multiline=False,
                                         font_size=self.font_size)

        def interact_with_command(*l):
            '''Text input to interact with the running command
            '''
            popen_obj = self.popen_obj
            if not popen_obj:
                return
            txt = l[0].text + '\n'
            popen_obj_stdin = popen_obj.stdin
            popen_obj_stdin.write(txt)
            popen_obj_stdin.flush()
            self.txtinput_run_command_refocus = True

        self.txtinput_run_command_refocus = False
        txtinput_run_command.bind(on_text_validate=interact_with_command)
        txtinput_run_command.bind(focus=self.on_focus)
        btn_kill = Button(text="Stop",
                          width=60,
                          size_hint=(None, 1))

        self.interact_layout = il = GridLayout(rows=1, cols=2, height=27,
                                               size_hint=(1, None))
        btn_kill.bind(on_press=self.kill_process)
        il.add_widget(txtinput_run_command)
        il.add_widget(btn_kill)
        parent.add_widget(il)

        txtinput_run_command.focus = True
        self.command_status = 'started'
        thread.start_new_thread(run_cmd, ())

Example 163

Project: kivy-designer
Source File: kivy_console.py
View license
    def on_enter(self, *args):
        '''When the user press enter and wants to run a command
        '''
        self.unbind(on_subprocess_done=self.on_enter)
        # if there is a command running, it's necessary to stop it first
        if self.command_status == 'started':
            self.kill_process()
            # restart this call after killing the running process
            self.bind(on_subprocess_done=self.on_enter)
            return

        txtinput_command_line = self.txtinput_command_line
        add_to_cache = self.add_to_cache
        command_history = self.command_history

        def remove_command_interaction_widgets(*args):
            '''command finished: remove widget responsible for interaction
            '''
            parent.remove_widget(self.interact_layout)
            self.interact_layout = None
            # enable running a new command
            try:
                parent.add_widget(self.txtinput_command_line)
            except:
                self._initialize(0)

            self._focus(txtinput_command_line, True)
            self.command_status = 'closed'
            self.dispatch('on_subprocess_done')

        def run_cmd(*args):
            '''Run the command
            '''
            # this is run inside a thread so take care, avoid gui ops
            try:
                _posix = True
                if sys.platform[0] == 'w':
                    _posix = False
                cmd = command
                if PY2 and isinstance(cmd, unicode):
                    cmd = command.encode(get_fs_encoding())
                if not self.shell:
                    cmd = shlex.split(cmd, posix=_posix)
                    for i in range(len(cmd)):
                        cmd[i] = cmd[i].replace('\x01', ' ')
                    map(lambda s: s.decode(get_fs_encoding()), cmd)
            except Exception as err:
                cmd = ''
                self.add_to_cache(''.join((str(err), ' <', command, ' >\n')))
            if len(cmd) > 0:
                prev_stdout = sys.stdout
                sys.stdout = self.stdout
                try:
                    # execute command
                    self.popen_obj = popen = subprocess.Popen(
                        cmd,
                        bufsize=0,
                        stdout=subprocess.PIPE,
                        stdin=subprocess.PIPE,
                        stderr=subprocess.STDOUT,
                        preexec_fn=None,
                        close_fds=False,
                        shell=self.shell,
                        cwd=self.cur_dir.encode(get_fs_encoding()),
                        universal_newlines=False,
                        startupinfo=None,
                        creationflags=0)
                    popen_stdout_r = popen.stdout.readline
                    popen_stdout_flush = popen.stdout.flush
                    txt = popen_stdout_r()
                    plat = platform()
                    while txt:
                        # skip flush on android
                        if plat[0] != 'a':
                            popen_stdout_flush()

                        if isinstance(txt, bytes):
                            txt = txt.decode(get_fs_encoding())
                        add_to_cache(txt)
                        txt = popen_stdout_r()
                except (OSError, ValueError) as err:
                    add_to_cache(''.join(
                            (str(err),
                             ' < ', command, ' >\n')))
                    self.command_status = 'closed'
                    self.dispatch('on_subprocess_done')
                sys.stdout = prev_stdout
            self.popen_obj = None
            Clock.schedule_once(remove_command_interaction_widgets, 0)

        # append text to textcache
        add_to_cache(self.txtinput_command_line.text + '\n')
        command = txtinput_command_line.text[len(self.prompt()):]

        if command == '':
            self.txtinput_command_line_refocus = True
            return

        # store command in command_history
        if self.command_history_pos > 0:
            self.command_history_pos = len(command_history)
            if command_history[self.command_history_pos - 1] != command:
                command_history.append(command)
        else:
            command_history.append(command)

        len_command_history = len(command_history)
        self.command_history_pos = len(command_history)

        # on reaching limit(cached_lines) pop first command
        if len_command_history >= self.cached_commands:
            self.command_history = command_history[1:]

        # replace $PATH with
        command = os.path.expandvars(command)

        if command == 'clear' or command == 'cls':
            self.clear()
            txtinput_command_line.text = self.prompt()
            self.txtinput_command_line_refocus = True
            self.command_status = 'closed'
            self.dispatch('on_subprocess_done')
            return
        # if command = cd change directory
        if command.startswith('cd ') or command.startswith('export '):
            if command[0] == 'e':
                e_q = command[7:].find('=')
                _exprt = command[7:]
                if e_q:
                    os.environ[_exprt[:e_q]] = _exprt[e_q + 1:]
                    self.environment = os.environ.copy()
            else:
                try:
                    command = re.sub('[ ]+', ' ', command)
                    if command[3] == os.sep:
                        os.chdir(command[3:])
                    else:
                        os.chdir(os.path.join(self.cur_dir, command[3:]))
                    if sys.version_info >= (3, 0):
                        self.cur_dir = os.getcwd()
                    else:
                        self.cur_dir = os.getcwdu()
                except OSError as err:
                    Logger.debug('Shell Console: err:' + err.strerror +
                                 ' directory:' + str(command[3:]))
                    add_to_cache(''.join((err.strerror, '\n')))
            txtinput_command_line.text = self.prompt()
            self.txtinput_command_line_refocus = True
            self.command_status = 'closed'
            self.dispatch('on_subprocess_done')
            return

        txtinput_command_line.text = self.prompt()
        # store output in textcache
        parent = txtinput_command_line.parent
        # disable running a new command while and old one is running
        parent.remove_widget(txtinput_command_line)
        # add widget for interaction with the running command
        txtinput_run_command = TextInput(multiline=False,
                                         font_size=self.font_size)

        def interact_with_command(*l):
            '''Text input to interact with the running command
            '''
            popen_obj = self.popen_obj
            if not popen_obj:
                return
            txt = l[0].text + '\n'
            popen_obj_stdin = popen_obj.stdin
            popen_obj_stdin.write(txt)
            popen_obj_stdin.flush()
            self.txtinput_run_command_refocus = True

        self.txtinput_run_command_refocus = False
        txtinput_run_command.bind(on_text_validate=interact_with_command)
        txtinput_run_command.bind(focus=self.on_focus)
        btn_kill = Button(text="Stop",
                          width=60,
                          size_hint=(None, 1))

        self.interact_layout = il = GridLayout(rows=1, cols=2, height=27,
                                               size_hint=(1, None))
        btn_kill.bind(on_press=self.kill_process)
        il.add_widget(txtinput_run_command)
        il.add_widget(btn_kill)
        parent.add_widget(il)

        txtinput_run_command.focus = True
        self.command_status = 'started'
        thread.start_new_thread(run_cmd, ())

Example 164

Project: calibre
Source File: textileml.py
View license
    def dump_text(self, elem, stylizer):
        '''
        @elem: The element in the etree that we are working on.
        @stylizer: The style information attached to the element.
        '''

        # We can only processes tags. If there isn't a tag return any text.
        if not isinstance(elem.tag, basestring) \
           or namespace(elem.tag) != XHTML_NS:
            p = elem.getparent()
            if p is not None and isinstance(p.tag, basestring) and namespace(p.tag) == XHTML_NS \
                    and elem.tail:
                return [elem.tail]
            return ['']

        # Setup our variables.
        text = ['']
        style = stylizer.style(elem)
        tags = []
        tag = barename(elem.tag)
        attribs = elem.attrib

        # Ignore anything that is set to not be displayed.
        if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
           or style['visibility'] == 'hidden':
            if hasattr(elem, 'tail') and elem.tail:
                return [elem.tail]
            return ['']

        # Soft scene breaks.
        if 'margin-top' in style.cssdict() and style['margin-top'] != 'auto':
            ems = min(int(round(float(style.marginTop) / style.fontSize) - 1), self.MAX_EM)
            if ems >= 1:
                text.append(u'\n\n\xa0' * ems)

        if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div'):
            if tag == 'div':
                tag = 'p'
            text.append(self.build_block(tag, style, attribs, stylizer))
            text.append('. ')
            tags.append('\n')

        if style['font-style'] == 'italic' or tag in ('i', 'em'):
            if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'cite'):
                if self.style_italic == False:  # noqa
                    if self.in_a_link:
                        text.append('_')
                        tags.append('_')
                    else:
                        text.append('[_')
                        tags.append('_]')
                    self.style_embed.append('_')
                    self.style_italic = True
        if style['font-weight'] in ('bold', 'bolder') or tag in ('b', 'strong'):
            if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
                if self.style_bold == False:  # noqa
                    if self.in_a_link:
                        text.append('*')
                        tags.append('*')
                    else:
                        text.append('[*')
                        tags.append('*]')
                    self.style_embed.append('*')
                    self.style_bold = True
        if style['text-decoration'] == 'underline' or tag in ('u', 'ins'):
            if tag != 'a':
                if self.style_under == False:  # noqa
                    text.append('[+')
                    tags.append('+]')
                    self.style_embed.append('+')
                    self.style_under = True
        if style['text-decoration'] == 'line-through' or tag in ('strike', 'del', 's'):
            if self.style_strike == False:  # noqa
                text.append('[-')
                tags.append('-]')
                self.style_embed.append('-')
                self.style_strike = True
        if tag == 'br':
            for i in reversed(self.style_embed):
                text.append(i)
            text.append('\n')
            for i in self.style_embed:
                text.append(i)
            tags.append('')
            self.remove_space_after_newline = True
        if tag == 'blockquote':
            text.append('\nbq. ')
            tags.append('\n')
        elif tag in ('abbr', 'acronym'):
            text.append('')
            txt = attribs['title']
            tags.append('(' + txt + ')')
        elif tag == 'sup':
            text.append('^')
            tags.append('^')
        elif tag == 'sub':
            text.append('~')
            tags.append('~')
        elif tag == 'code':
            if self.in_pre:
                text.append('\nbc. ')
                tags.append('')
            else:
                text.append('@')
                tags.append('@')
        elif tag == 'cite':
            text.append('??')
            tags.append('??')
        elif tag == 'hr':
            text.append('\n***')
            tags.append('\n')
        elif tag == 'pre':
            self.in_pre = True
            text.append('\npre. ')
            tags.append('pre\n')
        elif tag == 'a':
            if self.opts.keep_links:
                if attribs.has_key('href'):  # noqa
                    text.append('"')
                    tags.append('a')
                    tags.append('":' + attribs['href'])
                    self.our_links.append(attribs['href'])
                    if attribs.has_key('title'):  # noqa
                        tags.append('(' + attribs['title'] + ')')
                    self.in_a_link = True
                else:
                    text.append('%')
                    tags.append('%')
        elif tag == 'img':
            if self.opts.keep_image_references:
                txt = '!' + self.check_halign(style)
                txt += self.check_valign(style)
                txt += attribs['src']
                text.append(txt)
                if attribs.has_key('alt'):  # noqa
                    txt = attribs['alt']
                    if txt != '':
                        text.append('(' + txt + ')')
                tags.append('!')
        elif tag in ('ol', 'ul'):
            self.list.append({'name': tag, 'num': 0})
            text.append('')
            tags.append(tag)
        elif tag == 'li':
            if self.list:
                li = self.list[-1]
            else:
                li = {'name': 'ul', 'num': 0}
            text.append('\n')
            if li['name'] == 'ul':
                text.append('*' * len(self.list) + ' ')
            elif li['name'] == 'ol':
                text.append('#' * len(self.list) + ' ')
            tags.append('')
        elif tag == 'dl':
            text.append('\n')
            tags.append('')
        elif tag == 'dt':
            text.append('')
            tags.append('\n')
        elif tag == 'dd':
            text.append('    ')
            tags.append('')
        elif tag == 'dd':
            text.append('')
            tags.append('\n')
        elif tag == 'table':
            txt = self.build_block(tag, style, attribs, stylizer)
            txt += '. \n'
            if txt != '\ntable. \n':
                text.append(txt)
            else:
                text.append('\n')
            tags.append('')
        elif tag == 'tr':
            txt = self.build_block('', style, attribs, stylizer)
            txt += '. '
            if txt != '\n. ':
                txt = re.sub('\n', '', txt)
                text.append(txt)
            tags.append('|\n')
        elif tag == 'td':
            text.append('|')
            txt = ''
            txt += self.check_halign(style)
            txt += self.check_valign(style)
            if attribs.has_key('colspan'):  # noqa
                txt += '\\' + attribs['colspan']
            if attribs.has_key('rowspan'):  # noqa
                txt += '/' + attribs['rowspan']
            txt += self.check_styles(style)
            if txt != '':
                text.append(txt + '. ')
            tags.append('')
        elif tag == 'th':
            text.append('|_. ')
            tags.append('')
        elif tag == 'span':
            if style['font-variant'] == 'small-caps':
                if self.style_smallcap == False:  # noqa
                    text.append('&')
                    tags.append('&')
                    self.style_smallcap = True
            else:
                if self.in_a_link == False:  # noqa
                    txt = '%'
                    if self.opts.keep_links:
                        txt += self.check_id_tag(attribs)
                        txt += self.check_styles(style)
                    if txt != '%':
                        text.append(txt)
                        tags.append('%')

        if self.opts.keep_links and attribs.has_key('id'):  # noqa
            if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'span', 'table'):
                text.append(self.check_id_tag(attribs))

        # Process the styles for any that we want to keep
        if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'hr', 'a', 'img',
                'span', 'table', 'tr', 'td'):
            if not self.in_a_link:
                text.append(self.check_styles(style))

        # Process tags that contain text.
        if hasattr(elem, 'text') and elem.text:
            txt = elem.text
            if not self.in_pre:
                txt = self.prepare_string_for_textile(self.remove_newlines(txt))
            text.append(txt)
            self.id_no_text = u''

        # Recurse down into tags within the tag we are in.
        for item in elem:
            text += self.dump_text(item, stylizer)

        # Close all open tags.
        tags.reverse()
        for t in tags:
            if t in ('pre', 'ul', 'ol', 'li', 'table'):
                if t == 'pre':
                    self.in_pre = False
                elif t in ('ul', 'ol'):
                    if self.list:
                        self.list.pop()
                    if not self.list:
                        text.append('\n')
            else:
                if t == 'a':
                    self.in_a_link = False
                    t = ''
                text.append(self.id_no_text)
                self.id_no_text = u''
                if t in ('*]', '*'):
                    self.style_bold = False
                elif t in ('_]', '_'):
                    self.style_italic = False
                elif t == '+]':
                    self.style_under = False
                elif t == '-]':
                    self.style_strike = False
                elif t == '&':
                    self.style_smallcap = False
                if t in ('*]', '_]', '+]', '-]', '*', '_'):
                    txt = self.style_embed.pop()
                text.append('%s' % t)

        # Soft scene breaks.
        if 'margin-bottom' in style.cssdict() and style['margin-bottom'] != 'auto':
            ems = min(int(round((float(style.marginBottom) / style.fontSize) - 1)), self.MAX_EM)
            if ems >= 1:
                text.append(u'\n\n\xa0' * ems)

        # Add the text that is outside of the tag.
        if hasattr(elem, 'tail') and elem.tail:
            tail = elem.tail
            if not self.in_pre:
                tail = self.prepare_string_for_textile(self.remove_newlines(tail))
            text.append(tail)

        return text

Example 165

Project: calibre
Source File: textileml.py
View license
    def dump_text(self, elem, stylizer):
        '''
        @elem: The element in the etree that we are working on.
        @stylizer: The style information attached to the element.
        '''

        # We can only processes tags. If there isn't a tag return any text.
        if not isinstance(elem.tag, basestring) \
           or namespace(elem.tag) != XHTML_NS:
            p = elem.getparent()
            if p is not None and isinstance(p.tag, basestring) and namespace(p.tag) == XHTML_NS \
                    and elem.tail:
                return [elem.tail]
            return ['']

        # Setup our variables.
        text = ['']
        style = stylizer.style(elem)
        tags = []
        tag = barename(elem.tag)
        attribs = elem.attrib

        # Ignore anything that is set to not be displayed.
        if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
           or style['visibility'] == 'hidden':
            if hasattr(elem, 'tail') and elem.tail:
                return [elem.tail]
            return ['']

        # Soft scene breaks.
        if 'margin-top' in style.cssdict() and style['margin-top'] != 'auto':
            ems = min(int(round(float(style.marginTop) / style.fontSize) - 1), self.MAX_EM)
            if ems >= 1:
                text.append(u'\n\n\xa0' * ems)

        if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div'):
            if tag == 'div':
                tag = 'p'
            text.append(self.build_block(tag, style, attribs, stylizer))
            text.append('. ')
            tags.append('\n')

        if style['font-style'] == 'italic' or tag in ('i', 'em'):
            if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'cite'):
                if self.style_italic == False:  # noqa
                    if self.in_a_link:
                        text.append('_')
                        tags.append('_')
                    else:
                        text.append('[_')
                        tags.append('_]')
                    self.style_embed.append('_')
                    self.style_italic = True
        if style['font-weight'] in ('bold', 'bolder') or tag in ('b', 'strong'):
            if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
                if self.style_bold == False:  # noqa
                    if self.in_a_link:
                        text.append('*')
                        tags.append('*')
                    else:
                        text.append('[*')
                        tags.append('*]')
                    self.style_embed.append('*')
                    self.style_bold = True
        if style['text-decoration'] == 'underline' or tag in ('u', 'ins'):
            if tag != 'a':
                if self.style_under == False:  # noqa
                    text.append('[+')
                    tags.append('+]')
                    self.style_embed.append('+')
                    self.style_under = True
        if style['text-decoration'] == 'line-through' or tag in ('strike', 'del', 's'):
            if self.style_strike == False:  # noqa
                text.append('[-')
                tags.append('-]')
                self.style_embed.append('-')
                self.style_strike = True
        if tag == 'br':
            for i in reversed(self.style_embed):
                text.append(i)
            text.append('\n')
            for i in self.style_embed:
                text.append(i)
            tags.append('')
            self.remove_space_after_newline = True
        if tag == 'blockquote':
            text.append('\nbq. ')
            tags.append('\n')
        elif tag in ('abbr', 'acronym'):
            text.append('')
            txt = attribs['title']
            tags.append('(' + txt + ')')
        elif tag == 'sup':
            text.append('^')
            tags.append('^')
        elif tag == 'sub':
            text.append('~')
            tags.append('~')
        elif tag == 'code':
            if self.in_pre:
                text.append('\nbc. ')
                tags.append('')
            else:
                text.append('@')
                tags.append('@')
        elif tag == 'cite':
            text.append('??')
            tags.append('??')
        elif tag == 'hr':
            text.append('\n***')
            tags.append('\n')
        elif tag == 'pre':
            self.in_pre = True
            text.append('\npre. ')
            tags.append('pre\n')
        elif tag == 'a':
            if self.opts.keep_links:
                if attribs.has_key('href'):  # noqa
                    text.append('"')
                    tags.append('a')
                    tags.append('":' + attribs['href'])
                    self.our_links.append(attribs['href'])
                    if attribs.has_key('title'):  # noqa
                        tags.append('(' + attribs['title'] + ')')
                    self.in_a_link = True
                else:
                    text.append('%')
                    tags.append('%')
        elif tag == 'img':
            if self.opts.keep_image_references:
                txt = '!' + self.check_halign(style)
                txt += self.check_valign(style)
                txt += attribs['src']
                text.append(txt)
                if attribs.has_key('alt'):  # noqa
                    txt = attribs['alt']
                    if txt != '':
                        text.append('(' + txt + ')')
                tags.append('!')
        elif tag in ('ol', 'ul'):
            self.list.append({'name': tag, 'num': 0})
            text.append('')
            tags.append(tag)
        elif tag == 'li':
            if self.list:
                li = self.list[-1]
            else:
                li = {'name': 'ul', 'num': 0}
            text.append('\n')
            if li['name'] == 'ul':
                text.append('*' * len(self.list) + ' ')
            elif li['name'] == 'ol':
                text.append('#' * len(self.list) + ' ')
            tags.append('')
        elif tag == 'dl':
            text.append('\n')
            tags.append('')
        elif tag == 'dt':
            text.append('')
            tags.append('\n')
        elif tag == 'dd':
            text.append('    ')
            tags.append('')
        elif tag == 'dd':
            text.append('')
            tags.append('\n')
        elif tag == 'table':
            txt = self.build_block(tag, style, attribs, stylizer)
            txt += '. \n'
            if txt != '\ntable. \n':
                text.append(txt)
            else:
                text.append('\n')
            tags.append('')
        elif tag == 'tr':
            txt = self.build_block('', style, attribs, stylizer)
            txt += '. '
            if txt != '\n. ':
                txt = re.sub('\n', '', txt)
                text.append(txt)
            tags.append('|\n')
        elif tag == 'td':
            text.append('|')
            txt = ''
            txt += self.check_halign(style)
            txt += self.check_valign(style)
            if attribs.has_key('colspan'):  # noqa
                txt += '\\' + attribs['colspan']
            if attribs.has_key('rowspan'):  # noqa
                txt += '/' + attribs['rowspan']
            txt += self.check_styles(style)
            if txt != '':
                text.append(txt + '. ')
            tags.append('')
        elif tag == 'th':
            text.append('|_. ')
            tags.append('')
        elif tag == 'span':
            if style['font-variant'] == 'small-caps':
                if self.style_smallcap == False:  # noqa
                    text.append('&')
                    tags.append('&')
                    self.style_smallcap = True
            else:
                if self.in_a_link == False:  # noqa
                    txt = '%'
                    if self.opts.keep_links:
                        txt += self.check_id_tag(attribs)
                        txt += self.check_styles(style)
                    if txt != '%':
                        text.append(txt)
                        tags.append('%')

        if self.opts.keep_links and attribs.has_key('id'):  # noqa
            if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'span', 'table'):
                text.append(self.check_id_tag(attribs))

        # Process the styles for any that we want to keep
        if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'hr', 'a', 'img',
                'span', 'table', 'tr', 'td'):
            if not self.in_a_link:
                text.append(self.check_styles(style))

        # Process tags that contain text.
        if hasattr(elem, 'text') and elem.text:
            txt = elem.text
            if not self.in_pre:
                txt = self.prepare_string_for_textile(self.remove_newlines(txt))
            text.append(txt)
            self.id_no_text = u''

        # Recurse down into tags within the tag we are in.
        for item in elem:
            text += self.dump_text(item, stylizer)

        # Close all open tags.
        tags.reverse()
        for t in tags:
            if t in ('pre', 'ul', 'ol', 'li', 'table'):
                if t == 'pre':
                    self.in_pre = False
                elif t in ('ul', 'ol'):
                    if self.list:
                        self.list.pop()
                    if not self.list:
                        text.append('\n')
            else:
                if t == 'a':
                    self.in_a_link = False
                    t = ''
                text.append(self.id_no_text)
                self.id_no_text = u''
                if t in ('*]', '*'):
                    self.style_bold = False
                elif t in ('_]', '_'):
                    self.style_italic = False
                elif t == '+]':
                    self.style_under = False
                elif t == '-]':
                    self.style_strike = False
                elif t == '&':
                    self.style_smallcap = False
                if t in ('*]', '_]', '+]', '-]', '*', '_'):
                    txt = self.style_embed.pop()
                text.append('%s' % t)

        # Soft scene breaks.
        if 'margin-bottom' in style.cssdict() and style['margin-bottom'] != 'auto':
            ems = min(int(round((float(style.marginBottom) / style.fontSize) - 1)), self.MAX_EM)
            if ems >= 1:
                text.append(u'\n\n\xa0' * ems)

        # Add the text that is outside of the tag.
        if hasattr(elem, 'tail') and elem.tail:
            tail = elem.tail
            if not self.in_pre:
                tail = self.prepare_string_for_textile(self.remove_newlines(tail))
            text.append(tail)

        return text

Example 166

Project: woodwind
Source File: views.py
View license
@flask_login.login_required
def update_micropub_syndicate_to():

    def adapt_expanded(targets):
        """Backcompat support for old-style "syndicate-to-expanded" properties,
        e.g.,
        {
          "id": "twitter::kylewmahan",
          "name": "@kylewmahan",
          "service": "Twitter"
        }
        """
        if targets:
            return [{
                'uid': t.get('id'),
                'name': '{} on {}'.format(t.get('name'), t.get('service')),
            } for t in targets]
        return targets

    endpt = flask_login.current_user.micropub_endpoint
    token = flask_login.current_user.access_token
    if not endpt or not token:
        return
    resp = util.requests_get(endpt, params={
        'q': 'syndicate-to',
    }, headers={
        'Authorization': 'Bearer ' + token,
        'Accept': 'application/json',
    })
    if resp.status_code // 100 != 2:
        flask.current_app.logger.warn(
            'Unexpected response querying micropub endpoint %s: %s',
            resp, resp.text)
        return

    flask.current_app.logger.debug('syndicate-to response: %s %s',
                                   resp, resp.text)

    content_type = resp.headers['content-type']
    if content_type:
        content_type = content_type.split(';', 1)[0]

    try:
        if content_type == 'application/json':
                blob = resp.json()
                syndicate_tos = adapt_expanded(blob.get('syndicate-to-expanded'))
                if not syndicate_tos:
                    syndicate_tos = blob.get('syndicate-to')

        else:  # try to parse query string
            syndicate_tos = pyquerystring.parse(resp.text).get('syndicate-to', [])
            if isinstance(syndicate_tos, list):
                syndicate_tos = list(syndicate_tos)

        flask_login.current_user.set_setting('syndicate-to', syndicate_tos)
        db.session.commit()
    except ValueError as e:
        flask.flash('Could not parse syndicate-to response: {}'.format(e)



@views.route('/deauthorize')
@flask_login.login_required
def deauthorize():
    flask_login.current_user.micropub_endpoint = None
    flask_login.current_user.access_token = None
    db.session.commit()
    return flask.redirect(flask.request.args.get('next')
                          or flask.url_for('.index'))


@login_mgr.user_loader
def load_user(url):
    alt = url.rstrip('/') if url.endswith('/') else url + '/'
    return User.query.filter(
        (User.url == url) | (User.url == alt)).first()


@views.route('/subscribe', methods=['GET', 'POST'])
@flask_login.login_required
def subscribe():
    origin = (flask.request.form.get('origin')
              or flask.request.args.get('origin'))
    if origin:
        type = None
        feed = None
        typed_feed = flask.request.form.get('feed')
        if typed_feed:
            type, feed = typed_feed.split('|', 1)
        else:
            feeds = find_possible_feeds(origin)
            if not feeds:
                flask.flash('No feeds found for: ' + origin)
                return flask.redirect(flask.url_for('.index'))
            if len(feeds) > 1:
                return flask.render_template(
                    'select-feed.jinja2', origin=origin, feeds=feeds)
            feed = feeds[0]['feed']
            type = feeds[0]['type']
        new_feed = add_subscription(origin, feed, type)
        flask.flash('Successfully subscribed to: {}'.format(new_feed.name))
        return flask.redirect(flask.url_for('.index'))

    if flask.request.method == 'POST':
        flask.abort(400)

    return flask.render_template('subscribe.jinja2')


def add_subscription(origin, feed_url, type, tags=None):
    feed = Feed.query.filter_by(feed=feed_url, type=type).first()

    if not feed:
        name = None
        if type == 'html':
            flask.current_app.logger.debug('mf2py parsing %s', feed_url)
            resp = util.requests_get(feed_url)
            feed_text = resp.text if 'charset' in resp.headers.get('content-type', '') else resp.content
            parsed = mf2util.interpret_feed(
                mf2py.parse(doc=feed_text, url=feed_url), feed_url)
            name = parsed.get('name')
        elif type == 'xml':
            flask.current_app.logger.debug('feedparser parsing %s', feed_url)
            parsed = feedparser.parse(feed_url, agent=util.USER_AGENT)
            if parsed.feed:
                name = parsed.feed.get('title')
        else:
            flask.current_app.logger.error('unknown feed type %s', type)
            flask.abort(400)

        if not name:
            p = urllib.parse.urlparse(origin)
            name = p.netloc + p.path
        feed = Feed(name=name[:140], origin=origin, feed=feed_url, type=type)

    if feed:
        db.session.add(feed)

        flask_login.current_user.subscriptions.append(
            Subscription(feed=feed, name=feed.name, tags=tags))

        db.session.commit()
        # go ahead and update the fed
        tasks.q.enqueue(tasks.update_feed, feed.id)
    return feed


def find_possible_feeds(origin):
    # scrape an origin source to find possible alternative feeds
    try:
        resp = util.requests_get(origin)
    except requests.exceptions.RequestException as e:
        flask.flash('Error fetching source {}'.format(repr(e)))
        flask.current_app.logger.warn(
            'Subscribe failed for %s with error %s', origin, repr(e))
        return None

    feeds = []

    xml_feed_types = [
        'application/rss+xml',
        'application/atom+xml',
        'application/rdf+xml',
        'application/xml',
        'text/xml',
    ]
    xml_mime_types = xml_feed_types + [
        'text/xml',
        'text/rss+xml',
        'text/atom+xml',
    ]
    html_feed_types = [
        'text/html',
        'application/xhtml+xml',
    ]

    content_type = resp.headers['content-type']
    content_type = content_type.split(';', 1)[0].strip()
    if content_type in xml_mime_types:
        feeds.append({
            'origin': origin,
            'feed': origin,
            'type': 'xml',
            'title': 'untitled xml feed',
        })

    elif content_type in html_feed_types:
        parsed = mf2py.parse(doc=resp.text, url=origin)
        # if text/html, then parse and look for h-entries
        hfeed = mf2util.interpret_feed(parsed, origin)
        if hfeed.get('entries'):
            ftitle = hfeed.get('name') or 'untitled h-feed'
            feeds.append({
                'origin': origin,
                'feed': resp.url,
                'type': 'html',
                'title': ftitle[:140]
            })

        # look for link="feed"
        for furl in parsed.get('rels', {}).get('feed', []):
            fprops = parsed.get('rel-urls', {}).get(furl, {})
            if not fprops.get('type') or fprops.get('type') in html_feed_types:
                feeds.append({
                    'origin': origin,
                    'feed': furl,
                    'type': 'html',
                    'title': fprops.get('title'),
                })

        # then look for link rel="alternate"
        for link in parsed.get('alternates', []):
            if link.get('type') in xml_feed_types:
                feeds.append({
                    'origin': origin,
                    'feed': link.get('url'),
                    'type': 'xml',
                    'title': link.get('title'),
                })

    return feeds


@views.app_template_filter()
def prettify_url(url):
    parsed = urllib.parse.urlparse(url)
    if parsed.path:
        return parsed.netloc + parsed.path
    return parsed.netloc


@views.app_template_filter()
def domain_for_url(url):
    parsed = urllib.parse.urlparse(url)
    return parsed.netloc


@views.app_template_filter()
def favicon_for_url(url):
    return '//www.google.com/s2/favicons?' + urllib.parse.urlencode({
        'domain': url,
    })


@views.app_template_filter()
def relative_time(dt):
    if dt:
        now = datetime.datetime.utcnow()
        diff = now - dt
        zero = datetime.timedelta(0)

        if diff == zero:
            pretty = 'Right now'
        elif diff > zero:
            years = diff.days // 365
            hours = diff.seconds // 60 // 60
            minutes = diff.seconds // 60

            if years > 1:
                pretty = str(years) + ' years ago'
            elif diff.days == 1:
                pretty = 'A day ago'
            elif diff.days > 1:
                pretty = str(diff.days) + ' days ago'
            elif hours == 1:
                pretty = 'An hour ago'
            elif hours > 1:
                pretty = str(hours) + ' hours ago'
            elif minutes == 1:
                pretty = 'A minute ago'
            elif minutes > 1:
                pretty = str(minutes) + ' minutes ago'
            else:
                pretty = str(diff.seconds) + ' seconds ago'
        else:
            diff = abs(diff)
            years = diff.days // 365
            hours = diff.seconds // 60 // 60
            minutes = diff.seconds // 60

            if years > 1:
                pretty = str(years) + ' years from now'
            elif diff.days == 1:
                pretty = 'A day from now'
            elif diff.days > 1:
                pretty = str(diff.days) + ' days from now'
            elif hours == 1:
                pretty = 'An hour from now'
            elif hours > 1:
                pretty = str(hours) + ' hours from now'
            elif minutes == 1:
                pretty = 'A minute from now'
            elif minutes > 1:
                pretty = str(minutes) + ' minutes from now'
            else:
                pretty = str(diff.seconds) + ' seconds from now'

        return '<time datetime="{}">{}</time>'.format(dt.isoformat(), pretty)


@views.app_template_filter()
def isoformat(dt):
    return dt and dt.isoformat()


@views.app_template_filter()
def add_preview(content):
    """If a post ends with the URL of a known media source (youtube,
    instagram, etc.), add the content inline.
    """
    if not content or any('<' + tag in content for tag in (
            'img', 'iframe', 'embed', 'audio', 'video')):
        # don't add  a preview to a post that already has one
        return content

    # flatten links and strip tags
    flat = content
    flat = re.sub(r'<a [^>]*href="([^"]+)"[^>]*>[^<]*</a>', r'\1', flat)
    flat = re.sub(r'</?\w+[^>]*>', '', flat)
    flat = flat.strip()

    instagram_regex = r'https?://(?:www\.)?instagram.com/p/[\w\-]+/?'
    vimeo_regex = r'https?://(?:www\.)?vimeo.com/(\d+)/?'
    youtube_regex = r'https?://(?:www\.)?youtube.com/watch\?v=([\w\-]+)'
    youtube_short_regex = r'https://youtu.be/([\w\-]+)'
    twitter_regex = r'https?://(?:www\.)?twitter.com/(\w+)/status/(\d+)'

    m = re.search(instagram_regex, flat)
    if m:
        ig_url = m.group(0)
        media_url = urllib.parse.urljoin(ig_url, 'media/?size=l')
        return '{}<a href="{}"><img src="{}" /></a>'.format(
            content, ig_url, media_url)

    m = re.search(vimeo_regex, flat)
    if m:
        # vimeo_url = m.group(0)
        vimeo_id = m.group(1)
        return (
            '{}<iframe src="//player.vimeo.com/video/{}" width="560" '
            'height="315" frameborder="0" webkitallowfullscreen '
            'mozallowfullscreen allowfullscreen></iframe>'
        ).format(content, vimeo_id)

    m = re.search(youtube_regex, flat)
    if not m:
        m = re.search(youtube_short_regex, content)

    if m:
        youtube_id = m.group(1)
        return (
            '{}<iframe width="560" height="315" '
            'src="https://www.youtube.com/embed/{}" frameborder="0" '
            'allowfullscreen></iframe>'
        ).format(content, youtube_id)

    m = re.search(twitter_regex + '$', flat)
    if m:
        tweet_url = m.group()
        return content + (
            '<blockquote class="twitter-tweet" lang="en" data-cards="hidden">'
            '<a href="{}"></a></blockquote>'
        ).format(tweet_url)

    return content


@views.app_template_filter()
def proxy_image(url):
    proxy_url = flask.current_app.config.get('IMAGEPROXY_URL')
    proxy_key = flask.current_app.config.get('IMAGEPROXY_KEY')
    if proxy_url and proxy_key:
        sig = base64.urlsafe_b64encode(
            hmac.new(proxy_key.encode(), url.encode(), hashlib.sha256).digest()
        ).decode()
        return '/'.join((proxy_url.rstrip('/'), 's' + sig, url))

    pilbox_url = flask.current_app.config.get('PILBOX_URL')
    pilbox_key = flask.current_app.config.get('PILBOX_KEY')
    if pilbox_url and pilbox_key:
        query = urllib.parse.urlencode({'url': url, 'op': 'noop'})
        sig = hmac.new(pilbox_key.encode(), query.encode(), hashlib.sha1).hexdigest()
        query += '&sig=' + sig
        return pilbox_url + '?' + query

    camo_url = flask.current_app.config.get('CAMO_URL')
    camo_key = flask.current_app.config.get('CAMO_KEY')
    if camo_url and camo_key:
        digest = hmac.new(camo_key.encode(), url.encode(), hashlib.sha1).hexdigest()
        return (urllib.parse.urljoin(camo_url, digest)
                + '?url=' + urllib.parse.quote_plus(url))
    return url


@views.app_template_filter()
def proxy_all(content):
    def repl(m):
        attrs = m.group(1)
        url = m.group(2)
        url = url.replace('&amp;', '&')
        return '<img{} src="{}"'.format(attrs, proxy_image(url))
    if content:
        return IMAGE_TAG_RE.sub(repl, content)


@views.app_template_global()
def url_for_other_page(page):
    """http://flask.pocoo.org/snippets/44/#URL+Generation+Helper
    """
    args = flask.request.view_args.copy()
    args.update(flask.request.args)
    args['page'] = page

    return flask.url_for(flask.request.endpoint, **args)


def dedupe_copies(entries):
    all_copies = set()
    for entry in entries:
        syndurls = entry.get_property('syndication')
        if syndurls:
            copies = [e for e in entries if e.permalink in syndurls]
            entry._syndicated_copies = copies
            all_copies.update(copies)
    return [e for e in entries if e not in all_copies]


def font_awesome_class_for_service(service):
    service = service.lower()
    if service == 'facebook':
        return 'fa fa-facebook'
    if service == 'twitter':
        return 'fa fa-twitter'
    if service == 'instagram':
        return 'fa fa-instagram'
    if service == 'flickr':
        return 'fa fa-flickr'
    if service == 'googleplus' or service == 'g+' or service == 'google plus' or service == 'google+':
        return 'fa fa-google-plus'
    if service == 'hacker news' or service == 'hackernews':
        return 'fa fa-hacker-news'
    if service == 'indienews':
        return 'fa fa-newspaper-o'
    if service == 'linkedin':
        return 'fa fa-linkedin'
    if service == 'foursquare' or service == 'swarm':
        return 'fa fa-foursquare'
    return 'fa fa-send'


@views.app_template_filter('render_syndication_target_id')
def render_syndication_target_id(target):
    if isinstance(target, dict):
        id = target.get('uid') or target.get('id')
    else:
        id = target
    return util.html_escape(id)


@views.app_template_filter('render_syndication_target')
def render_syndication_target(target):
    if isinstance(target, dict):
        full_name = target.get('name')
        return util.html_escape(full_name)

    return util.html_escape(prettify_url(target))


@views.app_template_test('syndicated_to')
def is_syndicated_to(entry, target):
    def same_domain(u1, u2):
        return domain_for_url(u1) == domain_for_url(u2)

    if isinstance(target, dict):
        return False  # TODO

    return same_domain(entry.permalink, target) or any(
        same_domain(syndurl, target)
        for syndurl in entry.get_property('syndication', []))

Example 167

Project: woodwind
Source File: views.py
View license
@flask_login.login_required
def update_micropub_syndicate_to():

    def adapt_expanded(targets):
        """Backcompat support for old-style "syndicate-to-expanded" properties,
        e.g.,
        {
          "id": "twitter::kylewmahan",
          "name": "@kylewmahan",
          "service": "Twitter"
        }
        """
        if targets:
            return [{
                'uid': t.get('id'),
                'name': '{} on {}'.format(t.get('name'), t.get('service')),
            } for t in targets]
        return targets

    endpt = flask_login.current_user.micropub_endpoint
    token = flask_login.current_user.access_token
    if not endpt or not token:
        return
    resp = util.requests_get(endpt, params={
        'q': 'syndicate-to',
    }, headers={
        'Authorization': 'Bearer ' + token,
        'Accept': 'application/json',
    })
    if resp.status_code // 100 != 2:
        flask.current_app.logger.warn(
            'Unexpected response querying micropub endpoint %s: %s',
            resp, resp.text)
        return

    flask.current_app.logger.debug('syndicate-to response: %s %s',
                                   resp, resp.text)

    content_type = resp.headers['content-type']
    if content_type:
        content_type = content_type.split(';', 1)[0]

    try:
        if content_type == 'application/json':
                blob = resp.json()
                syndicate_tos = adapt_expanded(blob.get('syndicate-to-expanded'))
                if not syndicate_tos:
                    syndicate_tos = blob.get('syndicate-to')

        else:  # try to parse query string
            syndicate_tos = pyquerystring.parse(resp.text).get('syndicate-to', [])
            if isinstance(syndicate_tos, list):
                syndicate_tos = list(syndicate_tos)

        flask_login.current_user.set_setting('syndicate-to', syndicate_tos)
        db.session.commit()
    except ValueError as e:
        flask.flash('Could not parse syndicate-to response: {}'.format(e)



@views.route('/deauthorize')
@flask_login.login_required
def deauthorize():
    flask_login.current_user.micropub_endpoint = None
    flask_login.current_user.access_token = None
    db.session.commit()
    return flask.redirect(flask.request.args.get('next')
                          or flask.url_for('.index'))


@login_mgr.user_loader
def load_user(url):
    alt = url.rstrip('/') if url.endswith('/') else url + '/'
    return User.query.filter(
        (User.url == url) | (User.url == alt)).first()


@views.route('/subscribe', methods=['GET', 'POST'])
@flask_login.login_required
def subscribe():
    origin = (flask.request.form.get('origin')
              or flask.request.args.get('origin'))
    if origin:
        type = None
        feed = None
        typed_feed = flask.request.form.get('feed')
        if typed_feed:
            type, feed = typed_feed.split('|', 1)
        else:
            feeds = find_possible_feeds(origin)
            if not feeds:
                flask.flash('No feeds found for: ' + origin)
                return flask.redirect(flask.url_for('.index'))
            if len(feeds) > 1:
                return flask.render_template(
                    'select-feed.jinja2', origin=origin, feeds=feeds)
            feed = feeds[0]['feed']
            type = feeds[0]['type']
        new_feed = add_subscription(origin, feed, type)
        flask.flash('Successfully subscribed to: {}'.format(new_feed.name))
        return flask.redirect(flask.url_for('.index'))

    if flask.request.method == 'POST':
        flask.abort(400)

    return flask.render_template('subscribe.jinja2')


def add_subscription(origin, feed_url, type, tags=None):
    feed = Feed.query.filter_by(feed=feed_url, type=type).first()

    if not feed:
        name = None
        if type == 'html':
            flask.current_app.logger.debug('mf2py parsing %s', feed_url)
            resp = util.requests_get(feed_url)
            feed_text = resp.text if 'charset' in resp.headers.get('content-type', '') else resp.content
            parsed = mf2util.interpret_feed(
                mf2py.parse(doc=feed_text, url=feed_url), feed_url)
            name = parsed.get('name')
        elif type == 'xml':
            flask.current_app.logger.debug('feedparser parsing %s', feed_url)
            parsed = feedparser.parse(feed_url, agent=util.USER_AGENT)
            if parsed.feed:
                name = parsed.feed.get('title')
        else:
            flask.current_app.logger.error('unknown feed type %s', type)
            flask.abort(400)

        if not name:
            p = urllib.parse.urlparse(origin)
            name = p.netloc + p.path
        feed = Feed(name=name[:140], origin=origin, feed=feed_url, type=type)

    if feed:
        db.session.add(feed)

        flask_login.current_user.subscriptions.append(
            Subscription(feed=feed, name=feed.name, tags=tags))

        db.session.commit()
        # go ahead and update the fed
        tasks.q.enqueue(tasks.update_feed, feed.id)
    return feed


def find_possible_feeds(origin):
    # scrape an origin source to find possible alternative feeds
    try:
        resp = util.requests_get(origin)
    except requests.exceptions.RequestException as e:
        flask.flash('Error fetching source {}'.format(repr(e)))
        flask.current_app.logger.warn(
            'Subscribe failed for %s with error %s', origin, repr(e))
        return None

    feeds = []

    xml_feed_types = [
        'application/rss+xml',
        'application/atom+xml',
        'application/rdf+xml',
        'application/xml',
        'text/xml',
    ]
    xml_mime_types = xml_feed_types + [
        'text/xml',
        'text/rss+xml',
        'text/atom+xml',
    ]
    html_feed_types = [
        'text/html',
        'application/xhtml+xml',
    ]

    content_type = resp.headers['content-type']
    content_type = content_type.split(';', 1)[0].strip()
    if content_type in xml_mime_types:
        feeds.append({
            'origin': origin,
            'feed': origin,
            'type': 'xml',
            'title': 'untitled xml feed',
        })

    elif content_type in html_feed_types:
        parsed = mf2py.parse(doc=resp.text, url=origin)
        # if text/html, then parse and look for h-entries
        hfeed = mf2util.interpret_feed(parsed, origin)
        if hfeed.get('entries'):
            ftitle = hfeed.get('name') or 'untitled h-feed'
            feeds.append({
                'origin': origin,
                'feed': resp.url,
                'type': 'html',
                'title': ftitle[:140]
            })

        # look for link="feed"
        for furl in parsed.get('rels', {}).get('feed', []):
            fprops = parsed.get('rel-urls', {}).get(furl, {})
            if not fprops.get('type') or fprops.get('type') in html_feed_types:
                feeds.append({
                    'origin': origin,
                    'feed': furl,
                    'type': 'html',
                    'title': fprops.get('title'),
                })

        # then look for link rel="alternate"
        for link in parsed.get('alternates', []):
            if link.get('type') in xml_feed_types:
                feeds.append({
                    'origin': origin,
                    'feed': link.get('url'),
                    'type': 'xml',
                    'title': link.get('title'),
                })

    return feeds


@views.app_template_filter()
def prettify_url(url):
    parsed = urllib.parse.urlparse(url)
    if parsed.path:
        return parsed.netloc + parsed.path
    return parsed.netloc


@views.app_template_filter()
def domain_for_url(url):
    parsed = urllib.parse.urlparse(url)
    return parsed.netloc


@views.app_template_filter()
def favicon_for_url(url):
    return '//www.google.com/s2/favicons?' + urllib.parse.urlencode({
        'domain': url,
    })


@views.app_template_filter()
def relative_time(dt):
    if dt:
        now = datetime.datetime.utcnow()
        diff = now - dt
        zero = datetime.timedelta(0)

        if diff == zero:
            pretty = 'Right now'
        elif diff > zero:
            years = diff.days // 365
            hours = diff.seconds // 60 // 60
            minutes = diff.seconds // 60

            if years > 1:
                pretty = str(years) + ' years ago'
            elif diff.days == 1:
                pretty = 'A day ago'
            elif diff.days > 1:
                pretty = str(diff.days) + ' days ago'
            elif hours == 1:
                pretty = 'An hour ago'
            elif hours > 1:
                pretty = str(hours) + ' hours ago'
            elif minutes == 1:
                pretty = 'A minute ago'
            elif minutes > 1:
                pretty = str(minutes) + ' minutes ago'
            else:
                pretty = str(diff.seconds) + ' seconds ago'
        else:
            diff = abs(diff)
            years = diff.days // 365
            hours = diff.seconds // 60 // 60
            minutes = diff.seconds // 60

            if years > 1:
                pretty = str(years) + ' years from now'
            elif diff.days == 1:
                pretty = 'A day from now'
            elif diff.days > 1:
                pretty = str(diff.days) + ' days from now'
            elif hours == 1:
                pretty = 'An hour from now'
            elif hours > 1:
                pretty = str(hours) + ' hours from now'
            elif minutes == 1:
                pretty = 'A minute from now'
            elif minutes > 1:
                pretty = str(minutes) + ' minutes from now'
            else:
                pretty = str(diff.seconds) + ' seconds from now'

        return '<time datetime="{}">{}</time>'.format(dt.isoformat(), pretty)


@views.app_template_filter()
def isoformat(dt):
    return dt and dt.isoformat()


@views.app_template_filter()
def add_preview(content):
    """If a post ends with the URL of a known media source (youtube,
    instagram, etc.), add the content inline.
    """
    if not content or any('<' + tag in content for tag in (
            'img', 'iframe', 'embed', 'audio', 'video')):
        # don't add  a preview to a post that already has one
        return content

    # flatten links and strip tags
    flat = content
    flat = re.sub(r'<a [^>]*href="([^"]+)"[^>]*>[^<]*</a>', r'\1', flat)
    flat = re.sub(r'</?\w+[^>]*>', '', flat)
    flat = flat.strip()

    instagram_regex = r'https?://(?:www\.)?instagram.com/p/[\w\-]+/?'
    vimeo_regex = r'https?://(?:www\.)?vimeo.com/(\d+)/?'
    youtube_regex = r'https?://(?:www\.)?youtube.com/watch\?v=([\w\-]+)'
    youtube_short_regex = r'https://youtu.be/([\w\-]+)'
    twitter_regex = r'https?://(?:www\.)?twitter.com/(\w+)/status/(\d+)'

    m = re.search(instagram_regex, flat)
    if m:
        ig_url = m.group(0)
        media_url = urllib.parse.urljoin(ig_url, 'media/?size=l')
        return '{}<a href="{}"><img src="{}" /></a>'.format(
            content, ig_url, media_url)

    m = re.search(vimeo_regex, flat)
    if m:
        # vimeo_url = m.group(0)
        vimeo_id = m.group(1)
        return (
            '{}<iframe src="//player.vimeo.com/video/{}" width="560" '
            'height="315" frameborder="0" webkitallowfullscreen '
            'mozallowfullscreen allowfullscreen></iframe>'
        ).format(content, vimeo_id)

    m = re.search(youtube_regex, flat)
    if not m:
        m = re.search(youtube_short_regex, content)

    if m:
        youtube_id = m.group(1)
        return (
            '{}<iframe width="560" height="315" '
            'src="https://www.youtube.com/embed/{}" frameborder="0" '
            'allowfullscreen></iframe>'
        ).format(content, youtube_id)

    m = re.search(twitter_regex + '$', flat)
    if m:
        tweet_url = m.group()
        return content + (
            '<blockquote class="twitter-tweet" lang="en" data-cards="hidden">'
            '<a href="{}"></a></blockquote>'
        ).format(tweet_url)

    return content


@views.app_template_filter()
def proxy_image(url):
    proxy_url = flask.current_app.config.get('IMAGEPROXY_URL')
    proxy_key = flask.current_app.config.get('IMAGEPROXY_KEY')
    if proxy_url and proxy_key:
        sig = base64.urlsafe_b64encode(
            hmac.new(proxy_key.encode(), url.encode(), hashlib.sha256).digest()
        ).decode()
        return '/'.join((proxy_url.rstrip('/'), 's' + sig, url))

    pilbox_url = flask.current_app.config.get('PILBOX_URL')
    pilbox_key = flask.current_app.config.get('PILBOX_KEY')
    if pilbox_url and pilbox_key:
        query = urllib.parse.urlencode({'url': url, 'op': 'noop'})
        sig = hmac.new(pilbox_key.encode(), query.encode(), hashlib.sha1).hexdigest()
        query += '&sig=' + sig
        return pilbox_url + '?' + query

    camo_url = flask.current_app.config.get('CAMO_URL')
    camo_key = flask.current_app.config.get('CAMO_KEY')
    if camo_url and camo_key:
        digest = hmac.new(camo_key.encode(), url.encode(), hashlib.sha1).hexdigest()
        return (urllib.parse.urljoin(camo_url, digest)
                + '?url=' + urllib.parse.quote_plus(url))
    return url


@views.app_template_filter()
def proxy_all(content):
    def repl(m):
        attrs = m.group(1)
        url = m.group(2)
        url = url.replace('&amp;', '&')
        return '<img{} src="{}"'.format(attrs, proxy_image(url))
    if content:
        return IMAGE_TAG_RE.sub(repl, content)


@views.app_template_global()
def url_for_other_page(page):
    """http://flask.pocoo.org/snippets/44/#URL+Generation+Helper
    """
    args = flask.request.view_args.copy()
    args.update(flask.request.args)
    args['page'] = page

    return flask.url_for(flask.request.endpoint, **args)


def dedupe_copies(entries):
    all_copies = set()
    for entry in entries:
        syndurls = entry.get_property('syndication')
        if syndurls:
            copies = [e for e in entries if e.permalink in syndurls]
            entry._syndicated_copies = copies
            all_copies.update(copies)
    return [e for e in entries if e not in all_copies]


def font_awesome_class_for_service(service):
    service = service.lower()
    if service == 'facebook':
        return 'fa fa-facebook'
    if service == 'twitter':
        return 'fa fa-twitter'
    if service == 'instagram':
        return 'fa fa-instagram'
    if service == 'flickr':
        return 'fa fa-flickr'
    if service == 'googleplus' or service == 'g+' or service == 'google plus' or service == 'google+':
        return 'fa fa-google-plus'
    if service == 'hacker news' or service == 'hackernews':
        return 'fa fa-hacker-news'
    if service == 'indienews':
        return 'fa fa-newspaper-o'
    if service == 'linkedin':
        return 'fa fa-linkedin'
    if service == 'foursquare' or service == 'swarm':
        return 'fa fa-foursquare'
    return 'fa fa-send'


@views.app_template_filter('render_syndication_target_id')
def render_syndication_target_id(target):
    if isinstance(target, dict):
        id = target.get('uid') or target.get('id')
    else:
        id = target
    return util.html_escape(id)


@views.app_template_filter('render_syndication_target')
def render_syndication_target(target):
    if isinstance(target, dict):
        full_name = target.get('name')
        return util.html_escape(full_name)

    return util.html_escape(prettify_url(target))


@views.app_template_test('syndicated_to')
def is_syndicated_to(entry, target):
    def same_domain(u1, u2):
        return domain_for_url(u1) == domain_for_url(u2)

    if isinstance(target, dict):
        return False  # TODO

    return same_domain(entry.permalink, target) or any(
        same_domain(syndurl, target)
        for syndurl in entry.get_property('syndication', []))

Example 168

View license
def parse(input_filename, output_filename):
    "Feed it a file, and it'll output a fixed one"

    # State storage
    if input_filename == "-":
        num_lines = -1
    else:
        num_lines = int(subprocess.check_output(["wc", "-l", input_filename]).strip().split()[0])
    tables = {}
    current_table = None
    creation_lines = []
    enum_types = []
    foreign_key_lines = []
    fulltext_key_lines = []
    sequence_lines = []
    cast_lines = []
    num_inserts = 0
    started = time.time()

    # Open output file and write header. Logging file handle will be stdout
    # unless we're writing output to stdout, in which case NO PROGRESS FOR YOU.
    if output_filename == "-":
        output = sys.stdout
        logging = open(os.devnull, "w")
    else:
        output = open(output_filename, "w")
        logging = sys.stdout

    if input_filename == "-":
        input_fh = sys.stdin
    else:
        input_fh = open(input_filename)


    output.write("-- Converted by db_converter\n")
    output.write("START TRANSACTION;\n")
    output.write("SET standard_conforming_strings=off;\n")
    output.write("SET escape_string_warning=off;\n")
    output.write("SET CONSTRAINTS ALL DEFERRED;\n\n")

    for i, line in enumerate(input_fh):
        time_taken = time.time() - started
        percentage_done = (i+1) / float(num_lines)
        secs_left = (time_taken / percentage_done) - time_taken
        logging.write("\rLine %i (of %s: %.2f%%) [%s tables] [%s inserts] [ETA: %i min %i sec]" % (
            i + 1,
            num_lines,
            ((i+1)/float(num_lines))*100,
            len(tables),
            num_inserts,
            secs_left // 60,
            secs_left % 60,
        ))
        logging.flush()
        line = line.decode("utf8").strip().replace(r"\\", "WUBWUBREALSLASHWUB").replace(r"\'", "''").replace("WUBWUBREALSLASHWUB", r"\\")
        # Ignore comment lines
        if line.startswith("--") or line.startswith("/*") or line.startswith("LOCK TABLES") or line.startswith("DROP TABLE") or line.startswith("UNLOCK TABLES") or not line:
            continue

        # Outside of anything handling
        if current_table is None:
            # Start of a table creation statement?
            if line.startswith("CREATE TABLE"):
                current_table = line.split('"')[1]
                tables[current_table] = {"columns": []}
                creation_lines = []
            # Inserting data into a table?
            elif line.startswith("INSERT INTO"):
                output.write(line.encode("utf8").replace("'0000-00-00 00:00:00'", "NULL") + "\n")
                num_inserts += 1
            # ???
            else:
                print "\n ! Unknown line in main body: %s" % line

        # Inside-create-statement handling
        else:
            # Is it a column?
            if line.startswith('"'):
                useless, name, definition = line.strip(",").split('"',2)
                try:
                    type, extra = definition.strip().split(" ", 1)

                    # This must be a tricky enum
                    if ')' in extra:
                        type, extra = definition.strip().split(")")

                except ValueError:
                    type = definition.strip()
                    extra = ""
                extra = re.sub("CHARACTER SET [\w\d]+\s*", "", extra.replace("unsigned", ""))
                extra = re.sub("COLLATE [\w\d]+\s*", "", extra.replace("unsigned", ""))

                # See if it needs type conversion
                final_type = None
                set_sequence = None
                if type.startswith("tinyint("):
                    type = "int4"
                    set_sequence = True
                    final_type = "boolean"
                elif type.startswith("int("):
                    type = "integer"
                    set_sequence = True
                elif type.startswith("bigint("):
                    type = "bigint"
                    set_sequence = True
                elif type == "longtext":
                    type = "text"
                elif type == "mediumtext":
                    type = "text"
                elif type == "tinytext":
                    type = "text"
                elif type.startswith("varchar("):
                    size = int(type.split("(")[1].rstrip(")"))
                    type = "varchar(%s)" % (size * 2)
                elif type.startswith("smallint("):
                    type = "int2"
                    set_sequence = True
                elif type == "datetime":
                    type = "timestamp with time zone"
                elif type == "double":
                    type = "double precision"
                elif type.endswith("blob"):
                    type = "bytea"
                elif type.startswith("enum(") or type.startswith("set("):

                    types_str = type.split("(")[1].rstrip(")").rstrip('"')
                    types_arr = [type_str.strip('\'') for type_str in types_str.split(",")]

                    # Considered using values to make a name, but its dodgy
                    # enum_name = '_'.join(types_arr)
                    enum_name = "{0}_{1}".format(current_table, name)

                    if enum_name not in enum_types:
                        output.write("CREATE TYPE {0} AS ENUM ({1}); \n".format(enum_name, types_str));
                        enum_types.append(enum_name)

                    type = enum_name

                if final_type:
                    cast_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"%s\" DROP DEFAULT, ALTER COLUMN \"%s\" TYPE %s USING CAST(\"%s\" as %s)" % (current_table, name, name, final_type, name, final_type))
                # ID fields need sequences [if they are integers?]
                if name == "id" and set_sequence is True:
                    sequence_lines.append("CREATE SEQUENCE %s_id_seq" % (current_table))
                    sequence_lines.append("SELECT setval('%s_id_seq', max(id)) FROM %s" % (current_table, current_table))
                    sequence_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"id\" SET DEFAULT nextval('%s_id_seq')" % (current_table, current_table))
                # Record it
                creation_lines.append('"%s" %s %s' % (name, type, extra))
                tables[current_table]['columns'].append((name, type, extra))
            # Is it a constraint or something?
            elif line.startswith("PRIMARY KEY"):
                creation_lines.append(line.rstrip(","))
            elif line.startswith("CONSTRAINT"):
                foreign_key_lines.append("ALTER TABLE \"%s\" ADD CONSTRAINT %s DEFERRABLE INITIALLY DEFERRED" % (current_table, line.split("CONSTRAINT")[1].strip().rstrip(",")))
                foreign_key_lines.append("CREATE INDEX ON \"%s\" %s" % (current_table, line.split("FOREIGN KEY")[1].split("REFERENCES")[0].strip().rstrip(",")))
            elif line.startswith("UNIQUE KEY"):
                creation_lines.append("UNIQUE (%s)" % line.split("(")[1].split(")")[0])
            elif line.startswith("FULLTEXT KEY"):

                fulltext_keys = " || ' ' || ".join( line.split('(')[-1].split(')')[0].replace('"', '').split(',') )
                fulltext_key_lines.append("CREATE INDEX ON %s USING gin(to_tsvector('english', %s))" % (current_table, fulltext_keys))

            elif line.startswith("KEY"):
                pass
            # Is it the end of the table?
            elif line == ");":
                output.write("CREATE TABLE \"%s\" (\n" % current_table)
                for i, line in enumerate(creation_lines):
                    output.write("    %s%s\n" % (line, "," if i != (len(creation_lines) - 1) else ""))
                output.write(');\n\n')
                current_table = None
            # ???
            else:
                print "\n ! Unknown line inside table creation: %s" % line


    # Finish file
    output.write("\n-- Post-data save --\n")
    output.write("COMMIT;\n")
    output.write("START TRANSACTION;\n")

    # Write typecasts out
    output.write("\n-- Typecasts --\n")
    for line in cast_lines:
        output.write("%s;\n" % line)

    # Write FK constraints out
    output.write("\n-- Foreign keys --\n")
    for line in foreign_key_lines:
        output.write("%s;\n" % line)

    # Write sequences out
    output.write("\n-- Sequences --\n")
    for line in sequence_lines:
        output.write("%s;\n" % line)

    # Write full-text indexkeyses out
    output.write("\n-- Full Text keys --\n")
    for line in fulltext_key_lines:
        output.write("%s;\n" % line)

    # Finish file
    output.write("\n")
    output.write("COMMIT;\n")
    print ""

Example 169

Project: credmap
Source File: website.py
View license
    def perform_login(self, credentials, cookie_handler):
        """
        Parses CSRF token if available and performs login request.
        """

        if self.cookies:
            self.add_cookies(cookie_handler, self.cookies)

        if self.csrf_token_name:
            self.url = self.csrf_url
            csrf_response = self.get_page()

            if not csrf_response:
                if(self.invalid_http_status and self.response_status and
                   int(self.invalid_http_status["value"]) == int(
                       self.response_status)):
                    if self.verbose:
                        print("%s %s\n" %
                              (INFO, self.invalid_http_status["msg"] if "msg"
                               in self.invalid_http_status else
                               "Your IP may have been blocked..."))
                elif self.verbose:
                    print("%s problem receiving HTTP response "
                          "while fetching token!\n" % ERROR)
                return

            if self.csrf_regex or self.csrf_setcookie:
                match = search(self.csrf_regex if self.csrf_regex else
                               self.csrf_setcookie, csrf_response if
                               self.csrf_regex else str(cookie_handler), I)
                if match:
                    self.csrf_token = (match.group("token") if "token" in
                                       match.groupdict() else match.group(1))
                else:
                    self.status = {"status": 0, "msg": "No token"}
            else:
                start = csrf_response.find(self.csrf_start)
                if start == -1 and self.verbose:  # lvl 1 verbose
                    self.status = {"status": 0, "msg": "No token"}
                end = csrf_response.find(self.csrf_end,
                                         start+len(self.csrf_start))
                self.csrf_token = csrf_response[start+len(self.csrf_start):end]

            if self.status["msg"] == "No token" or not self.csrf_token:
                if self.verbose:
                    print "%s CSRF token not found. Skipping site...\n" % WARN
                return

            if self.verbose:
                print("%s Authentication Token: \"%s\"" %
                      (INFO, color(self.csrf_token)))

        def replace_param(string, param, value, param_format=None):
            """
            Replace data in parameters with given string.
            Parameter format can be json or normal POST data.
            """

            if param_format == "json":
                return sub(r"(?P<json_replacement>\"%s\"\s*:\s*)\"\s*\"" %
                           escape(str(param)), "\\1\"%s\"" % value, string)
            elif param_format == "header":
                return sub(r"%s=[^\\n]*" % escape(str(param)), r"%s=%s" %
                           (str(param).encode('string-escape'),
                            str(value).encode('string-escape')), string)
            else:
                return sub(r"%s=[^&]*" % escape(str(param)), r"%s=%s" %
                           (str(param).encode('string-escape'),
                            str(value).encode('string-escape')), string)

        if self.multiple_params:
            multiple_params_response = ""
            if(self.csrf_token_name and
               self.csrf_url == self.multiple_params_url):
                multiple_params_response = csrf_response
            else:
                self.url = self.multiple_params_url
                multiple_params_response = self.get_page()

            if(self.invalid_http_status and self.response_status and
               int(self.invalid_http_status["value"]) == int(
                   self.response_status) and self.verbose):
                print("%s %s\n" % (INFO, self.invalid_http_status["msg"]
                                   if "msg" in self.invalid_http_status else
                                   "Your IP may have been blocked..."))
                return

            if not multiple_params_response:
                print("%s problem receiving HTTP response while fetching "
                      "params! Skipping site...\n" % ERROR)
                return

            for _ in self.multiple_params:
                regex = (_["regex"] if "regex" in _ else
                         r"<\w+[^>]*(value\s*=\s*\"[^\"]*\"|name\s*=\s*"
                         r"\"?{0}(?:\"|\s))[^>]*(value\s*=\s*\"[^\"]*\""
                         r"|name\s*=\s*\"?{0}(?:\"|\s))[^>]*>"
                         .format(escape(_["value"])))
                match = search(regex, multiple_params_response)

                if not match or "value" not in _:
                    if self.verbose:
                        print("%s no match for parameter \"%s\"! "
                              "Skipping site...\n" %
                              (WARN, color(_["value"])))
                        self.status = {"status": 0, "msg": "No token"}
                    return

                if "regex" in _:
                    value = (match.group("value")
                             if "value" in match.groupdict() else
                             match.group(1))
                elif "value" in _:
                    for attrib in match.groups():
                        attrib = str(attrib).strip().split("=", 1)
                        if attrib[0] == "value":
                            value = attrib[1].strip("\"")

                if "type" not in _:
                    _["type"] = "data"

                if _["type"] == "data" and self.data:
                    self.data = replace_param(self.data, _["value"], value)
                elif _["type"] == "header":
                    self.headers = replace_param(self.headers, _["value"],
                                                 value, "header")
                elif _["type"] == "cookie":
                    self.add_cookies(cookie_handler, "%s=%s;" % (_["value"],
                                                                 value))
                else:
                    pass  # NEED TO REPLACE GET PARAMS

        if(credentials["email"] and
           self.username_or_email in("email", "both")):
            login = credentials["email"]
        elif(credentials["email"] and self.email_exception and
             self.username_or_email == "username" and
             search(self.email_exception, credentials["email"])):
            login = credentials["email"]
        else:
            login = credentials["username"]

        # need to implement support for GET logins lulz

        if self.time_parameter:
            if "type" not in self.time_parameter:
                self.time_parameter["type"] = "epoch"

            if self.time_parameter["type"] == "epoch":
                if self.data:
                    self.data = replace_param(self.data,
                                              self.time_parameter["value"],
                                              time())

        if self.data:
            self.data = replace_param(self.data, self.login_parameter,
                                      login, self.login_parameter_type)
            self.data = replace_param(self.data, self.password_parameter,
                                      credentials["password"],
                                      self.login_parameter_type)

        # need to be able to specify where tokens can be replaced
        if self.csrf_token:
            self.csrf_token = quote(self.csrf_token)
            if self.data:
                self.data = replace_param(self.data,
                                          self.csrf_token_name,
                                          self.csrf_token,
                                          self.login_parameter_type)
            if self.headers:
                self.headers = replace_param(self.headers,
                                             self.csrf_token_name,
                                             self.csrf_token, "header")
            if self.cookies:
                self.cookies = replace_param(self.cookies,
                                             self.csrf_token_name,
                                             self.csrf_token)
                self.add_cookies(cookie_handler, self.cookies)

        self.url = self.login_url
        login_response = self.get_page(data=True if self.data else False,
                                       header=True if self.headers else False)

        if not login_response:
            if self.verbose:
                print("%s no response received! Skipping site...\n" % WARN)
            return False

        if self.login_redirect:
            if self.login_redirect_type == "regex":
                self.url = search(self.login_redirect, login_response)
                self.url = (self.url.group("URL")
                            if "URL" in self.url.groupdict()
                            else self.url.group(1))
            else:
                self.url = self.login_redirect

            self.url = self.url.replace("\\", "")

            login_response = self.get_page(data=True if self.data else False,
                                           header=True if self.headers
                                           else False)

        if not login_response:
            if self.verbose:
                print("%s no response received during login redirect! "
                      "Skipping site...\n" % WARN)
            return False

        # The code for these IF checks need to be cleaned up
        # If invalid credentials http status code is returned
        elif (self.invalid_http_status and self.response_status and
              str(self.invalid_http_status["value"]) ==
              str(self.response_status)):
            if("msg" in self.invalid_http_status or not
               login_response.strip("[]")):
                if self.verbose:
                    print("%s %s\n" %
                          (INFO, self.invalid_http_status["msg"] if "msg"
                           in self.invalid_http_status else
                           "The provided credentials are incorrect "
                           "or the account doesn't exist.\n"))
                return False
        # If captcha flag is set and found in login response
        if self.captcha_flag and self.captcha_flag in login_response:
            if self.verbose:
                print "%s captcha detected! Skipping site...\n" % WARN
            return False
        # If custom search is set and found in response
        elif self.custom_search and search(self.custom_search['regex'],
                                           login_response):
            if self.verbose:
                print "%s %s\n" % (INFO, self.custom_search["value"])
            return False
        # Valid password string in response
        elif self.valid_password and self.valid_password in login_response:
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header type REGEX
        elif (self.valid_response_header and
              self.valid_response_header_type == "regex" and
              search(self.valid_response_header,
                     str(self.response_headers))):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header for cookies type REGEX
        elif (self.valid_response_header and
              self.valid_response_header_type == "regex" and
              search(self.valid_response_header, str(cookie_handler))):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header type normal
        elif (self.valid_response_header and self.valid_response_header
              in str(self.response_headers)):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header for cookies type normal
        elif (self.valid_response_header and self.valid_response_header
              in str(cookie_handler)):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Custom message when specified invalid header is detected
        elif (self.custom_response_header and
              self.custom_response_header["value"] in
              str(self.response_headers)):
            if self.verbose:
                print "%s %s" % (INFO, self.custom_response_header["msg"])
            return False
        # Invalid account string found in login response
        elif self.invalid_account and self.invalid_account in login_response:
            if self.verbose:
                print("%s The provided account doesn't exist on this site.\n"
                      % INFO)
            return False
        # User exists, but account isn't activate.
        elif self.inactive_account and self.inactive_account in login_response:
            if self.verbose:
                print("%s The user exists, but the account isn't activate.\n"
                      % INFO)
            return False
        # User exists, but invalid password string in login response
        elif (self.invalid_password and self.invalid_account and
              self.invalid_password in login_response):
            if self.verbose:
                print("%s The user exists, but the password is incorrect.\n"
                      % INFO)
            return False
        # Invalid password string in login response
        elif (self.invalid_password and not self.invalid_account and
              self.invalid_password in login_response):
            if self.verbose:
                print("%s The provided credentials are incorrect "
                      "or the account doesn't exist.\n" % INFO)
            return False
        # Unhandled case
        else:
            if self.verbose:
                print "%s Unable to login! Skipping site...\n" % WARN
            return False

Example 170

Project: credmap
Source File: website.py
View license
    def perform_login(self, credentials, cookie_handler):
        """
        Parses CSRF token if available and performs login request.
        """

        if self.cookies:
            self.add_cookies(cookie_handler, self.cookies)

        if self.csrf_token_name:
            self.url = self.csrf_url
            csrf_response = self.get_page()

            if not csrf_response:
                if(self.invalid_http_status and self.response_status and
                   int(self.invalid_http_status["value"]) == int(
                       self.response_status)):
                    if self.verbose:
                        print("%s %s\n" %
                              (INFO, self.invalid_http_status["msg"] if "msg"
                               in self.invalid_http_status else
                               "Your IP may have been blocked..."))
                elif self.verbose:
                    print("%s problem receiving HTTP response "
                          "while fetching token!\n" % ERROR)
                return

            if self.csrf_regex or self.csrf_setcookie:
                match = search(self.csrf_regex if self.csrf_regex else
                               self.csrf_setcookie, csrf_response if
                               self.csrf_regex else str(cookie_handler), I)
                if match:
                    self.csrf_token = (match.group("token") if "token" in
                                       match.groupdict() else match.group(1))
                else:
                    self.status = {"status": 0, "msg": "No token"}
            else:
                start = csrf_response.find(self.csrf_start)
                if start == -1 and self.verbose:  # lvl 1 verbose
                    self.status = {"status": 0, "msg": "No token"}
                end = csrf_response.find(self.csrf_end,
                                         start+len(self.csrf_start))
                self.csrf_token = csrf_response[start+len(self.csrf_start):end]

            if self.status["msg"] == "No token" or not self.csrf_token:
                if self.verbose:
                    print "%s CSRF token not found. Skipping site...\n" % WARN
                return

            if self.verbose:
                print("%s Authentication Token: \"%s\"" %
                      (INFO, color(self.csrf_token)))

        def replace_param(string, param, value, param_format=None):
            """
            Replace data in parameters with given string.
            Parameter format can be json or normal POST data.
            """

            if param_format == "json":
                return sub(r"(?P<json_replacement>\"%s\"\s*:\s*)\"\s*\"" %
                           escape(str(param)), "\\1\"%s\"" % value, string)
            elif param_format == "header":
                return sub(r"%s=[^\\n]*" % escape(str(param)), r"%s=%s" %
                           (str(param).encode('string-escape'),
                            str(value).encode('string-escape')), string)
            else:
                return sub(r"%s=[^&]*" % escape(str(param)), r"%s=%s" %
                           (str(param).encode('string-escape'),
                            str(value).encode('string-escape')), string)

        if self.multiple_params:
            multiple_params_response = ""
            if(self.csrf_token_name and
               self.csrf_url == self.multiple_params_url):
                multiple_params_response = csrf_response
            else:
                self.url = self.multiple_params_url
                multiple_params_response = self.get_page()

            if(self.invalid_http_status and self.response_status and
               int(self.invalid_http_status["value"]) == int(
                   self.response_status) and self.verbose):
                print("%s %s\n" % (INFO, self.invalid_http_status["msg"]
                                   if "msg" in self.invalid_http_status else
                                   "Your IP may have been blocked..."))
                return

            if not multiple_params_response:
                print("%s problem receiving HTTP response while fetching "
                      "params! Skipping site...\n" % ERROR)
                return

            for _ in self.multiple_params:
                regex = (_["regex"] if "regex" in _ else
                         r"<\w+[^>]*(value\s*=\s*\"[^\"]*\"|name\s*=\s*"
                         r"\"?{0}(?:\"|\s))[^>]*(value\s*=\s*\"[^\"]*\""
                         r"|name\s*=\s*\"?{0}(?:\"|\s))[^>]*>"
                         .format(escape(_["value"])))
                match = search(regex, multiple_params_response)

                if not match or "value" not in _:
                    if self.verbose:
                        print("%s no match for parameter \"%s\"! "
                              "Skipping site...\n" %
                              (WARN, color(_["value"])))
                        self.status = {"status": 0, "msg": "No token"}
                    return

                if "regex" in _:
                    value = (match.group("value")
                             if "value" in match.groupdict() else
                             match.group(1))
                elif "value" in _:
                    for attrib in match.groups():
                        attrib = str(attrib).strip().split("=", 1)
                        if attrib[0] == "value":
                            value = attrib[1].strip("\"")

                if "type" not in _:
                    _["type"] = "data"

                if _["type"] == "data" and self.data:
                    self.data = replace_param(self.data, _["value"], value)
                elif _["type"] == "header":
                    self.headers = replace_param(self.headers, _["value"],
                                                 value, "header")
                elif _["type"] == "cookie":
                    self.add_cookies(cookie_handler, "%s=%s;" % (_["value"],
                                                                 value))
                else:
                    pass  # NEED TO REPLACE GET PARAMS

        if(credentials["email"] and
           self.username_or_email in("email", "both")):
            login = credentials["email"]
        elif(credentials["email"] and self.email_exception and
             self.username_or_email == "username" and
             search(self.email_exception, credentials["email"])):
            login = credentials["email"]
        else:
            login = credentials["username"]

        # need to implement support for GET logins lulz

        if self.time_parameter:
            if "type" not in self.time_parameter:
                self.time_parameter["type"] = "epoch"

            if self.time_parameter["type"] == "epoch":
                if self.data:
                    self.data = replace_param(self.data,
                                              self.time_parameter["value"],
                                              time())

        if self.data:
            self.data = replace_param(self.data, self.login_parameter,
                                      login, self.login_parameter_type)
            self.data = replace_param(self.data, self.password_parameter,
                                      credentials["password"],
                                      self.login_parameter_type)

        # need to be able to specify where tokens can be replaced
        if self.csrf_token:
            self.csrf_token = quote(self.csrf_token)
            if self.data:
                self.data = replace_param(self.data,
                                          self.csrf_token_name,
                                          self.csrf_token,
                                          self.login_parameter_type)
            if self.headers:
                self.headers = replace_param(self.headers,
                                             self.csrf_token_name,
                                             self.csrf_token, "header")
            if self.cookies:
                self.cookies = replace_param(self.cookies,
                                             self.csrf_token_name,
                                             self.csrf_token)
                self.add_cookies(cookie_handler, self.cookies)

        self.url = self.login_url
        login_response = self.get_page(data=True if self.data else False,
                                       header=True if self.headers else False)

        if not login_response:
            if self.verbose:
                print("%s no response received! Skipping site...\n" % WARN)
            return False

        if self.login_redirect:
            if self.login_redirect_type == "regex":
                self.url = search(self.login_redirect, login_response)
                self.url = (self.url.group("URL")
                            if "URL" in self.url.groupdict()
                            else self.url.group(1))
            else:
                self.url = self.login_redirect

            self.url = self.url.replace("\\", "")

            login_response = self.get_page(data=True if self.data else False,
                                           header=True if self.headers
                                           else False)

        if not login_response:
            if self.verbose:
                print("%s no response received during login redirect! "
                      "Skipping site...\n" % WARN)
            return False

        # The code for these IF checks need to be cleaned up
        # If invalid credentials http status code is returned
        elif (self.invalid_http_status and self.response_status and
              str(self.invalid_http_status["value"]) ==
              str(self.response_status)):
            if("msg" in self.invalid_http_status or not
               login_response.strip("[]")):
                if self.verbose:
                    print("%s %s\n" %
                          (INFO, self.invalid_http_status["msg"] if "msg"
                           in self.invalid_http_status else
                           "The provided credentials are incorrect "
                           "or the account doesn't exist.\n"))
                return False
        # If captcha flag is set and found in login response
        if self.captcha_flag and self.captcha_flag in login_response:
            if self.verbose:
                print "%s captcha detected! Skipping site...\n" % WARN
            return False
        # If custom search is set and found in response
        elif self.custom_search and search(self.custom_search['regex'],
                                           login_response):
            if self.verbose:
                print "%s %s\n" % (INFO, self.custom_search["value"])
            return False
        # Valid password string in response
        elif self.valid_password and self.valid_password in login_response:
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header type REGEX
        elif (self.valid_response_header and
              self.valid_response_header_type == "regex" and
              search(self.valid_response_header,
                     str(self.response_headers))):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header for cookies type REGEX
        elif (self.valid_response_header and
              self.valid_response_header_type == "regex" and
              search(self.valid_response_header, str(cookie_handler))):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header type normal
        elif (self.valid_response_header and self.valid_response_header
              in str(self.response_headers)):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Valid response header for cookies type normal
        elif (self.valid_response_header and self.valid_response_header
              in str(cookie_handler)):
            print "%s Credentials worked! Successfully logged in.\n" % PLUS
            return True
        # Custom message when specified invalid header is detected
        elif (self.custom_response_header and
              self.custom_response_header["value"] in
              str(self.response_headers)):
            if self.verbose:
                print "%s %s" % (INFO, self.custom_response_header["msg"])
            return False
        # Invalid account string found in login response
        elif self.invalid_account and self.invalid_account in login_response:
            if self.verbose:
                print("%s The provided account doesn't exist on this site.\n"
                      % INFO)
            return False
        # User exists, but account isn't activate.
        elif self.inactive_account and self.inactive_account in login_response:
            if self.verbose:
                print("%s The user exists, but the account isn't activate.\n"
                      % INFO)
            return False
        # User exists, but invalid password string in login response
        elif (self.invalid_password and self.invalid_account and
              self.invalid_password in login_response):
            if self.verbose:
                print("%s The user exists, but the password is incorrect.\n"
                      % INFO)
            return False
        # Invalid password string in login response
        elif (self.invalid_password and not self.invalid_account and
              self.invalid_password in login_response):
            if self.verbose:
                print("%s The provided credentials are incorrect "
                      "or the account doesn't exist.\n" % INFO)
            return False
        # Unhandled case
        else:
            if self.verbose:
                print "%s Unable to login! Skipping site...\n" % WARN
            return False

Example 171

Project: webassets
Source File: rjsmin.py
View license
def _make_jsmin(python_only=False):
    """
    Generate JS minifier based on `jsmin.c by Douglas Crockford`_

    .. _jsmin.c by Douglas Crockford:
       http://www.crockford.com/javascript/jsmin.c

    :Parameters:
      `python_only` : ``bool``
        Use only the python variant. If true, the c extension is not even
        tried to be loaded.

    :Return: Minifier
    :Rtype: ``callable``
    """
    # pylint: disable = unused-variable
    # pylint: disable = too-many-locals

    if not python_only:
        try:
            import _rjsmin
        except ImportError:
            pass
        else:
            return _rjsmin.jsmin
    try:
        xrange
    except NameError:
        xrange = range  # pylint: disable = redefined-builtin

    space_chars = r'[\000-\011\013\014\016-\040]'

    line_comment = r'(?://[^\r\n]*)'
    space_comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
    space_comment_nobang = r'(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*\*+)*/)'
    bang_comment = r'(?:/\*![^*]*\*+(?:[^/*][^*]*\*+)*/)'

    string1 = \
        r'(?:\047[^\047\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)'
    string2 = r'(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^"\\\r\n]*)*")'
    strings = r'(?:%s|%s)' % (string1, string2)

    charclass = r'(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\])'
    nospecial = r'[^/\\\[\r\n]'
    regex = r'(?:/(?![\r\n/*])%s*(?:(?:\\[^\r\n]|%s)%s*)*/)' % (
        nospecial, charclass, nospecial
    )
    space = r'(?:%s|%s)' % (space_chars, space_comment)
    newline = r'(?:%s?[\r\n])' % line_comment

    def fix_charclass(result):
        """ Fixup string of chars to fit into a regex char class """
        pos = result.find('-')
        if pos >= 0:
            result = r'%s%s-' % (result[:pos], result[pos + 1:])

        def sequentize(string):
            """
            Notate consecutive characters as sequence

            (1-4 instead of 1234)
            """
            first, last, result = None, None, []
            for char in map(ord, string):
                if last is None:
                    first = last = char
                elif last + 1 == char:
                    last = char
                else:
                    result.append((first, last))
                    first = last = char
            if last is not None:
                result.append((first, last))
            return ''.join(['%s%s%s' % (
                chr(first),
                last > first + 1 and '-' or '',
                last != first and chr(last) or ''
            ) for first, last in result])  # noqa

        return _re.sub(
            r'([\000-\040\047])',  # \047 for better portability
            lambda m: '\\%03o' % ord(m.group(1)), (
                sequentize(result)
                .replace('\\', '\\\\')
                .replace('[', '\\[')
                .replace(']', '\\]')
            )
        )

    def id_literal_(what):
        """ Make id_literal like char class """
        match = _re.compile(what).match
        result = ''.join([
            chr(c) for c in xrange(127) if not match(chr(c))
        ])
        return '[^%s]' % fix_charclass(result)

    def not_id_literal_(keep):
        """ Make negated id_literal like char class """
        match = _re.compile(id_literal_(keep)).match
        result = ''.join([
            chr(c) for c in xrange(127) if not match(chr(c))
        ])
        return r'[%s]' % fix_charclass(result)

    not_id_literal = not_id_literal_(r'[a-zA-Z0-9_$]')
    preregex1 = r'[(,=:\[!&|?{};\r\n]'
    preregex2 = r'%(not_id_literal)sreturn' % locals()

    id_literal = id_literal_(r'[a-zA-Z0-9_$]')
    id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]')
    id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047+-]')
    post_regex_off = id_literal_(r'[^\000-\040}\])?:|,;.&=+-]')

    dull = r'[^\047"/\000-\040]'

    space_sub_simple = _re.compile((
        # noqa pylint: disable = bad-continuation

        r'(%(dull)s+)'                                         # 0
        r'|(%(strings)s%(dull)s*)'                             # 1
        r'|(?<=%(preregex1)s)'
            r'%(space)s*(?:%(newline)s%(space)s*)*'
            r'(%(regex)s)'                                     # 2
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 3
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(preregex2)s)'
            r'%(space)s*(?:(%(newline)s)%(space)s*)*'          # 4
            r'(%(regex)s)'                                     # 5
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 6
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(id_literal_close)s)'
            r'%(space)s*(?:(%(newline)s)%(space)s*)+'          # 7
            r'(?=%(id_literal_open)s)'
        r'|(?<=%(id_literal)s)(%(space)s)+(?=%(id_literal)s)'  # 8
        r'|(?<=\+)(%(space)s)+(?=\+)'                          # 9
        r'|(?<=-)(%(space)s)+(?=-)'                            # 10
        r'|%(space)s+'
        r'|(?:%(newline)s%(space)s*)+'
    ) % locals()).sub

    # print space_sub_simple.__self__.pattern

    def space_subber_simple(match):
        """ Substitution callback """
        # pylint: disable = too-many-return-statements

        groups = match.groups()
        if groups[0]:
            return groups[0]
        elif groups[1]:
            return groups[1]
        elif groups[2]:
            if groups[3]:
                return groups[2] + '\n'
            return groups[2]
        elif groups[5]:
            return "%s%s%s" % (
                groups[4] and '\n' or '',
                groups[5],
                groups[6] and '\n' or '',
            )
        elif groups[7]:
            return '\n'
        elif groups[8] or groups[9] or groups[10]:
            return ' '
        else:
            return ''

    space_sub_banged = _re.compile((
        # noqa pylint: disable = bad-continuation

        r'(%(dull)s+)'                                         # 0
        r'|(%(strings)s%(dull)s*)'                             # 1
        r'|(?<=%(preregex1)s)'
            r'(%(space)s*(?:%(newline)s%(space)s*)*)'          # 2
            r'(%(regex)s)'                                     # 3
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 4
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(preregex2)s)'
            r'(%(space)s*(?:(%(newline)s)%(space)s*)*)'        # 5, 6
            r'(%(regex)s)'                                     # 7
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 8
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(id_literal_close)s)'
            r'(%(space)s*(?:%(newline)s%(space)s*)+)'          # 9
            r'(?=%(id_literal_open)s)'
        r'|(?<=%(id_literal)s)(%(space)s+)(?=%(id_literal)s)'  # 10
        r'|(?<=\+)(%(space)s+)(?=\+)'                          # 11
        r'|(?<=-)(%(space)s+)(?=-)'                            # 12
        r'|(%(space)s+)'                                       # 13
        r'|((?:%(newline)s%(space)s*)+)'                       # 14
    ) % locals()).sub

    # print space_sub_banged.__self__.pattern

    keep = _re.compile((
        r'%(space_chars)s+|%(space_comment_nobang)s+|%(newline)s+'
        r'|(%(bang_comment)s+)'
    ) % locals()).sub
    keeper = lambda m: m.groups()[0] or ''

    # print keep.__self__.pattern

    def space_subber_banged(match):
        """ Substitution callback """
        # pylint: disable = too-many-return-statements

        groups = match.groups()
        if groups[0]:
            return groups[0]
        elif groups[1]:
            return groups[1]
        elif groups[3]:
            return "%s%s%s%s" % (
                keep(keeper, groups[2]),
                groups[3],
                keep(keeper, groups[4] or ''),
                groups[4] and '\n' or '',
            )
        elif groups[7]:
            return "%s%s%s%s%s" % (
                keep(keeper, groups[5]),
                groups[6] and '\n' or '',
                groups[7],
                keep(keeper, groups[8] or ''),
                groups[8] and '\n' or '',
            )
        elif groups[9]:
            return keep(keeper, groups[9]) + '\n'
        elif groups[10] or groups[11] or groups[12]:
            return keep(keeper, groups[10] or groups[11] or groups[12]) or ' '
        else:
            return keep(keeper, groups[13] or groups[14])

    def jsmin(script, keep_bang_comments=False):
        r"""
        Minify javascript based on `jsmin.c by Douglas Crockford`_\.

        Instead of parsing the stream char by char, it uses a regular
        expression approach which minifies the whole script with one big
        substitution regex.

        .. _jsmin.c by Douglas Crockford:
           http://www.crockford.com/javascript/jsmin.c

        :Parameters:
          `script` : ``str``
            Script to minify

          `keep_bang_comments` : ``bool``
            Keep comments starting with an exclamation mark? (``/*!...*/``)

        :Return: Minified script
        :Rtype: ``str``
        """
        # pylint: disable = redefined-outer-name

        if keep_bang_comments:
            return space_sub_banged(
                space_subber_banged, '\n%s\n' % script
            ).strip()
        else:
            return space_sub_simple(
                space_subber_simple, '\n%s\n' % script
            ).strip()

    return jsmin

Example 172

Project: Arelle
Source File: Validate.py
View license
    def validateTestcase(self, testcase):
        self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
        self.modelXbrl.viewModelObject(testcase.objectId())
        if hasattr(testcase, "testcaseVariations"):
            for modelTestcaseVariation in testcase.testcaseVariations:
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                # is this a versioning report?
                resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
                resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
                resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
                formulaOutputInstance = None
                inputDTSes = defaultdict(list)
                baseForElement = testcase.baseForElement(modelTestcaseVariation)
                # try to load instance document
                self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
                                    modelObject=modelTestcaseVariation, 
                                    id=modelTestcaseVariation.id, 
                                    name=modelTestcaseVariation.name, 
                                    expected=modelTestcaseVariation.expected, 
                                    description=modelTestcaseVariation.description)
                errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
                parameters = modelTestcaseVariation.parameters.copy()
                for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                    if isinstance(readMeFirstUri,tuple):
                        # dtsName is for formula instances, but is from/to dts if versioning
                        dtsName, readMeFirstUri = readMeFirstUri
                    elif resultIsVersioningReport:
                        if inputDTSes: dtsName = "to"
                        else: dtsName = "from"
                    else:
                        dtsName = None
                    if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                        if dtsName in inputDTSes:
                            dtsName = inputDTSes[dtsName]
                        else:
                            modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                         Type.DTSENTRIES,
                                         self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                         isEntry=True,
                                         errorCaptureLevel=errorCaptureLevel)
                        DTSdoc = modelXbrl.modelDocument
                        DTSdoc.inDTS = True
                        doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
                        if doc is not None:
                            DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement)  #fake import
                            doc.inDTS = True
                    elif resultIsTaxonomyPackage:
                        from arelle import PackageManager, PrototypeInstanceObject
                        dtsName = readMeFirstUri
                        modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
                        PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
                    else: # not a multi-schemaRef versioning report
                        if self.useFileSource.isArchive:
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       readMeFirstUri,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       useFileSource=self.useFileSource,
                                                       errorCaptureLevel=errorCaptureLevel)
                        else: # need own file source, may need instance discovery
                            filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
                            if filesource and not filesource.selection and filesource.isArchive:
                                for _archiveFile in filesource.dir: # find instance document in archive
                                    filesource.select(_archiveFile)
                                    if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
                                        break # use this selection
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       filesource,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       errorCaptureLevel=errorCaptureLevel)
                        modelXbrl.isTestcaseVariation = True
                    if modelXbrl.modelDocument is None:
                        modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s document not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        self.determineNotLoadedTestStatus(modelTestcaseVariation)
                        modelXbrl.close()
                    elif resultIsVersioningReport or resultIsTaxonomyPackage:
                        inputDTSes[dtsName] = modelXbrl
                    elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
                        ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif testcase.type == Type.REGISTRYTESTCASE:
                        self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                        self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                   modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        self.instValidator.close()
                        modelXbrl.close()
                    else:
                        inputDTSes[dtsName].append(modelXbrl)
                        # validate except for formulas
                        _hasFormulae = modelXbrl.hasFormulae
                        modelXbrl.hasFormulae = False
                        try:
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
                            self.instValidator.validate(modelXbrl, parameters)
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                        modelXbrl.hasFormulae = _hasFormulae
                if resultIsVersioningReport and modelXbrl.modelDocument:
                    versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                        modelTestcaseVariation.versioningReportUri, baseForElement)
                    if os.path.exists(versReportFile): #validate existing
                        modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                        if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
                            ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                            self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
                            modelVersReport.close()
                    elif len(inputDTSes) == 2:
                        ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                              versReportFile, inputDTSes["from"], inputDTSes["to"])
                        modelTestcaseVariation.status = "generated"
                    else:
                        modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        modelTestcaseVariation.status = "failed"
                    for inputDTS in inputDTSes.values():
                        inputDTS.close()
                    del inputDTSes # dereference
                elif resultIsTaxonomyPackage:
                    self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                    modelXbrl.close()
                elif inputDTSes:
                    # validate schema, linkbase, or instance
                    modelXbrl = inputDTSes[None][0]
                    for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                        if dtsName: # named instance
                            parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
                        elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
                            parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
                    if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
                        RenderingEvaluator.init(modelXbrl)
                    if modelXbrl.hasFormulae:
                        try:
                            # validate only formulae
                            self.instValidator.parameters = parameters
                            ValidateFormula.validate(self.instValidator)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
                        for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
                            pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
                        infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                 modelTestcaseVariation.resultInfosetUri,
                                                   _("loading result infoset"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if infoset.modelDocument is None:
                            modelXbrl.error("arelle:notLoaded",
                                _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                            modelTestcaseVariation.status = "result infoset not loadable"
                        else:   # check infoset
                            ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                        infoset.close()
                    if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
                        # diff (or generate) table infoset
                        resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
                        if not any(alternativeValidation(modelXbrl, resultTableUri)
                                   for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
                            ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True)  # false to save infoset files
                    self.instValidator.close()
                    extraErrors = []
                    for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
                        pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
                    self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
                    if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                        # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                        modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                        self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
                        if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                            formulaOutputInstance = modelXbrl.formulaOutputInstance
                            modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                        self.instValidator.close()
                    compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
                    if compareIxResultInstance:
                        formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
                        errMsgPrefix = "ix"
                    else: # delete input instances before formula output comparision
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                        errMsgPrefix = "formula"
                    if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                        expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                   modelTestcaseVariation.resultXbrlInstanceUri,
                                                   _("loading expected result XBRL instance"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if expectedInstance.modelDocument is None:
                            self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
                                _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
                                messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
                            modelTestcaseVariation.status = "result not loadable"
                        else:   # compare facts
                            if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                                formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
                                    _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                    modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                    expectedFacts=len(expectedInstance.facts),
                                    messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
                            else:
                                formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
                                expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
                                def factFootnotes(fact, footnotesRelSet):
                                    footnotes = []
                                    footnoteRels = footnotesRelSet.fromModelObject(fact)
                                    if footnoteRels:
                                        # most process rels in same order between two instances, use labels to sort
                                        for i, footnoteRel in enumerate(sorted(footnoteRels,
                                                                               key=lambda r: (r.fromLabel,r.toLabel))):
                                            modelObject = footnoteRel.toModelObject
                                            if isinstance(modelObject, ModelResource):
                                                footnotes.append("Footnote {}: {}".format(
                                                   i+1, # compare footnote with normalize-space
                                                   re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))))
                                            elif isinstance(modelObject, ModelFact):
                                                footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
                                                    i+1,
                                                    modelObject.qname,
                                                    modelObject.contextID,
                                                    collapseWhitespace(modelObject.value)))
                                    return footnotes
                                for expectedInstanceFact in expectedInstance.facts:
                                    unmatchedFactsStack = []
                                    formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
                                    if formulaOutputFact is None:
                                        if unmatchedFactsStack: # get missing nested tuple fact, if possible
                                            missingFact = unmatchedFactsStack[-1]
                                        else:
                                            missingFact = expectedInstanceFact
                                        formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
                                            _("Output missing expected fact %(fact)s"),
                                            modelXbrl=missingFact, fact=missingFact.qname,
                                            messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
                                    else: # compare footnotes
                                        expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
                                        formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
                                        if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
                                            formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
                                                _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
                                                modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
                                                messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))

                            # for debugging uncomment next line to save generated instance document
                            # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
                        expectedInstance.close()
                        del expectedInstance # dereference
                        self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
                        formulaOutputInstance.close()
                        del formulaOutputInstance
                    if compareIxResultInstance:
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                    
            self.modelXbrl.modelManager.showStatus(_("ready"), 2000)

Example 173

Project: Arelle
Source File: Validate.py
View license
    def validateTestcase(self, testcase):
        self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
        self.modelXbrl.viewModelObject(testcase.objectId())
        if hasattr(testcase, "testcaseVariations"):
            for modelTestcaseVariation in testcase.testcaseVariations:
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                # is this a versioning report?
                resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
                resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
                resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
                formulaOutputInstance = None
                inputDTSes = defaultdict(list)
                baseForElement = testcase.baseForElement(modelTestcaseVariation)
                # try to load instance document
                self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
                                    modelObject=modelTestcaseVariation, 
                                    id=modelTestcaseVariation.id, 
                                    name=modelTestcaseVariation.name, 
                                    expected=modelTestcaseVariation.expected, 
                                    description=modelTestcaseVariation.description)
                errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
                parameters = modelTestcaseVariation.parameters.copy()
                for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                    if isinstance(readMeFirstUri,tuple):
                        # dtsName is for formula instances, but is from/to dts if versioning
                        dtsName, readMeFirstUri = readMeFirstUri
                    elif resultIsVersioningReport:
                        if inputDTSes: dtsName = "to"
                        else: dtsName = "from"
                    else:
                        dtsName = None
                    if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                        if dtsName in inputDTSes:
                            dtsName = inputDTSes[dtsName]
                        else:
                            modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                         Type.DTSENTRIES,
                                         self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                         isEntry=True,
                                         errorCaptureLevel=errorCaptureLevel)
                        DTSdoc = modelXbrl.modelDocument
                        DTSdoc.inDTS = True
                        doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
                        if doc is not None:
                            DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement)  #fake import
                            doc.inDTS = True
                    elif resultIsTaxonomyPackage:
                        from arelle import PackageManager, PrototypeInstanceObject
                        dtsName = readMeFirstUri
                        modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
                        PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
                    else: # not a multi-schemaRef versioning report
                        if self.useFileSource.isArchive:
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       readMeFirstUri,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       useFileSource=self.useFileSource,
                                                       errorCaptureLevel=errorCaptureLevel)
                        else: # need own file source, may need instance discovery
                            filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
                            if filesource and not filesource.selection and filesource.isArchive:
                                for _archiveFile in filesource.dir: # find instance document in archive
                                    filesource.select(_archiveFile)
                                    if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
                                        break # use this selection
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       filesource,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       errorCaptureLevel=errorCaptureLevel)
                        modelXbrl.isTestcaseVariation = True
                    if modelXbrl.modelDocument is None:
                        modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s document not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        self.determineNotLoadedTestStatus(modelTestcaseVariation)
                        modelXbrl.close()
                    elif resultIsVersioningReport or resultIsTaxonomyPackage:
                        inputDTSes[dtsName] = modelXbrl
                    elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
                        ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif testcase.type == Type.REGISTRYTESTCASE:
                        self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                        self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                   modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        self.instValidator.close()
                        modelXbrl.close()
                    else:
                        inputDTSes[dtsName].append(modelXbrl)
                        # validate except for formulas
                        _hasFormulae = modelXbrl.hasFormulae
                        modelXbrl.hasFormulae = False
                        try:
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
                            self.instValidator.validate(modelXbrl, parameters)
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                        modelXbrl.hasFormulae = _hasFormulae
                if resultIsVersioningReport and modelXbrl.modelDocument:
                    versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                        modelTestcaseVariation.versioningReportUri, baseForElement)
                    if os.path.exists(versReportFile): #validate existing
                        modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                        if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
                            ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                            self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
                            modelVersReport.close()
                    elif len(inputDTSes) == 2:
                        ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                              versReportFile, inputDTSes["from"], inputDTSes["to"])
                        modelTestcaseVariation.status = "generated"
                    else:
                        modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        modelTestcaseVariation.status = "failed"
                    for inputDTS in inputDTSes.values():
                        inputDTS.close()
                    del inputDTSes # dereference
                elif resultIsTaxonomyPackage:
                    self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                    modelXbrl.close()
                elif inputDTSes:
                    # validate schema, linkbase, or instance
                    modelXbrl = inputDTSes[None][0]
                    for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                        if dtsName: # named instance
                            parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
                        elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
                            parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
                    if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
                        RenderingEvaluator.init(modelXbrl)
                    if modelXbrl.hasFormulae:
                        try:
                            # validate only formulae
                            self.instValidator.parameters = parameters
                            ValidateFormula.validate(self.instValidator)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
                        for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
                            pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
                        infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                 modelTestcaseVariation.resultInfosetUri,
                                                   _("loading result infoset"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if infoset.modelDocument is None:
                            modelXbrl.error("arelle:notLoaded",
                                _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                            modelTestcaseVariation.status = "result infoset not loadable"
                        else:   # check infoset
                            ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                        infoset.close()
                    if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
                        # diff (or generate) table infoset
                        resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
                        if not any(alternativeValidation(modelXbrl, resultTableUri)
                                   for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
                            ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True)  # false to save infoset files
                    self.instValidator.close()
                    extraErrors = []
                    for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
                        pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
                    self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
                    if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                        # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                        modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                        self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
                        if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                            formulaOutputInstance = modelXbrl.formulaOutputInstance
                            modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                        self.instValidator.close()
                    compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
                    if compareIxResultInstance:
                        formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
                        errMsgPrefix = "ix"
                    else: # delete input instances before formula output comparision
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                        errMsgPrefix = "formula"
                    if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                        expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                   modelTestcaseVariation.resultXbrlInstanceUri,
                                                   _("loading expected result XBRL instance"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if expectedInstance.modelDocument is None:
                            self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
                                _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
                                messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
                            modelTestcaseVariation.status = "result not loadable"
                        else:   # compare facts
                            if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                                formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
                                    _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                    modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                    expectedFacts=len(expectedInstance.facts),
                                    messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
                            else:
                                formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
                                expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
                                def factFootnotes(fact, footnotesRelSet):
                                    footnotes = []
                                    footnoteRels = footnotesRelSet.fromModelObject(fact)
                                    if footnoteRels:
                                        # most process rels in same order between two instances, use labels to sort
                                        for i, footnoteRel in enumerate(sorted(footnoteRels,
                                                                               key=lambda r: (r.fromLabel,r.toLabel))):
                                            modelObject = footnoteRel.toModelObject
                                            if isinstance(modelObject, ModelResource):
                                                footnotes.append("Footnote {}: {}".format(
                                                   i+1, # compare footnote with normalize-space
                                                   re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))))
                                            elif isinstance(modelObject, ModelFact):
                                                footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
                                                    i+1,
                                                    modelObject.qname,
                                                    modelObject.contextID,
                                                    collapseWhitespace(modelObject.value)))
                                    return footnotes
                                for expectedInstanceFact in expectedInstance.facts:
                                    unmatchedFactsStack = []
                                    formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
                                    if formulaOutputFact is None:
                                        if unmatchedFactsStack: # get missing nested tuple fact, if possible
                                            missingFact = unmatchedFactsStack[-1]
                                        else:
                                            missingFact = expectedInstanceFact
                                        formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
                                            _("Output missing expected fact %(fact)s"),
                                            modelXbrl=missingFact, fact=missingFact.qname,
                                            messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
                                    else: # compare footnotes
                                        expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
                                        formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
                                        if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
                                            formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
                                                _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
                                                modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
                                                messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))

                            # for debugging uncomment next line to save generated instance document
                            # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
                        expectedInstance.close()
                        del expectedInstance # dereference
                        self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
                        formulaOutputInstance.close()
                        del formulaOutputInstance
                    if compareIxResultInstance:
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                    
            self.modelXbrl.modelManager.showStatus(_("ready"), 2000)

Example 174

Project: dashman
Source File: dashvote.py
View license
def main(screen):

    global stdscr
    global votecount
    global window_width
    global max_yeacount_len
    global max_naycount_len
    global max_percentage_len
    global ballot_entries
    global votewin
    global masternodes
    global C_YELLOW, C_GREEN, C_RED, C_CYAN

    stdscr = screen
    stdscr.scrollok(1)

    git_describe = run_command(
        'GIT_DIR=%s GIT_WORK_TREE=%s git describe' %
        (git_dir + '/.git', git_dir)).rstrip("\n").split('-')
    try:
        GIT_VERSION = ('-').join((git_describe[i] for i in [1, 2]))
        version = 'v' + VERSION + ' (' + GIT_VERSION + ')'
    except IndexError:
        version = 'v' + VERSION

    try:
        curses.curs_set(2)
    except:
        pass
    if curses.has_colors():
        curses.start_color()
        curses.use_default_colors()
        for i in range(0, curses.COLORS):
            curses.init_pair(i + 1, i, -1)

    C_CYAN = curses.color_pair(7)
    C_YELLOW = curses.color_pair(4)
    C_GREEN = curses.color_pair(3)
    C_RED = curses.color_pair(2)

    # test dash-cli in path -- TODO make robust
    try:
        run_command('dash-cli getinfo')
    except subprocess.CalledProcessError:
        quit(
            "--> cannot find dash-cli in $PATH\n" +
            "    do: export PATH=/path/to/dash-cli-folder:$PATH\n" +
            "    and try again\n")

    loadwin = curses.newwin(40, 40, 1, 2)

    loadwin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
    loadwin.addstr(2, 2, 'loading votes... please wait', C_GREEN)
    loadwin.refresh()

    mncount = int(run_command('dash-cli masternode count'))
    block_height = int(run_command('dash-cli getblockcount'))
    # get ballot
    ballots = json.loads(run_command('dash-cli mnbudget show'))
    ballot = {}
    for entry in ballots:
        # prune expired proposals
        if ballots[entry][u'BlockEnd'] < block_height:
            continue
        # prune completely funded proposals
        if ballots[entry][u'RemainingPaymentCount'] < 1:
            continue
        ballots[entry][u'vote'] = 'ABSTAIN'
        ballots[entry][u'votes'] = json.loads(
            run_command(
                'dash-cli mnbudget getvotes %s' %
                entry))
        ballot[entry] = ballots[entry]
    ballot_entries = sorted(ballot, key=lambda s: ballot[s]['BlockStart'])
    votecount = len(ballot_entries)
    max_proposal_len = 0
    max_yeacount_len = 0
    max_naycount_len = 0
    max_percentage_len = 0
    for entry in ballot_entries:
        yeas = ballot[entry][u'Yeas']
        nays = ballot[entry][u'Nays']
        percentage = "{0:.1f}".format(
            (float((yeas + nays)) / float(mncount)) * 100)
        ballot[entry][u'vote_turnout'] = percentage
        ballot[entry][u'vote_threshold'] = (
            yeas + nays) > mncount/10 and True or False
        ballot[entry][u'vote_passing'] = (
            yeas - nays) > mncount/10 and True or False
        max_proposal_len = max(
            max_proposal_len,
            len(entry))
        max_yeacount_len = max(max_yeacount_len, len(str(yeas)))
        max_naycount_len = max(max_naycount_len, len(str(nays)))
        max_percentage_len = max(max_percentage_len, len(str(percentage)))

    # extract mnprivkey,txid-txidx from masternode.conf
    masternodes = {}
    with open(os.path.join(dash_conf_dir, 'masternode.conf'), 'r') as f:
        lines = list(
            line
            for line in
            (l.strip() for l in f)
            if line and not line.startswith('#'))
        for line in lines:
            conf = line.split()
            masternodes[conf[3] + '-' + conf[4]] = {
                "alias": conf[0],
                "mnprivkey": conf[2],
                "fundtx": conf[3] +
                '-' +
                conf[4],
                "txid": conf[3],
                "txout": conf[4]}
    if not masternodes:
        # fallback to dash.conf entries if no masternode.conf entries
        with open(os.path.join(dash_conf_dir, 'dash.conf'), 'r') as f:
            lines = list(
                line
                for line in
                (l.strip() for l in f)
                if line and not line.startswith('#'))
            conf = {}
            for line in lines:
                n, v = line.split('=')
                conf[n.strip(' ')] = v.strip(' ')
            conf['masternodeaddr'] = re.sub(
                '[\[\]]',
                '',
                conf['masternodeaddr'])
            if all(k in conf for k in ('masternode', 'masternodeaddr', 'masternodeprivkey')):
                # get funding tx from dashninja
                import urllib2
                mninfo = urllib2.urlopen(
                    "https://dashninja.pl/api/masternodes?ips=[\"" +
                    conf['masternodeaddr'] +
                    "\"]&portcheck=1").read()
                try:
                    mndata = json.loads(mninfo)
                    d = mndata[u'data'][0]
                except:
                    quit('cannot retrieve masternode info from dashninja')
                vin = str(d[u'MasternodeOutputHash'])
                vidx = str(d[u'MasternodeOutputIndex'])
                masternodes[vin + '-' + vidx] = {
                    "alias": conf['masternodeaddr'],
                    "mnprivkey": conf['masternodeprivkey'],
                    "fundtx": vin +
                    '-' +
                    vidx,
                    "txid": vin,
                    "txout": vidx}
            else:
                quit('cannot find masternode information in dash.conf')

    # TODO open previous votes/local storage something
    for entry in ballot:
        ballot[entry][u'previously_voted'] = 0
        for hash in ballot[entry][u'votes']:
            if hash in masternodes:
                if ballot[entry][u'votes'][hash][u'Vote'] == 'YES':
                    ballot[entry][u'previously_voted'] = 1
                else:
                    ballot[entry][u'previously_voted'] = 2

    loadwin.erase()
    window_width = 35
    window_width = max(window_width, max_proposal_len +
                       max_percentage_len +
                       max_yeacount_len +
                       max_naycount_len +
                       len(str(len(masternodes))))
    votewin = curses.newwin(votecount + 9, window_width + 17, 1, 2)
    votewin.keypad(1)
    votewin.border()

    votewin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
    votewin.addstr(
        2,
        2,
        'use arrow keys to set votes for %s masternodes' %
        len(masternodes),
        C_YELLOW)
    votewin.addstr(3, 2, 'hit enter on CONFIRM to vote - q to quit', C_YELLOW)
    votewin.addstr(4, 3, '*', C_GREEN)
    votewin.addstr(4, 4, '/', C_CYAN)
    votewin.addstr(4, 5, '*', C_RED)
    votewin.addstr(4, 7, '== previously voted proposal (yes/no)', C_YELLOW)
    _y = 5
    for entry in ballot_entries:
        _y += 1
        x = 4
        yeas = ballot[entry][u'Yeas']
        nays = ballot[entry][u'Nays']
        percentage = ballot[entry][u'vote_turnout']
        passing = ballot[entry][u'vote_passing']
        threshold = ballot[entry][u'vote_threshold']
        if ballot[entry][u'previously_voted'] > 0:
            direction = ballot[entry][u'previously_voted']
            votewin.addstr(_y, x-1, '*', direction == 1 and C_GREEN or C_RED)

        fmt_entry = "%-"+str(max_proposal_len + 2)+"s"
        votewin.addstr(
            _y,
            x,
            fmt_entry % entry,
            passing and C_GREEN or threshold and C_RED or C_YELLOW)

        for x in range(max_yeacount_len - len(str(yeas))):
            votewin.addstr(' ')

        votewin.addstr(str(yeas), C_GREEN)
        votewin.addstr('/', C_CYAN)
        votewin.addstr(str(nays), C_RED)
        votewin.addstr(' ')

        for x in range(max_naycount_len - len(str(nays))):
            votewin.addstr(' ')

        for x in range(max_percentage_len - len(str(percentage))):
            votewin.addstr(' ')

        votewin.addstr(str(percentage) + "%", C_CYAN)

        votewin.addstr(' ')
        votewin.addstr('ABSTAIN', C_YELLOW)
    votewin.addstr(
        _y + 2,
        window_width + 7,
        'confirm',
        C_YELLOW)
    votewin.move(0 + 6, window_width + 7)

    votewin.refresh()

    keys = {
        113: lambda s: quit(),
        curses.KEY_UP: lambda s: prev_vote(s),
        curses.KEY_DOWN: lambda s: next_vote(s),
        curses.KEY_RIGHT: lambda s: set_vote(ballot, s, 1),
        curses.KEY_LEFT: lambda s: set_vote(ballot, s, -1),
        107: lambda s: prev_vote(s),
        106: lambda s: next_vote(s),
        108: lambda s: set_vote(ballot, s, 1),
        104: lambda s: set_vote(ballot, s, -1),
        10: lambda s: submit_votes(stdscr, ballot, s)
    }

    sel_vote = 0
    while True:
        key = votewin.getch()
        f = keys.get(key)
        if hasattr(f, '__call__'):
            sel_vote = f(sel_vote)
            try:
                entry_vote = ballot[ballot_entries[sel_vote]][u'vote']
            except IndexError:
                # CONFIRM button
                entry_vote = ''
            if key != 10:
                update_vote_display(votewin, sel_vote, entry_vote)

Example 175

Project: dashman
Source File: dashvote.py
View license
def main(screen):

    global stdscr
    global votecount
    global window_width
    global max_yeacount_len
    global max_naycount_len
    global max_percentage_len
    global ballot_entries
    global votewin
    global masternodes
    global C_YELLOW, C_GREEN, C_RED, C_CYAN

    stdscr = screen
    stdscr.scrollok(1)

    git_describe = run_command(
        'GIT_DIR=%s GIT_WORK_TREE=%s git describe' %
        (git_dir + '/.git', git_dir)).rstrip("\n").split('-')
    try:
        GIT_VERSION = ('-').join((git_describe[i] for i in [1, 2]))
        version = 'v' + VERSION + ' (' + GIT_VERSION + ')'
    except IndexError:
        version = 'v' + VERSION

    try:
        curses.curs_set(2)
    except:
        pass
    if curses.has_colors():
        curses.start_color()
        curses.use_default_colors()
        for i in range(0, curses.COLORS):
            curses.init_pair(i + 1, i, -1)

    C_CYAN = curses.color_pair(7)
    C_YELLOW = curses.color_pair(4)
    C_GREEN = curses.color_pair(3)
    C_RED = curses.color_pair(2)

    # test dash-cli in path -- TODO make robust
    try:
        run_command('dash-cli getinfo')
    except subprocess.CalledProcessError:
        quit(
            "--> cannot find dash-cli in $PATH\n" +
            "    do: export PATH=/path/to/dash-cli-folder:$PATH\n" +
            "    and try again\n")

    loadwin = curses.newwin(40, 40, 1, 2)

    loadwin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
    loadwin.addstr(2, 2, 'loading votes... please wait', C_GREEN)
    loadwin.refresh()

    mncount = int(run_command('dash-cli masternode count'))
    block_height = int(run_command('dash-cli getblockcount'))
    # get ballot
    ballots = json.loads(run_command('dash-cli mnbudget show'))
    ballot = {}
    for entry in ballots:
        # prune expired proposals
        if ballots[entry][u'BlockEnd'] < block_height:
            continue
        # prune completely funded proposals
        if ballots[entry][u'RemainingPaymentCount'] < 1:
            continue
        ballots[entry][u'vote'] = 'ABSTAIN'
        ballots[entry][u'votes'] = json.loads(
            run_command(
                'dash-cli mnbudget getvotes %s' %
                entry))
        ballot[entry] = ballots[entry]
    ballot_entries = sorted(ballot, key=lambda s: ballot[s]['BlockStart'])
    votecount = len(ballot_entries)
    max_proposal_len = 0
    max_yeacount_len = 0
    max_naycount_len = 0
    max_percentage_len = 0
    for entry in ballot_entries:
        yeas = ballot[entry][u'Yeas']
        nays = ballot[entry][u'Nays']
        percentage = "{0:.1f}".format(
            (float((yeas + nays)) / float(mncount)) * 100)
        ballot[entry][u'vote_turnout'] = percentage
        ballot[entry][u'vote_threshold'] = (
            yeas + nays) > mncount/10 and True or False
        ballot[entry][u'vote_passing'] = (
            yeas - nays) > mncount/10 and True or False
        max_proposal_len = max(
            max_proposal_len,
            len(entry))
        max_yeacount_len = max(max_yeacount_len, len(str(yeas)))
        max_naycount_len = max(max_naycount_len, len(str(nays)))
        max_percentage_len = max(max_percentage_len, len(str(percentage)))

    # extract mnprivkey,txid-txidx from masternode.conf
    masternodes = {}
    with open(os.path.join(dash_conf_dir, 'masternode.conf'), 'r') as f:
        lines = list(
            line
            for line in
            (l.strip() for l in f)
            if line and not line.startswith('#'))
        for line in lines:
            conf = line.split()
            masternodes[conf[3] + '-' + conf[4]] = {
                "alias": conf[0],
                "mnprivkey": conf[2],
                "fundtx": conf[3] +
                '-' +
                conf[4],
                "txid": conf[3],
                "txout": conf[4]}
    if not masternodes:
        # fallback to dash.conf entries if no masternode.conf entries
        with open(os.path.join(dash_conf_dir, 'dash.conf'), 'r') as f:
            lines = list(
                line
                for line in
                (l.strip() for l in f)
                if line and not line.startswith('#'))
            conf = {}
            for line in lines:
                n, v = line.split('=')
                conf[n.strip(' ')] = v.strip(' ')
            conf['masternodeaddr'] = re.sub(
                '[\[\]]',
                '',
                conf['masternodeaddr'])
            if all(k in conf for k in ('masternode', 'masternodeaddr', 'masternodeprivkey')):
                # get funding tx from dashninja
                import urllib2
                mninfo = urllib2.urlopen(
                    "https://dashninja.pl/api/masternodes?ips=[\"" +
                    conf['masternodeaddr'] +
                    "\"]&portcheck=1").read()
                try:
                    mndata = json.loads(mninfo)
                    d = mndata[u'data'][0]
                except:
                    quit('cannot retrieve masternode info from dashninja')
                vin = str(d[u'MasternodeOutputHash'])
                vidx = str(d[u'MasternodeOutputIndex'])
                masternodes[vin + '-' + vidx] = {
                    "alias": conf['masternodeaddr'],
                    "mnprivkey": conf['masternodeprivkey'],
                    "fundtx": vin +
                    '-' +
                    vidx,
                    "txid": vin,
                    "txout": vidx}
            else:
                quit('cannot find masternode information in dash.conf')

    # TODO open previous votes/local storage something
    for entry in ballot:
        ballot[entry][u'previously_voted'] = 0
        for hash in ballot[entry][u'votes']:
            if hash in masternodes:
                if ballot[entry][u'votes'][hash][u'Vote'] == 'YES':
                    ballot[entry][u'previously_voted'] = 1
                else:
                    ballot[entry][u'previously_voted'] = 2

    loadwin.erase()
    window_width = 35
    window_width = max(window_width, max_proposal_len +
                       max_percentage_len +
                       max_yeacount_len +
                       max_naycount_len +
                       len(str(len(masternodes))))
    votewin = curses.newwin(votecount + 9, window_width + 17, 1, 2)
    votewin.keypad(1)
    votewin.border()

    votewin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
    votewin.addstr(
        2,
        2,
        'use arrow keys to set votes for %s masternodes' %
        len(masternodes),
        C_YELLOW)
    votewin.addstr(3, 2, 'hit enter on CONFIRM to vote - q to quit', C_YELLOW)
    votewin.addstr(4, 3, '*', C_GREEN)
    votewin.addstr(4, 4, '/', C_CYAN)
    votewin.addstr(4, 5, '*', C_RED)
    votewin.addstr(4, 7, '== previously voted proposal (yes/no)', C_YELLOW)
    _y = 5
    for entry in ballot_entries:
        _y += 1
        x = 4
        yeas = ballot[entry][u'Yeas']
        nays = ballot[entry][u'Nays']
        percentage = ballot[entry][u'vote_turnout']
        passing = ballot[entry][u'vote_passing']
        threshold = ballot[entry][u'vote_threshold']
        if ballot[entry][u'previously_voted'] > 0:
            direction = ballot[entry][u'previously_voted']
            votewin.addstr(_y, x-1, '*', direction == 1 and C_GREEN or C_RED)

        fmt_entry = "%-"+str(max_proposal_len + 2)+"s"
        votewin.addstr(
            _y,
            x,
            fmt_entry % entry,
            passing and C_GREEN or threshold and C_RED or C_YELLOW)

        for x in range(max_yeacount_len - len(str(yeas))):
            votewin.addstr(' ')

        votewin.addstr(str(yeas), C_GREEN)
        votewin.addstr('/', C_CYAN)
        votewin.addstr(str(nays), C_RED)
        votewin.addstr(' ')

        for x in range(max_naycount_len - len(str(nays))):
            votewin.addstr(' ')

        for x in range(max_percentage_len - len(str(percentage))):
            votewin.addstr(' ')

        votewin.addstr(str(percentage) + "%", C_CYAN)

        votewin.addstr(' ')
        votewin.addstr('ABSTAIN', C_YELLOW)
    votewin.addstr(
        _y + 2,
        window_width + 7,
        'confirm',
        C_YELLOW)
    votewin.move(0 + 6, window_width + 7)

    votewin.refresh()

    keys = {
        113: lambda s: quit(),
        curses.KEY_UP: lambda s: prev_vote(s),
        curses.KEY_DOWN: lambda s: next_vote(s),
        curses.KEY_RIGHT: lambda s: set_vote(ballot, s, 1),
        curses.KEY_LEFT: lambda s: set_vote(ballot, s, -1),
        107: lambda s: prev_vote(s),
        106: lambda s: next_vote(s),
        108: lambda s: set_vote(ballot, s, 1),
        104: lambda s: set_vote(ballot, s, -1),
        10: lambda s: submit_votes(stdscr, ballot, s)
    }

    sel_vote = 0
    while True:
        key = votewin.getch()
        f = keys.get(key)
        if hasattr(f, '__call__'):
            sel_vote = f(sel_vote)
            try:
                entry_vote = ballot[ballot_entries[sel_vote]][u'vote']
            except IndexError:
                # CONFIRM button
                entry_vote = ''
            if key != 10:
                update_vote_display(votewin, sel_vote, entry_vote)

Example 176

Project: Honeybee
Source File: Honeybee_Generator_PV.py
View license
def main(_name_,_HBSurfaces,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVgen,_powerOutput,PVinverter,sandiaMode_):
 
    """ This function is the heart of this component it takes all the component arguments and writes one PV generator onto each Honeybee surface connected to this component
 
     Args:
            The arguements seen in the function definition are the same as the arguements on the panel.
                
        Returns:
            The properties of the PV generators connected to this component these properties are then written to an IDF file in Honeybee_ Run Energy Simulation component.
    """
    
    HBSurfacesfromhive = hb_hive.callFromHoneybeeHive(_HBSurfaces) # Call Honeybee surfaces from hive
    
    PVgencount = 0
    
    # CHECK that the same surfaces are not having PV generators added to them twice. If they are stop the component.
    
    class myDict(dict):

        def __init__(self):
            self = dict()
    
        def add(self, key, value):
            self[key] = value
    
    HBsurfaces = myDict()
    
    for HBsurface in HBSurfacesfromhive:
        
        # In order of Honeybee surface, Honeybee surface ID and Brep which corresponds to that Honeybee surface
        
        HBsurfaces.add(HBsurface,HBsurface.ID)
        
    if len(HBsurfaces.values()) != len(set(HBsurfaces.values())):
        
        # Identify duplicateIDs
        
        duplicateIDs = collections.Counter([item for item in HBsurfaces.values()]).keys()
        
        surfaceCount = 0
        
        for surface,surfaceID in HBsurfaces.items():
            
            if surfaceID in duplicateIDs:
                
                warn = "Into the input _HBSurfaces you have input the same surface " + HBsurface.name + " twice. Use Geometry preview to make sure \n"+\
                "that the same surfaces are not being input several times. This component will not run until this is fixed."
                
                ghenv.Component.AddRuntimeMessage(gh.GH_RuntimeMessageLevel.Error, warn)
                
                # Draw a preview of the geometry so the user can identify it.
                
                preview = Grasshopper.Kernel.GH_PreviewUtil()
                
                preview.AddBrep(_HBSurfaces[surfaceCount])
                
                surfaceCount =+ 1
                
                return

    try:
    
        for name,surface,celleff,mode,parallel,series,modelcost,powerout in itertools.izip_longest(_name_,HBSurfacesfromhive,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVPerModule,_powerOutputPerModule): 
    
            surface.containsPVgen = True # Set this it true so the surface can be identified in run and write IDF component
            
            surface.PVgenlist = [] # Set the PVgenlist of each surface back to empty otherwise PVgen objects will accumulate on each run
            
            
            namePVperform = "PVperformance" + str(PVgencount)+ " " + str(surface.name) # Create a name for the PVperformance object for each PV generator - this is always created by automatically here not by the user
    
            # Raise some warnings for the following circumstances 
            
            centerPt, normalVector = surface.getSrfCenPtandNormalAlternate()
            
            if normalVector[2] == -1:
                
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = "Are you sure that you want to mount PV generators on " + str(surface.name) +" \n"+\
                "the surface normal of this surface is facing down!"
                
                ghenv.Component.AddRuntimeMessage(w, warn)
                
            if surface.type == 2.5:
                
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = str(surface.name) +" is a on-ground slab you probably don't want to mount PV panels on it \n"+\
                "but we will go ahead and do it anyway."
                
                ghenv.Component.AddRuntimeMessage(w, warn)
    
            if surface.type == 2.25:
                
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = str(surface.name) +" is a underground slab you probably don't want to mount PV panels on it \n"+\
                "but we will go ahead and do it anyway."
                
                ghenv.Component.AddRuntimeMessage(w, warn)
    
                
            if surface.type == 0.5:
            
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = str(surface.name) +" is a underground wall you probably don't want to mount PV panels on it \n"+\
                "but we will go ahead and do it anyway."
                
                ghenv.Component.AddRuntimeMessage(w, warn)
            
            
            try:
    
                name = _name_[PVgencount]
                message0 = _name_[PVgencount] + " is mounted on Honeybee surface "+ str(surface.name)
    
            except IndexError:
                
                name = "PVgenerator" + str(PVgencount)
                message0 = "For this generator no name has been given so a default name of PVgenerator" + str(PVgencount) + " has been assigned. This generator is mounted on Honeybee surface " + str(surface.name) # If no name given for a PV generator assign one.
            
            if sandiaMode_ == []:
                
                try:
        
                    _cellsEfficiency[PVgencount]
                    message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
        
                except IndexError:
                    
                    celleff = _cellsEfficiency[0]
                    message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
        
            try:
    
                _integrationMode[PVgencount]
                
                message3 = "The integration mode is " + returnmodename(mode)
            
                
            except IndexError:
                
                mode = _integrationMode[0]
                message3 = "The integration mode is "+ returnmodename(mode)
                
            try:
    
                _NoParallel[PVgencount]
                
                message4 = "The number of PV modules in parallel is "+ str(parallel)
            
            except IndexError:
                
                parallel = _NoParallel[0]
                
                message4 = "The number of PV modules in parallel is "+ str(parallel) 
            
    
            try:
    
                _Noseries[PVgencount]
                message5 = "The number of PV modules in series is " + str(series)
    
                
            except IndexError:
                
                series = _Noseries[0]
                message5 = "The number of PV modules in series is " + str(series)
                
                
            message6 = "So " + name + " is made up of "+ str(series*parallel) + " PV modules"
                
            try:
                _costPVPerModule[PVgencount]
                message7 = "The cost per PV module is " + str(modelcost) + " US dollars" + " \n " +\
                "Therefore the total cost of "+ name +" is " + str(modelcost*float(parallel*series)) + " US dollars"
                
                
            except IndexError:
                
                 modelcost = _costPVPerModule[0]
                 message7 = "The cost per PV module is " + str(_costPVPerModule[0]) + " \n " +\
                 "Therefore the total cost of "+ name + " is " + str(modelcost*float(parallel*series)) + " in US dollars "
        
            try:
                _powerOutputPerModule[PVgencount]
    
                message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
                "Therefore the total power output of "+ name+" is " + str(powerout*float(parallel*series)) + " W "
    
                
            except IndexError:
                
                powerout = _powerOutputPerModule[0]
                message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
                "Therefore the total power output of "+ name+ " is " + str(powerout*float(parallel*series)) + " W "
            
            # The surface area of the surface that the PV panels cover are calculated from this equation.
    
            if sandiaMode_ == []:
                
                SA_solarcell = (powerout*parallel*series)/(1000*surface.getArea()*celleff)
            
            if sandiaMode_ != []:
                
                # Get Active Area from Sandria
                for lineCount,line in enumerate(sandiaMode_):
                    
                    if "!- Active Area {m2}" in line:
                        
                        # [^.0-9] remove all non numeric characters from a string using Regex!
                        areaPerPanel = float(re.sub(r'[^.0-9]', "", line))
                        
                        SA_solarcell = (parallel*series*areaPerPanel)/(surface.getArea())
                        
            if (SA_solarcell > 0.85) and (SA_solarcell < 1):
                
                warn = "According to your inputs solar panels cover more than 85% of the surface area of the Honeybee surface "+ surface.name+ "\n"+\
                "Are you sure that the power rating and the number of modules on this surface are correct? \n "+\
                "This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on. \n"+ \
                "Don't worry though this component will still run!"
                w = gh.GH_RuntimeMessageLevel.Warning
                ghenv.Component.AddRuntimeMessage(w, warn)
                
                
            if SA_solarcell >= 1:
                
                warn = "Error! According to your inputs solar panels cover more than 100% of the surface area of the Honeybee surface " + surface.name+"\n"+\
                "Are you sure that the power rating and the number of modules on each surface is correct? \n "+\
                "This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on."
                "This component will not run until the surface area is less than 100%."
                w = gh.GH_RuntimeMessageLevel.Error
                ghenv.Component.AddRuntimeMessage(w, warn)
                
                return 
                
            message1 = "According to the data you entered it is calculated that the modules of this PV generator cover roughly "+str(round(SA_solarcell*100,2))+ " % of the surface area of this surface"
                
            # A hb_EPShdSurface
            if surface.type == 6: ## A bit of a hack to get context surface name the same as being produced by EPShdSurface
                
                coordinatesList = surface.extractPoints()
    
                if type(coordinatesList[0])is not list and type(coordinatesList[0]) is not tuple: coordinatesList = [coordinatesList]
                
                for count in range(len(coordinatesList)):
                    
                    PVsurfacename = surface.name + '_' + `count`
                    surface.name = surface.name + '_' + `count`
    
                surface.PVgenlist.append(PV_gen(name,PVsurfacename,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_)) # Last three inputs are for instance method PV_performance
            
            # Not a hb_EPShdSurface
            else:
                
                surface.PVgenlist.append(PV_gen(name,surface.name,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_))
                
            # Assign the inverter to each PVgenerator.
            
            for PVgen in surface.PVgenlist:
                
                PVgen.inverter = PVinverter
            
            if sandiaMode_ == []:
                
                message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +message2+"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
            
            if sandiaMode_ != []:
            
                message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
            
            readmedatatree.Add(message,gh.Data.GH_Path(PVgencount))
                
            PVgencount = PVgencount+1

    except:
        
        # This catches an error when there is a missing member exception ie length of one of inputs is longer than 
        # number of Honeybee surfaces not sure how to just catch missing member exception!
        warn = "The length of a list of inputs into either _name_,_cellsEfficiency \n" + \
                "_integrationMode,_NoParallel,_Noseries,_costPVgen or _powerOutput \n" + \
                "is longer than the number of Honeybee surfaces connected to this component!\n" + \
                "e.g if you have 2 Honeybee surfaces you cannot have 3 values input into _costPVgen!\n" + \
                "Please check the inputs and try again!"
    
        print warn
        w = gh.GH_RuntimeMessageLevel.Warning
        ghenv.Component.AddRuntimeMessage(w, warn)
        
        return -1 

        
    ModifiedHBSurfaces = hb_hive.addToHoneybeeHive(HBSurfacesfromhive, ghenv.Component)
    
    return ModifiedHBSurfaces

Example 177

Project: Honeybee
Source File: Honeybee_Generator_PV.py
View license
def main(_name_,_HBSurfaces,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVgen,_powerOutput,PVinverter,sandiaMode_):
 
    """ This function is the heart of this component it takes all the component arguments and writes one PV generator onto each Honeybee surface connected to this component
 
     Args:
            The arguements seen in the function definition are the same as the arguements on the panel.
                
        Returns:
            The properties of the PV generators connected to this component these properties are then written to an IDF file in Honeybee_ Run Energy Simulation component.
    """
    
    HBSurfacesfromhive = hb_hive.callFromHoneybeeHive(_HBSurfaces) # Call Honeybee surfaces from hive
    
    PVgencount = 0
    
    # CHECK that the same surfaces are not having PV generators added to them twice. If they are stop the component.
    
    class myDict(dict):

        def __init__(self):
            self = dict()
    
        def add(self, key, value):
            self[key] = value
    
    HBsurfaces = myDict()
    
    for HBsurface in HBSurfacesfromhive:
        
        # In order of Honeybee surface, Honeybee surface ID and Brep which corresponds to that Honeybee surface
        
        HBsurfaces.add(HBsurface,HBsurface.ID)
        
    if len(HBsurfaces.values()) != len(set(HBsurfaces.values())):
        
        # Identify duplicateIDs
        
        duplicateIDs = collections.Counter([item for item in HBsurfaces.values()]).keys()
        
        surfaceCount = 0
        
        for surface,surfaceID in HBsurfaces.items():
            
            if surfaceID in duplicateIDs:
                
                warn = "Into the input _HBSurfaces you have input the same surface " + HBsurface.name + " twice. Use Geometry preview to make sure \n"+\
                "that the same surfaces are not being input several times. This component will not run until this is fixed."
                
                ghenv.Component.AddRuntimeMessage(gh.GH_RuntimeMessageLevel.Error, warn)
                
                # Draw a preview of the geometry so the user can identify it.
                
                preview = Grasshopper.Kernel.GH_PreviewUtil()
                
                preview.AddBrep(_HBSurfaces[surfaceCount])
                
                surfaceCount =+ 1
                
                return

    try:
    
        for name,surface,celleff,mode,parallel,series,modelcost,powerout in itertools.izip_longest(_name_,HBSurfacesfromhive,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVPerModule,_powerOutputPerModule): 
    
            surface.containsPVgen = True # Set this it true so the surface can be identified in run and write IDF component
            
            surface.PVgenlist = [] # Set the PVgenlist of each surface back to empty otherwise PVgen objects will accumulate on each run
            
            
            namePVperform = "PVperformance" + str(PVgencount)+ " " + str(surface.name) # Create a name for the PVperformance object for each PV generator - this is always created by automatically here not by the user
    
            # Raise some warnings for the following circumstances 
            
            centerPt, normalVector = surface.getSrfCenPtandNormalAlternate()
            
            if normalVector[2] == -1:
                
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = "Are you sure that you want to mount PV generators on " + str(surface.name) +" \n"+\
                "the surface normal of this surface is facing down!"
                
                ghenv.Component.AddRuntimeMessage(w, warn)
                
            if surface.type == 2.5:
                
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = str(surface.name) +" is a on-ground slab you probably don't want to mount PV panels on it \n"+\
                "but we will go ahead and do it anyway."
                
                ghenv.Component.AddRuntimeMessage(w, warn)
    
            if surface.type == 2.25:
                
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = str(surface.name) +" is a underground slab you probably don't want to mount PV panels on it \n"+\
                "but we will go ahead and do it anyway."
                
                ghenv.Component.AddRuntimeMessage(w, warn)
    
                
            if surface.type == 0.5:
            
                w = gh.GH_RuntimeMessageLevel.Warning
                
                warn = str(surface.name) +" is a underground wall you probably don't want to mount PV panels on it \n"+\
                "but we will go ahead and do it anyway."
                
                ghenv.Component.AddRuntimeMessage(w, warn)
            
            
            try:
    
                name = _name_[PVgencount]
                message0 = _name_[PVgencount] + " is mounted on Honeybee surface "+ str(surface.name)
    
            except IndexError:
                
                name = "PVgenerator" + str(PVgencount)
                message0 = "For this generator no name has been given so a default name of PVgenerator" + str(PVgencount) + " has been assigned. This generator is mounted on Honeybee surface " + str(surface.name) # If no name given for a PV generator assign one.
            
            if sandiaMode_ == []:
                
                try:
        
                    _cellsEfficiency[PVgencount]
                    message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
        
                except IndexError:
                    
                    celleff = _cellsEfficiency[0]
                    message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
        
            try:
    
                _integrationMode[PVgencount]
                
                message3 = "The integration mode is " + returnmodename(mode)
            
                
            except IndexError:
                
                mode = _integrationMode[0]
                message3 = "The integration mode is "+ returnmodename(mode)
                
            try:
    
                _NoParallel[PVgencount]
                
                message4 = "The number of PV modules in parallel is "+ str(parallel)
            
            except IndexError:
                
                parallel = _NoParallel[0]
                
                message4 = "The number of PV modules in parallel is "+ str(parallel) 
            
    
            try:
    
                _Noseries[PVgencount]
                message5 = "The number of PV modules in series is " + str(series)
    
                
            except IndexError:
                
                series = _Noseries[0]
                message5 = "The number of PV modules in series is " + str(series)
                
                
            message6 = "So " + name + " is made up of "+ str(series*parallel) + " PV modules"
                
            try:
                _costPVPerModule[PVgencount]
                message7 = "The cost per PV module is " + str(modelcost) + " US dollars" + " \n " +\
                "Therefore the total cost of "+ name +" is " + str(modelcost*float(parallel*series)) + " US dollars"
                
                
            except IndexError:
                
                 modelcost = _costPVPerModule[0]
                 message7 = "The cost per PV module is " + str(_costPVPerModule[0]) + " \n " +\
                 "Therefore the total cost of "+ name + " is " + str(modelcost*float(parallel*series)) + " in US dollars "
        
            try:
                _powerOutputPerModule[PVgencount]
    
                message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
                "Therefore the total power output of "+ name+" is " + str(powerout*float(parallel*series)) + " W "
    
                
            except IndexError:
                
                powerout = _powerOutputPerModule[0]
                message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
                "Therefore the total power output of "+ name+ " is " + str(powerout*float(parallel*series)) + " W "
            
            # The surface area of the surface that the PV panels cover are calculated from this equation.
    
            if sandiaMode_ == []:
                
                SA_solarcell = (powerout*parallel*series)/(1000*surface.getArea()*celleff)
            
            if sandiaMode_ != []:
                
                # Get Active Area from Sandria
                for lineCount,line in enumerate(sandiaMode_):
                    
                    if "!- Active Area {m2}" in line:
                        
                        # [^.0-9] remove all non numeric characters from a string using Regex!
                        areaPerPanel = float(re.sub(r'[^.0-9]', "", line))
                        
                        SA_solarcell = (parallel*series*areaPerPanel)/(surface.getArea())
                        
            if (SA_solarcell > 0.85) and (SA_solarcell < 1):
                
                warn = "According to your inputs solar panels cover more than 85% of the surface area of the Honeybee surface "+ surface.name+ "\n"+\
                "Are you sure that the power rating and the number of modules on this surface are correct? \n "+\
                "This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on. \n"+ \
                "Don't worry though this component will still run!"
                w = gh.GH_RuntimeMessageLevel.Warning
                ghenv.Component.AddRuntimeMessage(w, warn)
                
                
            if SA_solarcell >= 1:
                
                warn = "Error! According to your inputs solar panels cover more than 100% of the surface area of the Honeybee surface " + surface.name+"\n"+\
                "Are you sure that the power rating and the number of modules on each surface is correct? \n "+\
                "This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on."
                "This component will not run until the surface area is less than 100%."
                w = gh.GH_RuntimeMessageLevel.Error
                ghenv.Component.AddRuntimeMessage(w, warn)
                
                return 
                
            message1 = "According to the data you entered it is calculated that the modules of this PV generator cover roughly "+str(round(SA_solarcell*100,2))+ " % of the surface area of this surface"
                
            # A hb_EPShdSurface
            if surface.type == 6: ## A bit of a hack to get context surface name the same as being produced by EPShdSurface
                
                coordinatesList = surface.extractPoints()
    
                if type(coordinatesList[0])is not list and type(coordinatesList[0]) is not tuple: coordinatesList = [coordinatesList]
                
                for count in range(len(coordinatesList)):
                    
                    PVsurfacename = surface.name + '_' + `count`
                    surface.name = surface.name + '_' + `count`
    
                surface.PVgenlist.append(PV_gen(name,PVsurfacename,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_)) # Last three inputs are for instance method PV_performance
            
            # Not a hb_EPShdSurface
            else:
                
                surface.PVgenlist.append(PV_gen(name,surface.name,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_))
                
            # Assign the inverter to each PVgenerator.
            
            for PVgen in surface.PVgenlist:
                
                PVgen.inverter = PVinverter
            
            if sandiaMode_ == []:
                
                message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +message2+"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
            
            if sandiaMode_ != []:
            
                message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
            
            readmedatatree.Add(message,gh.Data.GH_Path(PVgencount))
                
            PVgencount = PVgencount+1

    except:
        
        # This catches an error when there is a missing member exception ie length of one of inputs is longer than 
        # number of Honeybee surfaces not sure how to just catch missing member exception!
        warn = "The length of a list of inputs into either _name_,_cellsEfficiency \n" + \
                "_integrationMode,_NoParallel,_Noseries,_costPVgen or _powerOutput \n" + \
                "is longer than the number of Honeybee surfaces connected to this component!\n" + \
                "e.g if you have 2 Honeybee surfaces you cannot have 3 values input into _costPVgen!\n" + \
                "Please check the inputs and try again!"
    
        print warn
        w = gh.GH_RuntimeMessageLevel.Warning
        ghenv.Component.AddRuntimeMessage(w, warn)
        
        return -1 

        
    ModifiedHBSurfaces = hb_hive.addToHoneybeeHive(HBSurfacesfromhive, ghenv.Component)
    
    return ModifiedHBSurfaces

Example 178

Project: socorro
Source File: setupdb_app.py
View license
    def main(self):

        database_name = self.config.database_name
        if not database_name:
            self.config.logger.error(
                '"database_name" cannot be an empty string'
            )
            return 1

        # superuser credentials for overall database
        superuser_pg_url = self.create_connection_url(
            'postgres',
            self.config.database_superusername,
            self.config.database_superuserpassword
        )

        # superuser credentials for working database
        superuser_normaldb_pg_url = self.create_connection_url(
            database_name,
            self.config.database_superusername,
            self.config.database_superuserpassword
        )

        # normal user credentials
        normal_user_pg_url = self.create_connection_url(
            database_name,
            self.config.database_username,
            self.config.database_password
        )

        # ensure that if on Heroku the the normal_user_pg_url and the
        # superuser_pg_url are the same
        if self.config.on_heroku and (normal_user_pg_url != superuser_pg_url):
            self.config.logger.error(
                'there is no superuser (%s) when using Heroku',
                self.config.database_superusername
            )
            return 1

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # table logging section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.unlogged:
            @compiles(CreateTable)
            def create_table(element, compiler, **kw):
                text = compiler.visit_create_table(element, **kw)
                text = re.sub("^\sCREATE(.*TABLE)",
                              lambda m: "CREATE UNLOGGED %s" %
                              m.group(1), text)
                return text

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # Postgres version check section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info(
            'Postgres version check section with %s',
            superuser_pg_url
        )
        with PostgreSQLAlchemyManager(
            superuser_pg_url,
            self.config.logger,
            autocommit=False
        ) as db:
            if not db.min_ver_check(90200):
                self.config.logger.error(
                    'unrecognized PostgreSQL version: %s',
                    db.version_string()
                )
                self.config.logger.error('Only 9.2+ is supported at this time')
                return 1

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # drop database section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # We can only do the following if the DB is not Heroku
        # XXX Might add the heroku commands for resetting a DB here
        if self.config.dropdb and not self.config.on_heroku:
            self.config.logger.info(
                'drop database section with %s',
                superuser_pg_url
            )
            with PostgreSQLAlchemyManager(
                superuser_pg_url,
                self.config.logger,
                autocommit=False
            ) as db:
                if 'test' not in database_name and not self.config.force:
                    confirm = raw_input(
                        'drop database %s [y/N]: ' % database_name)
                    if not confirm == "y":
                        self.config.logger.warn('NOT dropping table')
                        return 2
                db.drop_database(database_name)
                db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # create database section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.createdb:
            self.config.logger.info(
                'create database section with %s',
                superuser_pg_url
            )
            with PostgreSQLAlchemyManager(
                superuser_pg_url,
                self.config.logger,
                autocommit=False
            ) as db:
                db.create_database(database_name)
                if self.config.no_roles:
                    self.config.logger.info("Skipping role creation")
                else:
                    db.create_roles(self.config)
                db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database extensions section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info(
            'database extensions section with %s',
            superuser_normaldb_pg_url
        )
        with PostgreSQLAlchemyManager(
            superuser_normaldb_pg_url,
            self.config.logger,
            autocommit=False,
            on_heroku=self.config.on_heroku
        ) as db:
            db.setup_extensions()
            db.grant_public_schema_ownership(self.config.database_username)
            db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database schema section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.no_schema:
            self.config.logger.info("not adding a schema")
            return 0

        alembic_cfg = Config(self.config.alembic_config)
        alembic_cfg.set_main_option('sqlalchemy.url', normal_user_pg_url)

        self.config.logger.info(
            'database schema section with %s',
            normal_user_pg_url
        )
        with PostgreSQLAlchemyManager(
            normal_user_pg_url,
            self.config.logger,
            autocommit=False,
            on_heroku=self.config.on_heroku
        ) as db:
            # Order matters below
            db.turn_function_body_checks_off()
            db.load_raw_sql('types')
            db.load_raw_sql('procs')
            # We need to commit to make a type visible for table creation
            db.commit()

            db.create_tables()
            db.load_raw_sql('views')
            db.commit()

            if not self.config.get('no_staticdata'):
                self.import_staticdata(db)
            if self.config['fakedata']:
                self.generate_fakedata(db, self.config['fakedata_days'])
            db.commit()
            command.stamp(alembic_cfg, "heads")
            db.session.close()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database owner section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info(
            'database extensions section with %s',
            superuser_normaldb_pg_url
        )
        with PostgreSQLAlchemyManager(
            superuser_normaldb_pg_url,
            self.config.logger,
            autocommit=False,
            on_heroku=self.config.on_heroku
        ) as db:
            db.set_table_owner(self.config.database_username)
            db.set_default_owner(database_name, self.config.database_username)
            db.set_grants(self.config)  # config has user lists

        return 0

Example 179

Project: socorro
Source File: setupdb_app.py
View license
    def main(self):

        database_name = self.config.database_name
        if not database_name:
            self.config.logger.error(
                '"database_name" cannot be an empty string'
            )
            return 1

        # superuser credentials for overall database
        superuser_pg_url = self.create_connection_url(
            'postgres',
            self.config.database_superusername,
            self.config.database_superuserpassword
        )

        # superuser credentials for working database
        superuser_normaldb_pg_url = self.create_connection_url(
            database_name,
            self.config.database_superusername,
            self.config.database_superuserpassword
        )

        # normal user credentials
        normal_user_pg_url = self.create_connection_url(
            database_name,
            self.config.database_username,
            self.config.database_password
        )

        # ensure that if on Heroku the the normal_user_pg_url and the
        # superuser_pg_url are the same
        if self.config.on_heroku and (normal_user_pg_url != superuser_pg_url):
            self.config.logger.error(
                'there is no superuser (%s) when using Heroku',
                self.config.database_superusername
            )
            return 1

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # table logging section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.unlogged:
            @compiles(CreateTable)
            def create_table(element, compiler, **kw):
                text = compiler.visit_create_table(element, **kw)
                text = re.sub("^\sCREATE(.*TABLE)",
                              lambda m: "CREATE UNLOGGED %s" %
                              m.group(1), text)
                return text

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # Postgres version check section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info(
            'Postgres version check section with %s',
            superuser_pg_url
        )
        with PostgreSQLAlchemyManager(
            superuser_pg_url,
            self.config.logger,
            autocommit=False
        ) as db:
            if not db.min_ver_check(90200):
                self.config.logger.error(
                    'unrecognized PostgreSQL version: %s',
                    db.version_string()
                )
                self.config.logger.error('Only 9.2+ is supported at this time')
                return 1

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # drop database section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # We can only do the following if the DB is not Heroku
        # XXX Might add the heroku commands for resetting a DB here
        if self.config.dropdb and not self.config.on_heroku:
            self.config.logger.info(
                'drop database section with %s',
                superuser_pg_url
            )
            with PostgreSQLAlchemyManager(
                superuser_pg_url,
                self.config.logger,
                autocommit=False
            ) as db:
                if 'test' not in database_name and not self.config.force:
                    confirm = raw_input(
                        'drop database %s [y/N]: ' % database_name)
                    if not confirm == "y":
                        self.config.logger.warn('NOT dropping table')
                        return 2
                db.drop_database(database_name)
                db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # create database section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.createdb:
            self.config.logger.info(
                'create database section with %s',
                superuser_pg_url
            )
            with PostgreSQLAlchemyManager(
                superuser_pg_url,
                self.config.logger,
                autocommit=False
            ) as db:
                db.create_database(database_name)
                if self.config.no_roles:
                    self.config.logger.info("Skipping role creation")
                else:
                    db.create_roles(self.config)
                db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database extensions section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info(
            'database extensions section with %s',
            superuser_normaldb_pg_url
        )
        with PostgreSQLAlchemyManager(
            superuser_normaldb_pg_url,
            self.config.logger,
            autocommit=False,
            on_heroku=self.config.on_heroku
        ) as db:
            db.setup_extensions()
            db.grant_public_schema_ownership(self.config.database_username)
            db.commit()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database schema section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        if self.config.no_schema:
            self.config.logger.info("not adding a schema")
            return 0

        alembic_cfg = Config(self.config.alembic_config)
        alembic_cfg.set_main_option('sqlalchemy.url', normal_user_pg_url)

        self.config.logger.info(
            'database schema section with %s',
            normal_user_pg_url
        )
        with PostgreSQLAlchemyManager(
            normal_user_pg_url,
            self.config.logger,
            autocommit=False,
            on_heroku=self.config.on_heroku
        ) as db:
            # Order matters below
            db.turn_function_body_checks_off()
            db.load_raw_sql('types')
            db.load_raw_sql('procs')
            # We need to commit to make a type visible for table creation
            db.commit()

            db.create_tables()
            db.load_raw_sql('views')
            db.commit()

            if not self.config.get('no_staticdata'):
                self.import_staticdata(db)
            if self.config['fakedata']:
                self.generate_fakedata(db, self.config['fakedata_days'])
            db.commit()
            command.stamp(alembic_cfg, "heads")
            db.session.close()

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # database owner section
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        self.config.logger.info(
            'database extensions section with %s',
            superuser_normaldb_pg_url
        )
        with PostgreSQLAlchemyManager(
            superuser_normaldb_pg_url,
            self.config.logger,
            autocommit=False,
            on_heroku=self.config.on_heroku
        ) as db:
            db.set_table_owner(self.config.database_username)
            db.set_default_owner(database_name, self.config.database_username)
            db.set_grants(self.config)  # config has user lists

        return 0

Example 180

Project: python-armet
Source File: options.py
View license
    def __init__(self, meta, name, data, bases):
        """
        Initializes the options object and defaults configuration not
        specified.

        @param[in] meta
            Dictionary of the merged meta attributes.

        @param[in] name
            Name of the resource class this is being instantiataed for.
        """
        #! Whether to allow display of debugging information
        #! to the client.
        self.debug = meta.get('debug')
        if self.debug is None:
            self.debug = False

        #! Whether to not actualize a resource from the described class.
        #! Abstract resources are meant as generic base classes.
        #!
        #! @note
        #!      Abstract only counts if it is directly set on the resource.
        self.abstract = data.get('abstract')

        #! Name of the resource to use in URIs; defaults to the dasherized
        #! version of the camel cased class name (eg. SomethingHere becomes
        #! something-here). The defaulted version also strips a trailing
        #! Resource from the name (eg. SomethingHereResource still becomes
        #! something-here).
        self.name = meta.get('name')
        if self.name is None:
            # Generate a name for the resource if one is not provided.
            # PascalCaseThis => pascal-case-this
            dashed = utils.dasherize(name).strip()
            if dashed:
                # Strip off a trailing Resource as well.
                self.name = re.sub(r'-resource$', '', dashed)

            else:
                # Something went wrong; just use the class name
                self.name = name

        elif callable(self.name):
            # If name is callable; pass in the class name and use what
            # we got back.
            self.name = self.name(name)

        #! True if the resource is expected to operate asynchronously.
        #!
        #! The side-effect of setting this to True is that returning from
        #! `dispatch` (and by extension `get`, `post`, etc.) does not
        #! terminate the connection. You must invoke `response.close()` to
        #! terminate the connection.
        self.asynchronous = meta.get('asynchronous', False)

        #! Connectors to use to connect to the environment.
        #!
        #! This is a dictionary that maps hooks (keys) to the connector to use
        #! for the hook.
        #!
        #! There is only 1 hook available currently, 'http', and it is
        #! required.
        #!
        #! The available connectors are as follows:
        #!  - http:
        #!      > django
        #!      > flask
        #!
        #! They may be used as follows:
        #!
        #! @code
        #! from armet import resources
        #! class Resource(resources.Resource):
        #!     class Meta:
        #!         connectors = {'http': 'django'}
        #! @endcode
        #!
        #! Connectors may also be specified as full paths to the connector
        #! module (if a connector is located somewhere other than
        #! armet.connectors) as follows:
        #!
        #! @code
        #! from armet import resources
        #! class Resource(resources.Resource):
        #!     class Meta:
        #!         connectors = {'http': 'some.other.place'}
        #! @endcode
        self.connectors = connectors = _merge(meta, 'connectors', bases, {})

        if not connectors.get('http') and not self.abstract:
            raise ImproperlyConfigured('No valid HTTP connector was detected.')

        # Pull out the connectors and convert them into module references.
        for key in connectors:
            connector = connectors[key]
            if isinstance(connector, six.string_types):
                if connector in getattr(included_connectors, key):
                    # Included shortname; prepend base.
                    connectors[key] = 'armet.connectors.{}'.format(connector)

        #! Additional options to handle and merge into the meta object
        #! at the class object level.
        #!
        #! This should be a simple set/list/tuple of options.
        #!
        #! @code
        #! from armet import resources
        #! class Resource(resources.Resource):
        #!     class Meta:
        #!         options = {'color', 'plant'}
        #!
        #! @endcode
        self.options = options = _merge(meta, 'options', bases, {})
        for name in options:
            # Pull out each option and stick it on the meta.
            setattr(self, name, meta.get(name))

        #! Regular-expression patterns to apply to the request path
        #! and pull arguments and traversal out of it.
        #!
        #! @code
        #! from armet import resources
        #! class Resource(resources.Resource):
        #!     class Meta:
        #!         patterns = [
        #!             # Match nothing after the resource name.
        #!             r'^$',
        #!
        #!             # Match a slug after the resource name.
        #!             r'^/(?P<slug>[^/]+)/?$',
        #!         ]
        #!
        #! @endcode
        #!
        #! Named parameters get auto-attached to `self` on the resource
        #! instance.
        #!
        #! This may be a list of lists/tuples as well to indicate simple
        #! traversal.
        #!
        #! @code
        #! from armet import resources
        #! class Resource(resources.Resource):
        #!     class Meta:
        #!         patterns = [
        #!             # Match nothing and don't traverse.
        #!             (None, r'^$'),
        #!
        #!             # Match the word 'user' and traverse.
        #!             # The remainder of the path is taken or
        #!             # the named gruop, "path" (if the whole path was
        #!             # matched) or the last-matched group (if none are
        #!             # named "path")
        #!             (UserResource, r'^/user'),
        #!         ]
        #!
        #! @endcode
        self.patterns = meta.get('patterns', [])
        for index, pattern in enumerate(self.patterns):
            # Coerce simple form.
            if isinstance(pattern, six.string_types):
                pattern = (None, pattern)

            # Compile the expression.
            self.patterns[index] = (pattern[0], re.compile(pattern[1]))

        #! Trailing slash handling.
        #! The value indicates which URI is the canonical URI and the
        #! alternative URI is then made to redirect (with a 301) to the
        #! canonical URI.
        self.trailing_slash = meta.get('trailing_slash', True)

        #! List of allowed HTTP methods.
        #! If not provided and allowed_operations was provided instead
        #! the operations are appropriately mapped; else, the default
        #! configuration is provided.
        self.http_allowed_methods = meta.get('http_allowed_methods')
        if self.http_allowed_methods is None:
            self.http_allowed_methods = (
                'HEAD',
                'OPTIONS',
                'GET',
                'POST',
                'PUT',
                'PATCH',
                'DELETE',
                'LINK',
                'UNLINK'
            )

        #! List of allowed HTTP headers.
        #! This is used only to request or prevent CORS requests.
        self.http_allowed_headers = meta.get('http_allowed_headers')
        if self.http_allowed_headers is None:
            self.http_allowed_headers = (
                'Content-Type',
                'Authorization',
                'Accept',
                'Origin'
            )

        #! List of exposed HTTP headers.
        #! This is used only to show headers to the client.
        self.http_exposed_headers = meta.get('http_exposed_headers')
        if self.http_exposed_headers is None:
            self.http_exposed_headers = (
                'Content-Type',
                'Authorization',
                'Accept',
                'Origin'
            )

        #! List of allowed HTTP origins.
        #! This is used to request or prevent CORS requests.
        #! No CORS requests will be allowed, at-all, unless this
        #! property is set.
        #! NOTE: This can be set to '*' to indicate any origin.
        self.http_allowed_origins = meta.get('http_allowed_origins')
        if self.http_allowed_origins is None:
            self.http_allowed_origins = ()

        #! Whether to use legacy redirects or not to inform the
        #! client the resource is available elsewhere. Legacy redirects
        #! require a combination of 301 and 307 in which 307 is not cacheable.
        #! Modern redirecting uses 308 and is in effect 307 with cacheing.
        #! Unfortunately unknown 3xx codes are treated as
        #! a 300 (Multiple choices) in which the user is supposed to chose
        #! the alternate link so the client is not supposed to auto follow
        #! redirects. Ensure all supported clients understand 308 before
        #! turning off legacy redirecting.
        #! As of 19 March 2013 only Firefox supports it since a year ago.
        self.legacy_redirect = meta.get('legacy_redirect', True)

        #! Mapping of serializers known by this resource.
        #! Values may either be a string reference to the serializer type
        #! or an serializer class object.
        self.serializers = serializers = meta.get('serializers')
        if not serializers:
            self.serializers = {
                'json': 'armet.serializers.JSONSerializer',
                'url': 'armet.serializers.URLSerializer'
            }

        # Expand the serializer name references.
        for name, serializer in six.iteritems(self.serializers):
            if isinstance(serializer, six.string_types):
                segments = serializer.split('.')
                module = '.'.join(segments[:-1])
                module = import_module(module)
                self.serializers[name] = getattr(module, segments[-1])

        #! List of allowed serializers of the understood serializers.
        self.allowed_serializers = meta.get('allowed_serializers')
        if not self.allowed_serializers:
            self.allowed_serializers = tuple(self.serializers.keys())

        # Check to ensure that all allowed serializers are
        # understood serializers.
        for name in self.allowed_serializers:
            if name not in self.serializers:
                raise ImproperlyConfigured(
                    'The allowed serializer, {}, is not one of the '
                    'understood serializers'.format(name))

        #! Name of the default serializer of the list of
        #! understood serializers.
        self.default_serializer = meta.get('default_serializer')
        if not self.default_serializer:
            if 'json' in self.allowed_serializers:
                self.default_serializer = 'json'

            else:
                self.default_serializer = self.allowed_serializers[0]

        if self.default_serializer not in self.allowed_serializers:
            raise ImproperlyConfigured(
                'The chosen default serializer, {}, is not one of the '
                'allowed serializers'.format(self.default_serializer))

        #! Mapping of deserializers known by this resource.
        #! Values may either be a string reference to the deserializer type
        #! or an deserializer class object.
        self.deserializers = deserializers = meta.get('deserializers')
        if not deserializers:
            self.deserializers = {
                'json': 'armet.deserializers.JSONDeserializer',
                'url': 'armet.deserializers.URLDeserializer'
            }

        # Expand the deserializer name references.
        for name, deserializer in six.iteritems(self.deserializers):
            if isinstance(deserializer, six.string_types):
                segments = deserializer.split('.')
                module = '.'.join(segments[:-1])
                module = import_module(module)
                self.deserializers[name] = getattr(module, segments[-1])

        #! List of allowed deserializers of the understood deserializers.
        self.allowed_deserializers = meta.get('allowed_deserializers')
        if not self.allowed_deserializers:
            self.allowed_deserializers = tuple(self.deserializers.keys())

        # Check to ensure that all allowed deserializers are
        # understood deserializers.
        for name in self.allowed_deserializers:
            if name not in self.deserializers:
                raise ImproperlyConfigured(
                    'The allowed deserializer, {}, is not one of the '
                    'understood deserializers'.format(name))

        #! List of authentication protocols to attempt in sequence
        #! to determine the authenticated user.
        self.authentication = meta.get('authentication')
        if self.authentication is None:
            # Default to a single instance of pass-through authentication.
            self.authentication = (authentication.Authentication(),)

        #! The authorization protocol to attempt to use
        #! to determine the if the user can access or is otherwise
        #! authorized.
        self.authorization = meta.get('authorization')
        if self.authorization is None:
            # Default is the pass-through authorization.
            self.authorization = authorization.Authorization()

Example 181

Project: mapit
Source File: mapit_import.py
View license
    def handle_label(self, filename, **options):

        missing_options = []
        for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']:
            if options[k]:
                continue
            else:
                missing_options.append(k)
        if missing_options:
            message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument "
            message = message_start + " ".join('--{0}'.format(k) for k in missing_options)
            raise CommandError(message)

        generation_id = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code = options['country_code']
        override_name = options['override_name']
        name_field = options['name_field']
        if not (override_name or name_field):
            name_field = 'Name'
        override_code = options['override_code']
        code_field = options['code_field']
        code_type_code = options['code_type']
        encoding = options['encoding'] or 'utf-8'

        if name_field and override_name:
            raise CommandError("You must not specify both --name_field and --override_name")
        if code_field and override_code:
            raise CommandError("You must not specify both --code_field and --override_code")

        using_code = (code_field or override_code)
        if (using_code and not code_type_code) or (not using_code and code_type_code):
            raise CommandError(
                "If you want to save a code, specify --code_type and either --code_field or --override_code")
        try:
            area_type = Type.objects.get(code=area_type_code)
        except:
            type_desc = input('Please give a description for area type code %s: ' % area_type_code)
            area_type = Type(code=area_type_code, description=type_desc)
            if options['commit']:
                area_type.save()

        try:
            name_type = NameType.objects.get(code=name_type_code)
        except:
            name_desc = input('Please give a description for name type code %s: ' % name_type_code)
            name_type = NameType(code=name_type_code, description=name_desc)
            if options['commit']:
                name_type.save()

        try:
            country = Country.objects.get(code=country_code)
        except:
            country_name = input('Please give the name for country code %s: ' % country_code)
            country = Country(code=country_code, name=country_name)
            if options['commit']:
                country.save()

        if code_type_code:
            try:
                code_type = CodeType.objects.get(code=code_type_code)
            except:
                code_desc = input('Please give a description for code type %s: ' % code_type_code)
                code_type = CodeType(code=code_type_code, description=code_desc)
                if options['commit']:
                    code_type.save()

        self.stdout.write("Importing from %s" % filename)

        if not options['commit']:
            self.stdout.write('(will not save to db as --commit not specified)')

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.get(id=generation_id)

        def verbose(*args):
            if int(options['verbosity']) > 1:
                self.stdout.write(" ".join(str(a) for a in args))

        ds = DataSource(filename)
        layer = ds[0]
        if (override_name or override_code) and len(layer) > 1:
            message = (
                "Warning: you have specified an override %s and this file contains more than one feature; "
                "multiple areas with the same %s will be created")
            if override_name:
                self.stdout.write(message % ('name', 'name'))
            if override_code:
                self.stdout.write(message % ('code', 'code'))

        for feat in layer:

            if override_name:
                name = override_name
            else:
                name = None
                for nf in name_field.split(','):
                    try:
                        name = feat[nf].value
                        break
                    except:
                        pass
                if name is None:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find name using name field '%s' - should it be something else? "
                        "It will be one of these: %s. Specify which with --name_field" % (name_field, choices))
                try:
                    if not isinstance(name, six.text_type):
                        name = name.decode(encoding)
                except:
                    raise CommandError(
                        "Could not decode name using encoding '%s' - is it in another encoding? "
                        "Specify one with --encoding" % encoding)

            name = re.sub('\s+', ' ', name)
            if not name:
                if options['ignore_blank']:
                    continue
                raise Exception("Could not find a name to use for area")

            code = None
            if override_code:
                code = override_code
            elif code_field:
                try:
                    code = feat[code_field].value
                except:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find code using code field '%s' - should it be something else? "
                        "It will be one of these: %s. Specify which with --code_field" % (code_field, choices))

            self.stdout.write("  looking at '%s'%s" % (name, (' (%s)' % code) if code else ''))

            g = None
            if hasattr(feat, 'geom'):
                g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)

            try:
                if options['new']:  # Always want a new area
                    raise Area.DoesNotExist
                if code:
                    matching_message = "code %s of code type %s" % (code, code_type)
                    areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
                else:
                    matching_message = "name %s of area type %s" % (name, area_type)
                    areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
                if len(areas) == 0:
                    verbose("    the area was not found - creating a new one")
                    raise Area.DoesNotExist
                m = areas[0]
                verbose("    found the area")
                if options['preserve']:
                    # Find whether we need to create a new Area:
                    previous_geos_geometry = m.polygons.collect()
                    if m.generation_high < current_generation.id:
                        # Then it was missing in current_generation:
                        verbose("    area existed previously, but was missing from", current_generation)
                        raise Area.DoesNotExist
                    elif g is None:
                        if previous_geos_geometry is not None:
                            verbose("    area is now empty")
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area has remained empty")
                    elif previous_geos_geometry is None:
                        # It was empty in the previous generation:
                        verbose("    area was empty in", current_generation)
                        raise Area.DoesNotExist
                    else:
                        # Otherwise, create a new Area unless the
                        # polygons were the same in current_generation:
                        previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
                        new_geos_geometry = g.geos.simplify(tolerance=0)
                        create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
                        p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
                        verbose("    change in area is:", "%.03f%%" % (100 * p,))
                        if create_new_area:
                            verbose("    the area", m, "has changed, creating a new area due to --preserve")
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area remained the same")
                else:
                    # If --preserve is not specified, the code or the name must be unique:
                    if len(areas) > 1:
                        raise Area.MultipleObjectsReturned(
                            "There was more than one area with %s, and --preserve was not specified" % (
                                matching_message,))

            except Area.DoesNotExist:
                m = Area(
                    name=name,
                    type=area_type,
                    country=country,
                    # parent_area=parent_area,
                    generation_low=new_generation,
                    generation_high=new_generation,
                )
                if options['use_code_as_id'] and code:
                    m.id = int(code)

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
            m.generation_high = new_generation

            if options['fix_invalid_polygons'] and g is not None:
                # Make a GEOS geometry only to check for validity:
                geos_g = g.geos
                if not geos_g.valid:
                    geos_g = fix_invalid_geos_geometry(geos_g)
                    if geos_g is None:
                        self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name)
                        g = None
                    else:
                        g = geos_g.ogr

            poly = [g] if g is not None else []

            if options['commit']:
                m.save()
                m.names.update_or_create(type=name_type, defaults={'name': name})
                if code:
                    m.codes.update_or_create(type=code_type, defaults={'code': code})
                save_polygons({m.id: (m, poly)})

Example 182

Project: mapit
Source File: mapit_import.py
View license
    def handle_label(self, filename, **options):

        missing_options = []
        for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']:
            if options[k]:
                continue
            else:
                missing_options.append(k)
        if missing_options:
            message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument "
            message = message_start + " ".join('--{0}'.format(k) for k in missing_options)
            raise CommandError(message)

        generation_id = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code = options['country_code']
        override_name = options['override_name']
        name_field = options['name_field']
        if not (override_name or name_field):
            name_field = 'Name'
        override_code = options['override_code']
        code_field = options['code_field']
        code_type_code = options['code_type']
        encoding = options['encoding'] or 'utf-8'

        if name_field and override_name:
            raise CommandError("You must not specify both --name_field and --override_name")
        if code_field and override_code:
            raise CommandError("You must not specify both --code_field and --override_code")

        using_code = (code_field or override_code)
        if (using_code and not code_type_code) or (not using_code and code_type_code):
            raise CommandError(
                "If you want to save a code, specify --code_type and either --code_field or --override_code")
        try:
            area_type = Type.objects.get(code=area_type_code)
        except:
            type_desc = input('Please give a description for area type code %s: ' % area_type_code)
            area_type = Type(code=area_type_code, description=type_desc)
            if options['commit']:
                area_type.save()

        try:
            name_type = NameType.objects.get(code=name_type_code)
        except:
            name_desc = input('Please give a description for name type code %s: ' % name_type_code)
            name_type = NameType(code=name_type_code, description=name_desc)
            if options['commit']:
                name_type.save()

        try:
            country = Country.objects.get(code=country_code)
        except:
            country_name = input('Please give the name for country code %s: ' % country_code)
            country = Country(code=country_code, name=country_name)
            if options['commit']:
                country.save()

        if code_type_code:
            try:
                code_type = CodeType.objects.get(code=code_type_code)
            except:
                code_desc = input('Please give a description for code type %s: ' % code_type_code)
                code_type = CodeType(code=code_type_code, description=code_desc)
                if options['commit']:
                    code_type.save()

        self.stdout.write("Importing from %s" % filename)

        if not options['commit']:
            self.stdout.write('(will not save to db as --commit not specified)')

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.get(id=generation_id)

        def verbose(*args):
            if int(options['verbosity']) > 1:
                self.stdout.write(" ".join(str(a) for a in args))

        ds = DataSource(filename)
        layer = ds[0]
        if (override_name or override_code) and len(layer) > 1:
            message = (
                "Warning: you have specified an override %s and this file contains more than one feature; "
                "multiple areas with the same %s will be created")
            if override_name:
                self.stdout.write(message % ('name', 'name'))
            if override_code:
                self.stdout.write(message % ('code', 'code'))

        for feat in layer:

            if override_name:
                name = override_name
            else:
                name = None
                for nf in name_field.split(','):
                    try:
                        name = feat[nf].value
                        break
                    except:
                        pass
                if name is None:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find name using name field '%s' - should it be something else? "
                        "It will be one of these: %s. Specify which with --name_field" % (name_field, choices))
                try:
                    if not isinstance(name, six.text_type):
                        name = name.decode(encoding)
                except:
                    raise CommandError(
                        "Could not decode name using encoding '%s' - is it in another encoding? "
                        "Specify one with --encoding" % encoding)

            name = re.sub('\s+', ' ', name)
            if not name:
                if options['ignore_blank']:
                    continue
                raise Exception("Could not find a name to use for area")

            code = None
            if override_code:
                code = override_code
            elif code_field:
                try:
                    code = feat[code_field].value
                except:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find code using code field '%s' - should it be something else? "
                        "It will be one of these: %s. Specify which with --code_field" % (code_field, choices))

            self.stdout.write("  looking at '%s'%s" % (name, (' (%s)' % code) if code else ''))

            g = None
            if hasattr(feat, 'geom'):
                g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)

            try:
                if options['new']:  # Always want a new area
                    raise Area.DoesNotExist
                if code:
                    matching_message = "code %s of code type %s" % (code, code_type)
                    areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
                else:
                    matching_message = "name %s of area type %s" % (name, area_type)
                    areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
                if len(areas) == 0:
                    verbose("    the area was not found - creating a new one")
                    raise Area.DoesNotExist
                m = areas[0]
                verbose("    found the area")
                if options['preserve']:
                    # Find whether we need to create a new Area:
                    previous_geos_geometry = m.polygons.collect()
                    if m.generation_high < current_generation.id:
                        # Then it was missing in current_generation:
                        verbose("    area existed previously, but was missing from", current_generation)
                        raise Area.DoesNotExist
                    elif g is None:
                        if previous_geos_geometry is not None:
                            verbose("    area is now empty")
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area has remained empty")
                    elif previous_geos_geometry is None:
                        # It was empty in the previous generation:
                        verbose("    area was empty in", current_generation)
                        raise Area.DoesNotExist
                    else:
                        # Otherwise, create a new Area unless the
                        # polygons were the same in current_generation:
                        previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
                        new_geos_geometry = g.geos.simplify(tolerance=0)
                        create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
                        p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
                        verbose("    change in area is:", "%.03f%%" % (100 * p,))
                        if create_new_area:
                            verbose("    the area", m, "has changed, creating a new area due to --preserve")
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area remained the same")
                else:
                    # If --preserve is not specified, the code or the name must be unique:
                    if len(areas) > 1:
                        raise Area.MultipleObjectsReturned(
                            "There was more than one area with %s, and --preserve was not specified" % (
                                matching_message,))

            except Area.DoesNotExist:
                m = Area(
                    name=name,
                    type=area_type,
                    country=country,
                    # parent_area=parent_area,
                    generation_low=new_generation,
                    generation_high=new_generation,
                )
                if options['use_code_as_id'] and code:
                    m.id = int(code)

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
            m.generation_high = new_generation

            if options['fix_invalid_polygons'] and g is not None:
                # Make a GEOS geometry only to check for validity:
                geos_g = g.geos
                if not geos_g.valid:
                    geos_g = fix_invalid_geos_geometry(geos_g)
                    if geos_g is None:
                        self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name)
                        g = None
                    else:
                        g = geos_g.ogr

            poly = [g] if g is not None else []

            if options['commit']:
                m.save()
                m.names.update_or_create(type=name_type, defaults={'name': name})
                if code:
                    m.codes.update_or_create(type=code_type, defaults={'code': code})
                save_polygons({m.id: (m, poly)})

Example 183

View license
def process_office(office, commit, start_date, end_date, na_member_lookup, geocode_cache):
    global locationsnotfound, personnotfound

    # Ensure that all the required kinds and other objects exist:
    ok_constituency_office, _ = OrganisationKind.objects.get_or_create(
        slug='constituency-office',
        name='Constituency Office')
    ok_constituency_area, _ = OrganisationKind.objects.get_or_create(
        slug='constituency-area',
        name='Constituency Area')
    pk_constituency_office, _ = PlaceKind.objects.get_or_create(
        slug='constituency-office',
        name='Constituency Office')
    pk_constituency_area, _ = PlaceKind.objects.get_or_create(
        slug='constituency-area',
        name='Constituency Area')

    constituency_kinds = {
        'area': ok_constituency_area,
        'office': ok_constituency_office
    }

    constituency_place_kinds = {
        'area': pk_constituency_area,
        'office': pk_constituency_office
    }

    ck_address, _ = ContactKind.objects.get_or_create(
        slug='address',
        name='Address')
    ck_postal_address, _ = ContactKind.objects.get_or_create(
        slug='postal_address',
        name='Postal Address')
    ck_email, _ = ContactKind.objects.get_or_create(
        slug='email',
        name='Email')
    ck_fax, _ = ContactKind.objects.get_or_create(
        slug='fax',
        name='Fax')
    ck_telephone, _ = ContactKind.objects.get_or_create(
        slug='voice',
        name='Voice')

    ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
        name='has_office'
    )

    pt_constituency_contact, _ = PositionTitle.objects.get_or_create(
        slug='constituency-contact',
        name='Constituency Contact')
    pt_administrator, _ = PositionTitle.objects.get_or_create(
        slug='administrator',
        name='Administrator')
    pt_administrator_volunteer, _ = PositionTitle.objects.get_or_create(
        slug='administrator-volunteer',
        name='Administrator (volunteer)')
    pt_volunteer, _ = PositionTitle.objects.get_or_create(
        slug='volunteer',
        name='Volunteer')
    pt_coordinator, _ = PositionTitle.objects.get_or_create(
        slug='coordinator',
        name='Coordinator')
    pt_community_development_field_worker, _ = PositionTitle.objects.get_or_create(
        slug='community-development-field-worker',
        name='Community Development Field Worker')

    position_titles = {
        'Constituency Contact': pt_constituency_contact,
        'Administrator': pt_administrator,
        'Administrator (volunteer)': pt_administrator_volunteer,
        'Volunteer': pt_volunteer,
        'Coordinator': pt_coordinator,
        'Community Development Field Worker': pt_community_development_field_worker}

    ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
        name='has_office')

    contact_kinds = {
        'E-mail': ck_email,
        'Tel': ck_telephone,
        'Fax': ck_fax,
        'Physical Address': ck_address,
        'Postal Address': ck_postal_address
    }

    print "\n", office['Title']

    infosources = []
    if 'Sources' in office:
        source_url = office['Sources'][0]['Source URL']
        first = True
        for source in office['Sources']:
            print 'Adding InformationSource %s (%s)' % (
                source['Source URL'],
                source['Source Note']
            )

            infosources.append({
                'source_url': source['Source URL'],
                'source_note': source['Source Note']
            })

            if first:
                first = False
                continue

            source_url += ' and ' + source['Source URL']
    else:
        source_url = office['Source URL']
        source_note = office['Source Note']
        infosources.append({
            'source_url': source_url,
            'source_note': source_note
        })
        print 'Adding InformationSource %s (%s)' % (source_url, source_note)


    if ('Physical Address' in office) and (not 'South Africa' in office['Physical Address']) and (office['Physical Address']!='TBA'):
        office['Physical Address'] = office['Physical Address'] + ', South Africa'

    if ('Location' in office) and (office['Location']!='TBA') and (not office['Province'].lower() in office['Location'].lower()):
        office['Location'] = office['Location'] + ', ' + office['Province'] + ', South Africa'

    if ('Location' in office) and (not 'South Africa' in office['Location']) and (office['Location']!='TBA'):
        office['Location'] = office['Location'] + ', South Africa'

    #first determine whether the office already exists
    organisation = None
    try:
        organisation = Organisation.objects.get(
            name=office['Title']
        )
    except ObjectDoesNotExist:
        #check identifiers
        try:
            if 'identifiers' in office:
                for identifier_scheme, party_code in office['identifiers'].items():
                    identifier = Identifier.objects.get(
                        identifier=party_code,
                        scheme=identifier_scheme)
                    organisation = identifier.content_object
        except ObjectDoesNotExist:
            pass

    if organisation:  #existing office
        if organisation.name != office['Title']:
            print 'Changing name from %s to %s' % (organisation.name, office['Title'])

            if commit:
                organisation.name = office['Title']
                organisation.save()

        if organisation.ended != 'future':
            print 'Changing ended date from %s to future' % (organisation.ended)

            if commit:
                organisation.ended = 'future'
                organisation.save()

    else:
        print 'Creating new %s' % (office['Type'])

        if commit:
            organisation = Organisation.objects.create(
                name=office['Title'],
                slug=slugify(office['Title']),
                kind=constituency_kinds[office['Type']],
                started=start_date,
                ended='future')

    #information source
    if commit:
        for infosource in infosources:
            InformationSource.objects.get_or_create(
                source = infosource['source_url'],
                note = infosource['source_note'],
                entered = True,
                content_type=organisation_content_type,
                object_id=organisation.id
            )

    #relationship to party
    try:
        party = Organisation.objects.get(slug=office['Party'].lower())

        OrganisationRelationship.objects.get(
            organisation_a=party,
            organisation_b=organisation,
            kind=ork_has_office
        )

        #if the relationship exists nothing needs to change
        print 'Retaining relationship with %s' % (party)

    except (ObjectDoesNotExist, AttributeError):
        print 'Adding relationship with %s' % (party)

        if commit:
            OrganisationRelationship.objects.create(
                organisation_a=party,
                organisation_b=organisation,
                kind=ork_has_office
            )

    office_fields = [
        'E-mail',
        'Tel',
        'Fax',
        'Physical Address',
        'Postal Address'
    ]

    for field in office_fields:
        if field in office:
            try:
                if not organisation:
                    raise ObjectDoesNotExist

                contact = Contact.objects.get(
                    object_id=organisation.id,
                    content_type=organisation_content_type,
                    kind=contact_kinds[field])
                if office[field] != contact.value:
                    print 'Changing %s from %s to %s' % (field, contact.value, office[field])

                    if commit:
                        contact.value = office[field]
                        contact.save()

                print 'Updating contact source to %s' % (source_url)
                if commit:
                    contact.source = source_url
                    contact.save()

            except ObjectDoesNotExist:
                print 'Creating new contact (%s: %s)' % (field, office[field])

                if commit:
                    Contact.objects.create(
                        object_id=organisation.id,
                        content_type=organisation_content_type,
                        kind=contact_kinds[field],
                        value=office[field],
                        source=source_url)

        else:
            try:
                if not organisation:
                    raise ObjectDoesNotExist

                contact = Contact.objects.get(
                    object_id=organisation.id,
                    content_type=organisation_content_type,
                    kind=contact_kinds[field])
                print 'Deleting', contact

                if commit:
                    contact.delete()

            except ObjectDoesNotExist:
                pass

    if 'Municipality' in office:
        mapit_municipality = get_mapit_municipality(office['Municipality'], office.get('Province', ''))

        if mapit_municipality:
            place_name = u'Municipality associated with ' + office['Title']
            try:
                if not organisation:
                    raise ObjectDoesNotExist

                place = Place.objects.get(
                    name__startswith=u'Municipality associated with ',
                    organisation=organisation)

                if place.name != place_name:
                    'Changing municipality association name from %s to %s' % (place.name, place_name)

                    if commit:
                        place.name = place_name
                        place.save()

                if place.mapit_area != mapit_municipality:
                    print 'Changing municipality mapit association from %s to %s' % (place.mapit_area, mapit_municipality)

                    if commit:
                        place.mapit_area = mapit_municipality
                        place.save()

            except ObjectDoesNotExist:
                print 'Create municipality association'
                to_add = {
                    'name': place_name,
                    'slug': slugify(place_name),
                    'kind': constituency_place_kinds[office['Type']],
                    'mapit_area': mapit_municipality,}
                print to_add

                if commit:
                    Place.objects.create(
                        name=to_add['name'],
                        slug=to_add['slug'],
                        kind=to_add['kind'],
                        mapit_area=to_add['mapit_area'],
                        organisation=organisation)

    if 'manual_lonlat' in office or 'Physical Address' in office or 'Location' in office:
        reference_location = ''
        try:
            if 'manual_lonlat' in office:
                #FIXME implement
                print 'manual'
            elif 'Location' in office:
                reference_location = office['Location']
                lon, lat, geocode_cache = geocode(office['Location'], geocode_cache, VERBOSE)
            elif 'Physical Address' in office:
                reference_location = office['Physical Address']
                #geocode physical address
                lon, lat, geocode_cache = geocode(office['Physical Address'], geocode_cache, VERBOSE)

            location = Point(lon, lat)
            if office['Type']=='area':
                name = u'Unknown sub-area of %s known as %s' % (office['Province'], office['Title'])
            else:
                name = u'Approximate position of ' + office['Title']

            try:
                if not organisation:
                    raise ObjectDoesNotExist

                if office['Type'] == 'area':
                    place = Place.objects.get(
                        name__startswith=u'Unknown sub-area of',
                        organisation=organisation)

                else:
                    place = Place.objects.get(
                        name__startswith=u'Approximate position of ',
                        organisation=organisation)

                if place.location != location:
                    print 'Changing location from %s to %s' % (place.location, location)

                    debug_location_change(place.location, location)

                    if commit:
                        place.location = location
                        place.save()

                if place.name != name:
                    print 'Changing location name from %s to %s' % (place.name, name)

                    if commit:
                        place.name = name
                        place.save()

            except ObjectDoesNotExist:
                print 'Create constituency location'

                if commit:
                    Place.objects.create(
                        name=name,
                        slug=slugify(name),
                        organisation=organisation,
                        location=location,
                        kind=constituency_place_kinds[office['Type']])

        except LocationNotFound:
            locationsnotfound.append([office['Title'], reference_location])
            print "XXX no results found for: " + reference_location

    else:
        print 'No office/area location specified'

    people_to_keep = []
    if 'People' in office:
        for person in office['People']:
            #person matching needs to be improved - for now attempt
            #find_pombola_person (from
            #south_africa_import_constituency_offices command) otherwise
            #direct match.
            pombola_person = find_pombola_person(person['Name'], na_member_lookup, VERBOSE)
            if not pombola_person:
                #use filter().distinct() instead of get due to multiple
                #rows being returned
                pombola_person = Person.objects.filter(
                    Q(legal_name=person['Name']) |
                    Q(alternative_names__alternative_name=person['Name'])).distinct()
                if len(pombola_person)==0:
                    pombola_person = None
                else:
                    pombola_person = pombola_person[0]

            #check person currently holds office
            accept_person = True
            if pombola_person and person['Position']=='Constituency Contact':
                position_check = Position.objects.filter(
                    person=pombola_person,
                    organisation__kind__slug__in=['parliament', 'provincial-legislature'])
                if len(position_check)==0:
                    accept_person=False
                    print '%s is not an MP or MPL' % (pombola_person.name)

            if pombola_person and accept_person:
                #check if the position already exists
                positions = Position.objects.filter(
                    person=pombola_person,
                    organisation=organisation,
                    title=position_titles[person['Position']]
                    ).currently_active()

                if not positions:
                    print 'Creating position (%s) for %s' % (person['Position'], pombola_person)

                    if commit:
                        positiontitle, _ = PositionTitle.objects.get_or_create(
                            name=person['Position'])

                        position = Position.objects.create(
                            person=pombola_person,
                            organisation=organisation,
                            title=positiontitle,
                            start_date=start_date,
                            end_date='future')

                        people_to_keep.append(position.id)

                        #information source
                        for infosource in infosources:
                            InformationSource.objects.get_or_create(
                                source = infosource['source_url'],
                                note = infosource['source_note'],
                                entered = True,
                                content_type=position_content_type,
                                object_id=position.id
                            )

                for position in positions:
                    people_to_keep.append(position.id)
                    print 'Retaining %s' % (position)

                #check cell number
                if 'Cell' in person:
                    contacts = Contact.objects.filter(
                        object_id=pombola_person.id,
                        content_type=person_content_type,
                        kind=ck_telephone)

                    #if only one cell exists replace
                    if len(contacts)==1:
                        print 'Updating tel number for', pombola_person, 'from', contacts[0].value, 'to', person['Cell']

                        if commit:
                            contacts[0].value = person['Cell']
                            contacts[0].save()
                    else:
                        #otherwise check if the cell
                        #has already been loaded
                        add = True
                        #check if already exists
                        for contact in contacts:
                            existing = re.sub('[^0-9]', '', contact.value)
                            new = re.sub('[^0-9]', '', person['Cell'])
                            if existing==new:
                                add = False

                        if add:
                            print pombola_person
                            print person['Cell']
                            print 'Adding tel number for', pombola_person, '-', person['Cell']

                            if commit:
                                Contact.objects.create(
                                    object_id=pombola_person.id,
                                    content_type=person_content_type,
                                    kind=ck_telephone,
                                    value=person['Cell'],
                                    source=source_url)

                    print 'Updating contact source to %s' % (source_url)

                    if commit:
                        for contact in contacts:
                            contact.source = source_url
                            contact.save()

                #check email
                if 'Email' in person:
                    contacts = Contact.objects.filter(
                        object_id=pombola_person.id,
                        content_type=person_content_type,
                        kind=ck_email)

                    #if only one email exists replace
                    if len(contacts)==1:
                        print 'Updating email for %s from %s to %s' % (pombola_person, contacts[0].value, person['Email'])

                        if commit:
                            contacts[0].value = person['Email']
                            contacts[0].save()
                    else:
                        #otherwise check if the email has already been
                        #loaded
                        add = True
                        #check if already exists
                        for contact in contacts:
                            existing = contact.value
                            new = person['Email']
                            if existing==new:
                                add = False

                        if add:
                            print 'Adding email for %s: %s' % (pombola_person, person['Email'])

                            if commit:
                                Contact.objects.create(
                                    object_id=pombola_person.id,
                                    content_type=person_content_type,
                                    kind=ck_email,
                                    value=person['Email'],
                                    source=source_url)

                    print 'Updating contact source to %s' % (source_url)

                    if commit:
                        for contact in contacts:
                            contact.source = source_url
                            contact.save()

                #check alternative name
                if 'Alternative Name' in person:
                    try:
                        AlternativePersonName.objects.get(
                            person=pombola_person,
                            alternative_name=person['Alternative Name'])
                    except ObjectDoesNotExist:
                        print 'Adding alternative name for %s: %s' % (pombola_person, person['Alternative Name'].encode('utf-8'))

                        if commit:
                            AlternativePersonName.objects.create(
                                person=pombola_person,
                                alternative_name=person['Alternative Name'])

            if not pombola_person:
                if person['Position'] == 'Constituency Contact':
                    personnotfound.append([office['Title'], person['Name']])
                    print 'Failed to match representative', person['Name']
                else:
                    print 'Creating person (%s) with position (%s)' % (person['Name'], person['Position'])

                    if commit:
                        create_person = Person.objects.create(
                            legal_name=person['Name'],
                            slug=slugify(person['Name']))

                        Position.objects.create(
                            person=create_person,
                            organisation=organisation,
                            title=position_titles[person['Position']],
                            start_date=start_date,
                            end_date='future')

                    if 'Cell' in person:
                        print 'Adding cell number %s' % (person['Cell'])

                        if commit:
                            Contact.objects.create(
                                object_id=create_person.id,
                                content_type=person_content_type,
                                kind=ck_telephone,
                                value=person['Cell'],
                                source=source_url)

                    if 'Alternative Name' in person:
                        print 'Adding alternative name %s' % (unicode(person['Alternative Name'], 'utf-8'))

                        if commit:
                            AlternativePersonName.objects.create(
                                person=create_person,
                                alternative_name=person['Alternative Name'])

    #find the positions to end
    if organisation:
        positions = Position.objects.filter(
            organisation=organisation
            ).exclude(id__in=people_to_keep).currently_active()

        for position in positions:
            print 'Ending %s' % (position)

            if commit:
                position.end_date = end_date
                position.save()

    #FIXME: check summary, kind, started, ended,
    #identifiers (not expected at present)

    return organisation

Example 184

View license
def process_office(office, commit, start_date, end_date, na_member_lookup, geocode_cache):
    global locationsnotfound, personnotfound

    # Ensure that all the required kinds and other objects exist:
    ok_constituency_office, _ = OrganisationKind.objects.get_or_create(
        slug='constituency-office',
        name='Constituency Office')
    ok_constituency_area, _ = OrganisationKind.objects.get_or_create(
        slug='constituency-area',
        name='Constituency Area')
    pk_constituency_office, _ = PlaceKind.objects.get_or_create(
        slug='constituency-office',
        name='Constituency Office')
    pk_constituency_area, _ = PlaceKind.objects.get_or_create(
        slug='constituency-area',
        name='Constituency Area')

    constituency_kinds = {
        'area': ok_constituency_area,
        'office': ok_constituency_office
    }

    constituency_place_kinds = {
        'area': pk_constituency_area,
        'office': pk_constituency_office
    }

    ck_address, _ = ContactKind.objects.get_or_create(
        slug='address',
        name='Address')
    ck_postal_address, _ = ContactKind.objects.get_or_create(
        slug='postal_address',
        name='Postal Address')
    ck_email, _ = ContactKind.objects.get_or_create(
        slug='email',
        name='Email')
    ck_fax, _ = ContactKind.objects.get_or_create(
        slug='fax',
        name='Fax')
    ck_telephone, _ = ContactKind.objects.get_or_create(
        slug='voice',
        name='Voice')

    ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
        name='has_office'
    )

    pt_constituency_contact, _ = PositionTitle.objects.get_or_create(
        slug='constituency-contact',
        name='Constituency Contact')
    pt_administrator, _ = PositionTitle.objects.get_or_create(
        slug='administrator',
        name='Administrator')
    pt_administrator_volunteer, _ = PositionTitle.objects.get_or_create(
        slug='administrator-volunteer',
        name='Administrator (volunteer)')
    pt_volunteer, _ = PositionTitle.objects.get_or_create(
        slug='volunteer',
        name='Volunteer')
    pt_coordinator, _ = PositionTitle.objects.get_or_create(
        slug='coordinator',
        name='Coordinator')
    pt_community_development_field_worker, _ = PositionTitle.objects.get_or_create(
        slug='community-development-field-worker',
        name='Community Development Field Worker')

    position_titles = {
        'Constituency Contact': pt_constituency_contact,
        'Administrator': pt_administrator,
        'Administrator (volunteer)': pt_administrator_volunteer,
        'Volunteer': pt_volunteer,
        'Coordinator': pt_coordinator,
        'Community Development Field Worker': pt_community_development_field_worker}

    ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
        name='has_office')

    contact_kinds = {
        'E-mail': ck_email,
        'Tel': ck_telephone,
        'Fax': ck_fax,
        'Physical Address': ck_address,
        'Postal Address': ck_postal_address
    }

    print "\n", office['Title']

    infosources = []
    if 'Sources' in office:
        source_url = office['Sources'][0]['Source URL']
        first = True
        for source in office['Sources']:
            print 'Adding InformationSource %s (%s)' % (
                source['Source URL'],
                source['Source Note']
            )

            infosources.append({
                'source_url': source['Source URL'],
                'source_note': source['Source Note']
            })

            if first:
                first = False
                continue

            source_url += ' and ' + source['Source URL']
    else:
        source_url = office['Source URL']
        source_note = office['Source Note']
        infosources.append({
            'source_url': source_url,
            'source_note': source_note
        })
        print 'Adding InformationSource %s (%s)' % (source_url, source_note)


    if ('Physical Address' in office) and (not 'South Africa' in office['Physical Address']) and (office['Physical Address']!='TBA'):
        office['Physical Address'] = office['Physical Address'] + ', South Africa'

    if ('Location' in office) and (office['Location']!='TBA') and (not office['Province'].lower() in office['Location'].lower()):
        office['Location'] = office['Location'] + ', ' + office['Province'] + ', South Africa'

    if ('Location' in office) and (not 'South Africa' in office['Location']) and (office['Location']!='TBA'):
        office['Location'] = office['Location'] + ', South Africa'

    #first determine whether the office already exists
    organisation = None
    try:
        organisation = Organisation.objects.get(
            name=office['Title']
        )
    except ObjectDoesNotExist:
        #check identifiers
        try:
            if 'identifiers' in office:
                for identifier_scheme, party_code in office['identifiers'].items():
                    identifier = Identifier.objects.get(
                        identifier=party_code,
                        scheme=identifier_scheme)
                    organisation = identifier.content_object
        except ObjectDoesNotExist:
            pass

    if organisation:  #existing office
        if organisation.name != office['Title']:
            print 'Changing name from %s to %s' % (organisation.name, office['Title'])

            if commit:
                organisation.name = office['Title']
                organisation.save()

        if organisation.ended != 'future':
            print 'Changing ended date from %s to future' % (organisation.ended)

            if commit:
                organisation.ended = 'future'
                organisation.save()

    else:
        print 'Creating new %s' % (office['Type'])

        if commit:
            organisation = Organisation.objects.create(
                name=office['Title'],
                slug=slugify(office['Title']),
                kind=constituency_kinds[office['Type']],
                started=start_date,
                ended='future')

    #information source
    if commit:
        for infosource in infosources:
            InformationSource.objects.get_or_create(
                source = infosource['source_url'],
                note = infosource['source_note'],
                entered = True,
                content_type=organisation_content_type,
                object_id=organisation.id
            )

    #relationship to party
    try:
        party = Organisation.objects.get(slug=office['Party'].lower())

        OrganisationRelationship.objects.get(
            organisation_a=party,
            organisation_b=organisation,
            kind=ork_has_office
        )

        #if the relationship exists nothing needs to change
        print 'Retaining relationship with %s' % (party)

    except (ObjectDoesNotExist, AttributeError):
        print 'Adding relationship with %s' % (party)

        if commit:
            OrganisationRelationship.objects.create(
                organisation_a=party,
                organisation_b=organisation,
                kind=ork_has_office
            )

    office_fields = [
        'E-mail',
        'Tel',
        'Fax',
        'Physical Address',
        'Postal Address'
    ]

    for field in office_fields:
        if field in office:
            try:
                if not organisation:
                    raise ObjectDoesNotExist

                contact = Contact.objects.get(
                    object_id=organisation.id,
                    content_type=organisation_content_type,
                    kind=contact_kinds[field])
                if office[field] != contact.value:
                    print 'Changing %s from %s to %s' % (field, contact.value, office[field])

                    if commit:
                        contact.value = office[field]
                        contact.save()

                print 'Updating contact source to %s' % (source_url)
                if commit:
                    contact.source = source_url
                    contact.save()

            except ObjectDoesNotExist:
                print 'Creating new contact (%s: %s)' % (field, office[field])

                if commit:
                    Contact.objects.create(
                        object_id=organisation.id,
                        content_type=organisation_content_type,
                        kind=contact_kinds[field],
                        value=office[field],
                        source=source_url)

        else:
            try:
                if not organisation:
                    raise ObjectDoesNotExist

                contact = Contact.objects.get(
                    object_id=organisation.id,
                    content_type=organisation_content_type,
                    kind=contact_kinds[field])
                print 'Deleting', contact

                if commit:
                    contact.delete()

            except ObjectDoesNotExist:
                pass

    if 'Municipality' in office:
        mapit_municipality = get_mapit_municipality(office['Municipality'], office.get('Province', ''))

        if mapit_municipality:
            place_name = u'Municipality associated with ' + office['Title']
            try:
                if not organisation:
                    raise ObjectDoesNotExist

                place = Place.objects.get(
                    name__startswith=u'Municipality associated with ',
                    organisation=organisation)

                if place.name != place_name:
                    'Changing municipality association name from %s to %s' % (place.name, place_name)

                    if commit:
                        place.name = place_name
                        place.save()

                if place.mapit_area != mapit_municipality:
                    print 'Changing municipality mapit association from %s to %s' % (place.mapit_area, mapit_municipality)

                    if commit:
                        place.mapit_area = mapit_municipality
                        place.save()

            except ObjectDoesNotExist:
                print 'Create municipality association'
                to_add = {
                    'name': place_name,
                    'slug': slugify(place_name),
                    'kind': constituency_place_kinds[office['Type']],
                    'mapit_area': mapit_municipality,}
                print to_add

                if commit:
                    Place.objects.create(
                        name=to_add['name'],
                        slug=to_add['slug'],
                        kind=to_add['kind'],
                        mapit_area=to_add['mapit_area'],
                        organisation=organisation)

    if 'manual_lonlat' in office or 'Physical Address' in office or 'Location' in office:
        reference_location = ''
        try:
            if 'manual_lonlat' in office:
                #FIXME implement
                print 'manual'
            elif 'Location' in office:
                reference_location = office['Location']
                lon, lat, geocode_cache = geocode(office['Location'], geocode_cache, VERBOSE)
            elif 'Physical Address' in office:
                reference_location = office['Physical Address']
                #geocode physical address
                lon, lat, geocode_cache = geocode(office['Physical Address'], geocode_cache, VERBOSE)

            location = Point(lon, lat)
            if office['Type']=='area':
                name = u'Unknown sub-area of %s known as %s' % (office['Province'], office['Title'])
            else:
                name = u'Approximate position of ' + office['Title']

            try:
                if not organisation:
                    raise ObjectDoesNotExist

                if office['Type'] == 'area':
                    place = Place.objects.get(
                        name__startswith=u'Unknown sub-area of',
                        organisation=organisation)

                else:
                    place = Place.objects.get(
                        name__startswith=u'Approximate position of ',
                        organisation=organisation)

                if place.location != location:
                    print 'Changing location from %s to %s' % (place.location, location)

                    debug_location_change(place.location, location)

                    if commit:
                        place.location = location
                        place.save()

                if place.name != name:
                    print 'Changing location name from %s to %s' % (place.name, name)

                    if commit:
                        place.name = name
                        place.save()

            except ObjectDoesNotExist:
                print 'Create constituency location'

                if commit:
                    Place.objects.create(
                        name=name,
                        slug=slugify(name),
                        organisation=organisation,
                        location=location,
                        kind=constituency_place_kinds[office['Type']])

        except LocationNotFound:
            locationsnotfound.append([office['Title'], reference_location])
            print "XXX no results found for: " + reference_location

    else:
        print 'No office/area location specified'

    people_to_keep = []
    if 'People' in office:
        for person in office['People']:
            #person matching needs to be improved - for now attempt
            #find_pombola_person (from
            #south_africa_import_constituency_offices command) otherwise
            #direct match.
            pombola_person = find_pombola_person(person['Name'], na_member_lookup, VERBOSE)
            if not pombola_person:
                #use filter().distinct() instead of get due to multiple
                #rows being returned
                pombola_person = Person.objects.filter(
                    Q(legal_name=person['Name']) |
                    Q(alternative_names__alternative_name=person['Name'])).distinct()
                if len(pombola_person)==0:
                    pombola_person = None
                else:
                    pombola_person = pombola_person[0]

            #check person currently holds office
            accept_person = True
            if pombola_person and person['Position']=='Constituency Contact':
                position_check = Position.objects.filter(
                    person=pombola_person,
                    organisation__kind__slug__in=['parliament', 'provincial-legislature'])
                if len(position_check)==0:
                    accept_person=False
                    print '%s is not an MP or MPL' % (pombola_person.name)

            if pombola_person and accept_person:
                #check if the position already exists
                positions = Position.objects.filter(
                    person=pombola_person,
                    organisation=organisation,
                    title=position_titles[person['Position']]
                    ).currently_active()

                if not positions:
                    print 'Creating position (%s) for %s' % (person['Position'], pombola_person)

                    if commit:
                        positiontitle, _ = PositionTitle.objects.get_or_create(
                            name=person['Position'])

                        position = Position.objects.create(
                            person=pombola_person,
                            organisation=organisation,
                            title=positiontitle,
                            start_date=start_date,
                            end_date='future')

                        people_to_keep.append(position.id)

                        #information source
                        for infosource in infosources:
                            InformationSource.objects.get_or_create(
                                source = infosource['source_url'],
                                note = infosource['source_note'],
                                entered = True,
                                content_type=position_content_type,
                                object_id=position.id
                            )

                for position in positions:
                    people_to_keep.append(position.id)
                    print 'Retaining %s' % (position)

                #check cell number
                if 'Cell' in person:
                    contacts = Contact.objects.filter(
                        object_id=pombola_person.id,
                        content_type=person_content_type,
                        kind=ck_telephone)

                    #if only one cell exists replace
                    if len(contacts)==1:
                        print 'Updating tel number for', pombola_person, 'from', contacts[0].value, 'to', person['Cell']

                        if commit:
                            contacts[0].value = person['Cell']
                            contacts[0].save()
                    else:
                        #otherwise check if the cell
                        #has already been loaded
                        add = True
                        #check if already exists
                        for contact in contacts:
                            existing = re.sub('[^0-9]', '', contact.value)
                            new = re.sub('[^0-9]', '', person['Cell'])
                            if existing==new:
                                add = False

                        if add:
                            print pombola_person
                            print person['Cell']
                            print 'Adding tel number for', pombola_person, '-', person['Cell']

                            if commit:
                                Contact.objects.create(
                                    object_id=pombola_person.id,
                                    content_type=person_content_type,
                                    kind=ck_telephone,
                                    value=person['Cell'],
                                    source=source_url)

                    print 'Updating contact source to %s' % (source_url)

                    if commit:
                        for contact in contacts:
                            contact.source = source_url
                            contact.save()

                #check email
                if 'Email' in person:
                    contacts = Contact.objects.filter(
                        object_id=pombola_person.id,
                        content_type=person_content_type,
                        kind=ck_email)

                    #if only one email exists replace
                    if len(contacts)==1:
                        print 'Updating email for %s from %s to %s' % (pombola_person, contacts[0].value, person['Email'])

                        if commit:
                            contacts[0].value = person['Email']
                            contacts[0].save()
                    else:
                        #otherwise check if the email has already been
                        #loaded
                        add = True
                        #check if already exists
                        for contact in contacts:
                            existing = contact.value
                            new = person['Email']
                            if existing==new:
                                add = False

                        if add:
                            print 'Adding email for %s: %s' % (pombola_person, person['Email'])

                            if commit:
                                Contact.objects.create(
                                    object_id=pombola_person.id,
                                    content_type=person_content_type,
                                    kind=ck_email,
                                    value=person['Email'],
                                    source=source_url)

                    print 'Updating contact source to %s' % (source_url)

                    if commit:
                        for contact in contacts:
                            contact.source = source_url
                            contact.save()

                #check alternative name
                if 'Alternative Name' in person:
                    try:
                        AlternativePersonName.objects.get(
                            person=pombola_person,
                            alternative_name=person['Alternative Name'])
                    except ObjectDoesNotExist:
                        print 'Adding alternative name for %s: %s' % (pombola_person, person['Alternative Name'].encode('utf-8'))

                        if commit:
                            AlternativePersonName.objects.create(
                                person=pombola_person,
                                alternative_name=person['Alternative Name'])

            if not pombola_person:
                if person['Position'] == 'Constituency Contact':
                    personnotfound.append([office['Title'], person['Name']])
                    print 'Failed to match representative', person['Name']
                else:
                    print 'Creating person (%s) with position (%s)' % (person['Name'], person['Position'])

                    if commit:
                        create_person = Person.objects.create(
                            legal_name=person['Name'],
                            slug=slugify(person['Name']))

                        Position.objects.create(
                            person=create_person,
                            organisation=organisation,
                            title=position_titles[person['Position']],
                            start_date=start_date,
                            end_date='future')

                    if 'Cell' in person:
                        print 'Adding cell number %s' % (person['Cell'])

                        if commit:
                            Contact.objects.create(
                                object_id=create_person.id,
                                content_type=person_content_type,
                                kind=ck_telephone,
                                value=person['Cell'],
                                source=source_url)

                    if 'Alternative Name' in person:
                        print 'Adding alternative name %s' % (unicode(person['Alternative Name'], 'utf-8'))

                        if commit:
                            AlternativePersonName.objects.create(
                                person=create_person,
                                alternative_name=person['Alternative Name'])

    #find the positions to end
    if organisation:
        positions = Position.objects.filter(
            organisation=organisation
            ).exclude(id__in=people_to_keep).currently_active()

        for position in positions:
            print 'Ending %s' % (position)

            if commit:
                position.end_date = end_date
                position.save()

    #FIXME: check summary, kind, started, ended,
    #identifiers (not expected at present)

    return organisation

Example 185

Project: yournextrepresentative
Source File: conf.py
View license
def get_settings(conf_file_leafname, election_app=None, tests=False):
    conf = get_conf(conf_file_leafname)

    debug = bool(int(conf.get('STAGING')))

    # Get the requested ELECTION_APP:
    if election_app is None:
        election_app = conf['ELECTION_APP']
    election_app_fully_qualified = 'elections.' + election_app
    election_settings_module = election_app_fully_qualified + '.settings'
    elections_module = importlib.import_module(election_settings_module)

    language_code = conf.get('LANGUAGE_CODE', 'en-gb')

    # Internationalization
    # https://docs.djangoproject.com/en/1.6/topics/i18n/
    locale_paths = [
        join(BASE_DIR, 'locale')
    ]
    # The code below sets LANGUAGES to only those we have translations
    # for, so at the time of writing that will be:
    #   [('en', 'English'), ('es-ar', 'Argentinian Spanish')]
    # whereas the default setting is a long list of languages which
    # includes:
    #   ('es', 'Spanish').
    # If someone's browser sends 'Accept-Language: es', that means that it
    # will be found in this list, but since there are no translations for 'es'
    # it'll fall back to LANGUAGE_CODE.  However, if there is no 'es' in
    # LANGUAGES, then Django will attempt to do a best match, so if
    # Accept-Language is 'es' then it will use the 'es-ar' translation.  We think
    # this is generally desirable (e.g. so someone can see YourNextMP in Spanish
    # if their browser asks for Spanish).
    languages = [
        l for l in LANGUAGES
        if exists(join(locale_paths[0], to_locale(l[0])))
    ]
    languages.append(('cy-gb', 'Welsh'))
    languages.append(('es-cr', 'Costa Rican Spanish'))

    # we need this to make the language switcher work because Django doesn't
    # have any language information for these so if you try to tell it to
    # switch to them it cannot and falls back to the existing/default language
    EXTRA_LANG_INFO = {
        'cy-gb': {
            'bidi': False,
            'code': 'cy-gb',
            'name': 'Welsh',
            'name_local': u'Cymraeg',
        },
        'es-cr': {
            'bidi': False,
            'code': 'es-cr',
            'name': 'Spanish',
            'name_local': u'español de Costa Rica',
        },
    }

    locale.LANG_INFO.update(EXTRA_LANG_INFO)

    # The language selection has been slightly complicated now that we
    # have two es- languages: es-ar and es-cr.  Chrome doesn't offer
    # Costa Rican Spanish as one of its language choices, so the best
    # you can do is choose 'Spanish - español'. (This might well be
    # the case in other browsers too.)  Since 'es-ar' comes first in
    # 'languages' after the preceding code, this means that someone
    # viewing the Costa Rica site with Chrome's preferred language set
    # to Spanish (i.e. with 'es' first in Accept-Language) will get
    # the Argentinian Spanish translations instead of Costa Rican
    # Spanish.  To get around this, look for the default language code
    # for the site, and if that's present, move it to the front of
    # 'languages'.  This should be generally helpful behaviour: the
    # default language code of the site should take precedence over
    # another language that happens to match based on the generic part
    # of the language code.
    language_code_index = next(
        (i for i, l in enumerate(languages) if l[0] == language_code),
        None
    )
    if language_code_index is not None:
        languages.insert(0, languages.pop(language_code_index))

    # Make sure the MEDIA_ROOT directory actually exists:
    media_root = conf.get('MEDIA_ROOT') or join(BASE_DIR, 'media')
    # Make sure that the MEDIA_ROOT and subdirectory for archived CSV
    # files exist:
    mkdir_p(join(media_root, 'csv-archives'))

    # Database
    # https://docs.djangoproject.com/en/1.6/ref/settings/#databases
    if conf.get('DATABASE_SYSTEM') == 'postgresql':
        databases = {
            'default': {
                'ENGINE':   'django.db.backends.postgresql_psycopg2',
                'NAME':     conf.get('YNMP_DB_NAME'),
                'USER':     conf.get('YNMP_DB_USER'),
                'PASSWORD': conf.get('YNMP_DB_PASS'),
                'HOST':     conf.get('YNMP_DB_HOST'),
                'PORT':     conf.get('YNMP_DB_PORT'),
            }
        }
    else:
        databases = {
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
                'NAME': join(BASE_DIR, 'db.sqlite3'),
            }
        }

    # Setup caches depending on DEBUG:
    if debug:
        cache = {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}
        cache_thumbnails = {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}
    else:
        cache = {
            'TIMEOUT': None, # cache keys never expire; we invalidate them
            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
            'LOCATION': '127.0.0.1:11211',
            'KEY_PREFIX': databases['default']['NAME'],
        }
        cache_thumbnails = {
            'TIMEOUT': 60 * 60 * 24 * 2, # expire after two days
            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
            'LOCATION': '127.0.0.1:11211',
            'KEY_PREFIX': databases['default']['NAME'] + "-thumbnails",
        }

    # Create a dictionary with these settings and other simpler ones:
    result = {
        'BASE_DIR': BASE_DIR,
        'ALLOWED_HOSTS': conf.get('ALLOWED_HOSTS'),
        'DEBUG': debug,
        'RUNNING_TESTS': tests,

        # Email addresses that error emails are sent to when DEBUG = False
        'ADMINS': conf['ADMINS'],

        # SECURITY WARNING: keep the secret key used in production secret!
        'SECRET_KEY': conf['SECRET_KEY'],

        'TEMPLATE_DEBUG': True,
        'TEMPLATE_DIRS': (
            join(BASE_DIR, 'mysite', 'templates'),
        ),
        'TEMPLATE_CONTEXT_PROCESSORS': TEMPLATE_CONTEXT_PROCESSORS + (
            # Required by allauth template tags
            "django.core.context_processors.request",
            # allauth specific context processors
            "allauth.account.context_processors.account",
            "allauth.socialaccount.context_processors.socialaccount",
            "django.contrib.messages.context_processors.messages",
            "mysite.context_processors.add_settings",
            "mysite.context_processors.election_date",
            "mysite.context_processors.add_group_permissions",
            "mysite.context_processors.add_notification_data",
            "mysite.context_processors.locale",
            "mysite.context_processors.add_site",
        ),

        'ELECTION_APP': election_app,
        'ELECTION_APP_FULLY_QUALIFIED': election_app_fully_qualified,

        # The Django applications in use:
        'INSTALLED_APPS': (
            'django.contrib.admin',
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'django.contrib.humanize',
            'django.contrib.sessions',
            'django.contrib.messages',
            'django.contrib.staticfiles',
            'django.contrib.sites',
            'django_nose',
            'django_extensions',
            'pipeline',
            'statici18n',
            'sorl.thumbnail',
            'rest_framework',
            'rest_framework.authtoken',
            'images',
            'haystack',
            'elections',
            'popolo',
            election_app_fully_qualified,
            'candidates',
            'tasks',
            'alerts',
            'cached_counts',
            'moderation_queue',
            'auth_helpers',
            'debug_toolbar',
            'template_timings_panel',
            'official_documents',
            'results',
            'notifications',
            'allauth',
            'allauth.account',
            'allauth.socialaccount',
            'allauth.socialaccount.providers.google',
            'allauth.socialaccount.providers.facebook',
            'allauth.socialaccount.providers.twitter',
            'corsheaders',
            'crispy_forms',
            'usersettings',
        ),

        'SITE_ID': 1,

        'USERSETTINGS_MODEL': 'candidates.SiteSettings',

        'MIDDLEWARE_CLASSES': (
            'debug_toolbar.middleware.DebugToolbarMiddleware',
            'corsheaders.middleware.CorsMiddleware',
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.locale.LocaleMiddleware',
            'django.middleware.common.CommonMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'candidates.middleware.CopyrightAssignmentMiddleware',
            'candidates.middleware.DisallowedUpdateMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.clickjacking.XFrameOptionsMiddleware',
            'usersettings.middleware.CurrentUserSettingsMiddleware',
        ),

        # django-allauth settings:
        'AUTHENTICATION_BACKENDS': (
            # Needed to login by username in Django admin, regardless of `allauth`
            "django.contrib.auth.backends.ModelBackend",
            # `allauth` specific authentication methods, such as login by e-mail
            "allauth.account.auth_backends.AuthenticationBackend",
        ),
        'SOCIALACCOUNT_PROVIDERS': {
            'google': {'SCOPE': ['https://www.googleapis.com/auth/userinfo.profile'],
                       'AUTH_PARAMS': {'access_type': 'online'}},
            'facebook': {'SCOPE': ['email',]},
        },
        'LOGIN_REDIRECT_URL': '/',
        'ACCOUNT_EMAIL_VERIFICATION': 'mandatory',
        'ACCOUNT_EMAIL_REQUIRED': True,
        'ACCOUNT_USERNAME_REQUIRED': True,
        'SOCIALACCOUNT_AUTO_SIGNUP': True,

        # use our own adapter that checks if user signup has been disabled
        'ACCOUNT_ADAPTER': 'mysite.account_adapter.CheckIfAllowedNewUsersAccountAdapter',

        'ROOT_URLCONF': 'mysite.urls',
        'WSGI_APPLICATION': 'mysite.wsgi.application',

        # Django Debug Toolbar settings:
        'DEBUG_TOOLBAR_PATCH_SETTINGS': False,
        'DEBUG_TOOLBAR_PANELS': [
            'debug_toolbar.panels.versions.VersionsPanel',
            'debug_toolbar.panels.timer.TimerPanel',
            'debug_toolbar.panels.settings.SettingsPanel',
            'debug_toolbar.panels.headers.HeadersPanel',
            'debug_toolbar.panels.request.RequestPanel',
            'debug_toolbar.panels.sql.SQLPanel',
            'debug_toolbar.panels.staticfiles.StaticFilesPanel',
            'debug_toolbar.panels.templates.TemplatesPanel',
            'debug_toolbar.panels.cache.CachePanel',
            'debug_toolbar.panels.signals.SignalsPanel',
            'debug_toolbar.panels.logging.LoggingPanel',
            'debug_toolbar.panels.redirects.RedirectsPanel',
            'template_timings_panel.panels.TemplateTimings.TemplateTimings',
        ],
        'INTERNAL_IPS': ['127.0.0.1'],

        # Language settings (calculated above):
        'LOCALE_PATHS': locale_paths,
        'LANGUAGES': languages,
        'LANGUAGE_CODE': language_code,
        'TIME_ZONE': conf.get('TIME_ZONE', 'Europe/London'),
        'USE_I18N': True,
        'USE_L10N': True,
        'USE_TZ': True,

        # The media and static file settings:
        'MEDIA_ROOT': media_root,
        'MEDIA_URL': '/media/',

        # Settings for staticfiles and Django pipeline:
        'STATIC_URL': '/static/',
        'STATIC_ROOT': join(BASE_DIR, 'static'),
        'STATICI18N_ROOT': join(BASE_DIR, 'mysite', 'static'),
        'STATICFILES_DIRS': (
            join(BASE_DIR, 'mysite', 'static'),
        ),
        'STATICFILES_FINDERS': (
            'django.contrib.staticfiles.finders.FileSystemFinder',
            'django.contrib.staticfiles.finders.AppDirectoriesFinder',
            'pipeline.finders.PipelineFinder',
        ),
        'PIPELINE': {
            'STYLESHEETS': {
                'image-review': {
                    'source_filenames': (
                        'moderation_queue/css/jquery.Jcrop.css',
                        'moderation_queue/css/crop.scss',
                    ),
                    'output_filename': 'css/image-review.css',
                },
                'official_documents': {
                    'source_filenames': (
                        'official_documents/css/official_documents.scss',
                    ),
                    'output_filename': 'css/official_documents.css',
                },
                'all': {
                    'source_filenames': (
                        'candidates/style.scss',
                        'cached_counts/style.scss',
                        'select2/select2.css',
                        'jquery/jquery-ui.css',
                        'jquery/jquery-ui.structure.css',
                        'jquery/jquery-ui.theme.css',
                        'moderation_queue/css/photo-upload.scss',
                    ),
                    'output_filename': 'css/all.css',
                }
            },
            'JAVASCRIPT': {
                'image-review': {
                    'source_filenames': (
                        'moderation_queue/js/jquery.color.js',
                        'moderation_queue/js/jquery.Jcrop.js',
                        'moderation_queue/js/crop.js',
                    ),
                    'output_filename': 'js/image-review.js',
                },
                'all': {
                    'source_filenames': (
                        'js/vendor/custom.modernizr.js',
                        'jquery/jquery-1.11.1.js',
                        'jquery/jquery-ui.js',
                        'foundation/js/foundation/foundation.js',
                        'foundation/js/foundation/foundation.equalizer.js',
                        'foundation/js/foundation/foundation.dropdown.js',
                        'foundation/js/foundation/foundation.tooltip.js',
                        'foundation/js/foundation/foundation.offcanvas.js',
                        'foundation/js/foundation/foundation.accordion.js',
                        'foundation/js/foundation/foundation.joyride.js',
                        'foundation/js/foundation/foundation.alert.js',
                        'foundation/js/foundation/foundation.topbar.js',
                        'foundation/js/foundation/foundation.reveal.js',
                        'foundation/js/foundation/foundation.slider.js',
                        'foundation/js/foundation/foundation.magellan.js',
                        'foundation/js/foundation/foundation.clearing.js',
                        'foundation/js/foundation/foundation.orbit.js',
                        'foundation/js/foundation/foundation.interchange.js',
                        'foundation/js/foundation/foundation.abide.js',
                        'foundation/js/foundation/foundation.tab.js',
                        'select2/select2.js',
                        'js/constituency.js',
                        'js/person_form.js',
                        'js/home_geolocation_form.js',
                        'js/versions.js',
                        'js/language-switcher.js',
                    ),
                    'output_filename': 'js/all.js'
                }
            },

            'COMPILERS': (
                'pipeline.compilers.sass.SASSCompiler',
            ),
            'SASS_BINARY': 'sassc',
            'CSS_COMPRESSOR': 'pipeline.compressors.yui.YUICompressor',
            'JS_COMPRESSOR': 'pipeline.compressors.yui.YUICompressor',
            # On some platforms this might be called "yuicompressor", so it may be
            # necessary to symlink it into your PATH as "yui-compressor".
            'YUI_BINARY': '/usr/bin/env yui-compressor',
        },


        'TEST_RUNNER': 'django_nose.NoseTestSuiteRunner',

        'SOURCE_HINTS': _(
            u"Please don't quote third-party candidate sites \u2014 "
            u"we prefer URLs of news stories or official candidate pages."
        ),

        # By default, cache successful results from MapIt for a day
        'MAPIT_CACHE_SECONDS': 86400,
        'DATABASES': databases,
        'CACHES': {
            'default': cache,
            'thumbnails': cache_thumbnails,
        },

        # sorl-thumbnail settings:
        'THUMBNAIL_CACHE': 'thumbnails',
        'THUMBNAIL_DEBUG': debug,

        # Django Rest Framework settings:
        'REST_FRAMEWORK': {
            'DEFAULT_PERMISSION_CLASSES': ('candidates.api_permissions.ReadOnly',),
            'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.URLPathVersioning',
            'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
            'PAGE_SIZE': 10,
        },

        # allow attaching extra data to notifications:
        'NOTIFICATIONS_USE_JSONFIELD': True,

        'HAYSTACK_SIGNAL_PROCESSOR': 'haystack.signals.RealtimeSignalProcessor',

        'HAYSTACK_CONNECTIONS': {
            'default': {
                'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
                'URL': 'http://127.0.0.1:9200/',
                'INDEX_NAME': '{0}_{1}'.format(conf.get('YNMP_DB_NAME'), conf.get('YNMP_DB_HOST')),
            },
        },

        # CORS config
        'CORS_ORIGIN_ALLOW_ALL': True,
        'CORS_URLS_REGEX': r'^/api/.*$',
        'CORS_ALLOW_METHODS': (
            'GET',
            'OPTIONS',
        ),
    }

    if tests:
        result['NOSE_ARGS'] = [
            '--nocapture',
            '--with-yanc',
            # There are problems with OpenCV on Travis, so don't even try to
            # import moderation_queue/faces.py
            '--ignore-files=faces',
        ]
        if election_app == 'example':
            result['NOSE_ARGS'].append('--with-doctest')
    else:
        # If we're not testing, use PipelineCachedStorage
        result['STATICFILES_STORAGE'] = \
            'pipeline.storage.PipelineCachedStorage'
    if conf.get('NGINX_SSL'):
        result['SECURE_PROXY_SSL_HEADER'] = ('HTTP_X_FORWARDED_PROTO', 'https')
        result['ACCOUNT_DEFAULT_HTTP_PROTOCOL'] = 'https'
    for optional_election_app_setting, default in (
            ('AREAS_TO_ALWAYS_RETURN', []),
    ):
        try:
            result[optional_election_app_setting] = \
                getattr(elections_module, optional_election_app_setting)
        except AttributeError:
            result[optional_election_app_setting] = default
    # Make sure there's a trailing slash at the end of base MapIt URL:
    result['MAPIT_BASE_URL'] = \
        re.sub(r'/*$', '/', elections_module.MAPIT_BASE_URL)

    return result

Example 186

Project: wok
Source File: page.py
View license
    def build_meta(self):
        """
        Ensures the guarantees about metadata for documents are valid.

        `page.title` - Will be a string.
        `page.slug` - Will be a string.
        `page.author` - Will have fields `name` and `email`.
        `page.authors` - Will be a list of Authors.
        `page.category` - Will be a list.
        `page.published` - Will exist.
        `page.datetime` - Will be a datetime, or None.
        `page.date` - Will be a date, or None.
        `page.time` - Will be a time, or None.
        `page.tags` - Will be a list.
        `page.url` - Will be the url of the page, relative to the web root.
        `page.subpages` - Will be a list containing every sub page of this page
        """

        self.engine.run_hook('page.meta.pre', self)

        if not self.meta:
            self.meta = {}

        # title
        if not 'title' in self.meta:
            if self.filename:
                # Take off the last file extension.
                self.meta['title'] = '.'.join(self.filename.split('.')[:-1])
                if (self.meta['title'] == ''):
                    self.meta['title'] = self.filename

                logging.warning("You didn't specify a title in {0}. Using the "
                                "file name as a title.".format(self.path))
            elif 'slug' in self.meta:
                self.meta['title'] = self.meta['slug']
                logging.warning("You didn't specify a title in {0}, which was "
                        "not generated from a file. Using the slug as a title."
                        .format(self.meta['slug']))
            else:
                logging.error("A page was generated that is not from a file, "
                        "has no title, and no slug. I don't know what to do. "
                        "Not using this page.")
                logging.info("Bad Meta's keys: {0}".format(self.meta.keys()))
                logging.debug("Bad Meta: {0}".format(self.meta))
                raise BadMetaException()

        # slug
        if not 'slug' in self.meta:
            if self.filename and self.options['slug_from_filename']:
                filename_no_ext = '.'.join(self.filename.split('.')[:-1])
                if filename_no_ext == '':
                    filename_no_ext = self.filename
                self.meta['slug'] = slugify(filename_no_ext)
                logging.info("You didn't specify a slug, generating it from the "
                             "filename.")
            else:
                self.meta['slug'] = slugify(unicode(self.meta['title']))
                logging.info("You didn't specify a slug, and no filename "
                             "exists. Generating the slug from the title.")

        elif self.meta['slug'] != slugify(self.meta['slug']):
            logging.warning('Your slug should probably be all lower case, and '
                            'match "[a-z0-9-]*"')

        # authors and author
        authors = self.meta.get('authors', self.meta.get('author', None))
        if isinstance(authors, list):
            self.meta['authors'] = [Author.parse(a) for a in authors]
        elif isinstance(authors, str):
            self.meta['authors'] = [Author.parse(a) for a in authors.split(',')]
            if len(self.meta['authors']) > 1:
                logging.warn('Deprecation Warning: Use YAML lists instead of '
                        'CSV for multiple authors. i.e. ["John Doe", "Jane '
                        'Smith"] instead of "John Doe, Jane Smith". In '
                        '{0}.'.format(self.path))

        elif authors is None:
            self.meta['authors'] = self.options.get('authors', [])
        else:
            # wait, what? Authors is of wrong type.
            self.meta['authors'] = []
            logging.error(('Authors in {0} is an unknown type. Valid types '
                           'are string or list. Instead it is a {1}')
                           .format(self.meta['slug']), authors.type)

        if self.meta['authors']:
            self.meta['author'] = self.meta['authors'][0]
        else:
            self.meta['author'] = Author()

        # category
        if 'category' in self.meta:
            if isinstance(self.meta['category'], str):
                self.meta['category'] = self.meta['category'].split('/')
            elif isinstance(self.meta['category'], list):
                pass
            else:
                # category is of wrong type.
                logging.error('Category in {0} is an unknown type. Valid '
                              'types are string or list. Instead it is a {1}'
                              .format(self.meta['slug'], type(self.meta['category'])))
                self.meta['category'] = []
        else:
            self.meta['category'] = []
        if self.meta['category'] == None:
            self.meta = []

        # published
        if not 'published' in self.meta:
            self.meta['published'] = True

        # make_file
        if not 'make_file' in self.meta:
            self.meta['make_file'] = True

        # datetime, date, time
        util.date_and_times(self.meta)

        # tags
        if 'tags' in self.meta:
            if isinstance(self.meta['tags'], list):
                # good
                pass
            elif isinstance(self.meta['tags'], str):
                self.meta['tags'] = [t.strip() for t in
                    self.meta['tags'].split(',')]
                if len(self.meta['tags']) > 1:
                    logging.warn('Deprecation Warning: Use YAML lists instead '
                            'of CSV for multiple tags. i.e. tags: [guide, '
                            'howto] instead of tags: guide, howto. In {0}.'
                            .format(self.path))
        else:
            self.meta['tags'] = []

        logging.debug('Tags for {0}: {1}'.
                format(self.meta['slug'], self.meta['tags']))

        # pagination
        if 'pagination' not in self.meta:
            self.meta['pagination'] = {}

        if 'cur_page' not in self.meta['pagination']:
            self.meta['pagination']['cur_page'] = 1
        if 'num_pages' not in self.meta['pagination']:
            self.meta['pagination']['num_pages'] = 1

        # template
        try:
            template_type = str(self.meta.get('type', 'default'))
            self.template = self.tmpl_env.get_template(template_type + '.*')
        except jinja2.loaders.TemplateNotFound:
            logging.error('No template "{0}.*" found in template directory. Aborting.'
                    .format(template_type))
            sys.exit()
        except AmbiguousTemplate:
            logging.error(('Ambiguous template found. There are two files that '
                          'match "{0}.*". Aborting.').format(template_type))
            sys.exit()

        # url
        parts = {
            'slug': self.meta['slug'],
            'category': '/'.join(self.meta['category']),
            'page': self.meta['pagination']['cur_page'],
            'date': self.meta['date'],
            'datetime': self.meta['datetime'],
            'time': self.meta['time'],
        }
        logging.debug('current page: ' + repr(parts['page']))

        # Pull extensions from the template's real file name.
        parts['ext'] = os.path.splitext(self.template.filename)[1]
        if parts['ext']:
            parts['ext'] = parts['ext'][1:] # remove leading dot
        # Deprecated
        parts['type'] = parts['ext']
        self.meta['ext'] = parts['ext']

        if parts['page'] == 1:
            parts['page'] = ''

        if 'url' in self.meta:
            logging.debug('Using page url pattern')
            self.url_pattern = self.meta['url']
        else:
            logging.debug('Using global url pattern')
            self.url_pattern = self.options['url_pattern']

        self.meta['url'] = self.url_pattern.format(**parts)

        logging.info('URL pattern is: {0}'.format(self.url_pattern))
        logging.info('URL parts are: {0}'.format(parts))

        # Get rid of extra slashes
        self.meta['url'] = re.sub(r'//+', '/', self.meta['url'])

        # If we have been asked to, rip out any plain "index.html"s
        if not self.options['url_include_index']:
            self.meta['url'] = re.sub(r'/index\.html$', '/', self.meta['url'])

        # To be used for writing page content
        self.meta['path'] = self.meta['url']
        # If site is going to be in a subdirectory
        if self.options.get('url_subdir'):
            self.meta['url'] = self.options['url_subdir'] + self.meta['url']

        # Some urls should start with /, some should not.
        if self.options['relative_urls'] and self.meta['url'][0] == '/':
            self.meta['url'] = self.meta['url'][1:]
        if not self.options['relative_urls'] and self.meta['url'][0] != '/':
            self.meta['url'] = '/' + self.meta['url']

        logging.debug('url is: ' + self.meta['url'])

        # subpages
        self.meta['subpages'] = []

        self.engine.run_hook('page.meta.post', self)

Example 187

Project: rjsmin
Source File: rjsmin.py
View license
def _make_jsmin(python_only=False):
    """
    Generate JS minifier based on `jsmin.c by Douglas Crockford`_

    .. _jsmin.c by Douglas Crockford:
       http://www.crockford.com/javascript/jsmin.c

    :Parameters:
      `python_only` : ``bool``
        Use only the python variant. If true, the c extension is not even
        tried to be loaded.

    :Return: Minifier
    :Rtype: ``callable``
    """
    # pylint: disable = unused-variable
    # pylint: disable = too-many-locals

    if not python_only:
        try:
            import _rjsmin
        except ImportError:
            pass
        else:
            return _rjsmin.jsmin
    try:
        xrange
    except NameError:
        xrange = range  # pylint: disable = redefined-builtin

    space_chars = r'[\000-\011\013\014\016-\040]'

    line_comment = r'(?://[^\r\n]*)'
    space_comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
    space_comment_nobang = r'(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*\*+)*/)'
    bang_comment = r'(?:/\*![^*]*\*+(?:[^/*][^*]*\*+)*/)'

    string1 = \
        r'(?:\047[^\047\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)'
    string2 = r'(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^"\\\r\n]*)*")'
    strings = r'(?:%s|%s)' % (string1, string2)

    charclass = r'(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\])'
    nospecial = r'[^/\\\[\r\n]'
    regex = r'(?:/(?![\r\n/*])%s*(?:(?:\\[^\r\n]|%s)%s*)*/)' % (
        nospecial, charclass, nospecial
    )
    space = r'(?:%s|%s)' % (space_chars, space_comment)
    newline = r'(?:%s?[\r\n])' % line_comment

    def fix_charclass(result):
        """ Fixup string of chars to fit into a regex char class """
        pos = result.find('-')
        if pos >= 0:
            result = r'%s%s-' % (result[:pos], result[pos + 1:])

        def sequentize(string):
            """
            Notate consecutive characters as sequence

            (1-4 instead of 1234)
            """
            first, last, result = None, None, []
            for char in map(ord, string):
                if last is None:
                    first = last = char
                elif last + 1 == char:
                    last = char
                else:
                    result.append((first, last))
                    first = last = char
            if last is not None:
                result.append((first, last))
            return ''.join(['%s%s%s' % (
                chr(first),
                last > first + 1 and '-' or '',
                last != first and chr(last) or ''
            ) for first, last in result])  # noqa

        return _re.sub(
            r'([\000-\040\047])',  # \047 for better portability
            lambda m: '\\%03o' % ord(m.group(1)), (
                sequentize(result)
                .replace('\\', '\\\\')
                .replace('[', '\\[')
                .replace(']', '\\]')
            )
        )

    def id_literal_(what):
        """ Make id_literal like char class """
        match = _re.compile(what).match
        result = ''.join([
            chr(c) for c in xrange(127) if not match(chr(c))
        ])
        return '[^%s]' % fix_charclass(result)

    def not_id_literal_(keep):
        """ Make negated id_literal like char class """
        match = _re.compile(id_literal_(keep)).match
        result = ''.join([
            chr(c) for c in xrange(127) if not match(chr(c))
        ])
        return r'[%s]' % fix_charclass(result)

    not_id_literal = not_id_literal_(r'[a-zA-Z0-9_$]')
    preregex1 = r'[(,=:\[!&|?{};\r\n]'
    preregex2 = r'%(not_id_literal)sreturn' % locals()

    id_literal = id_literal_(r'[a-zA-Z0-9_$]')
    id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]')
    id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047+-]')
    post_regex_off = id_literal_(r'[^\000-\040}\])?:|,;.&=+-]')

    dull = r'[^\047"/\000-\040]'

    space_sub_simple = _re.compile((
        # noqa pylint: disable = bad-continuation

        r'(%(dull)s+)'                                         # 0
        r'|(%(strings)s%(dull)s*)'                             # 1
        r'|(?<=%(preregex1)s)'
            r'%(space)s*(?:%(newline)s%(space)s*)*'
            r'(%(regex)s)'                                     # 2
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 3
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(preregex2)s)'
            r'%(space)s*(?:(%(newline)s)%(space)s*)*'          # 4
            r'(%(regex)s)'                                     # 5
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 6
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(id_literal_close)s)'
            r'%(space)s*(?:(%(newline)s)%(space)s*)+'          # 7
            r'(?=%(id_literal_open)s)'
        r'|(?<=%(id_literal)s)(%(space)s)+(?=%(id_literal)s)'  # 8
        r'|(?<=\+)(%(space)s)+(?=\+)'                          # 9
        r'|(?<=-)(%(space)s)+(?=-)'                            # 10
        r'|%(space)s+'
        r'|(?:%(newline)s%(space)s*)+'
    ) % locals()).sub

    # print space_sub_simple.__self__.pattern

    def space_subber_simple(match):
        """ Substitution callback """
        # pylint: disable = too-many-return-statements

        groups = match.groups()
        if groups[0]:
            return groups[0]
        elif groups[1]:
            return groups[1]
        elif groups[2]:
            if groups[3]:
                return groups[2] + '\n'
            return groups[2]
        elif groups[5]:
            return "%s%s%s" % (
                groups[4] and '\n' or '',
                groups[5],
                groups[6] and '\n' or '',
            )
        elif groups[7]:
            return '\n'
        elif groups[8] or groups[9] or groups[10]:
            return ' '
        else:
            return ''

    space_sub_banged = _re.compile((
        # noqa pylint: disable = bad-continuation

        r'(%(dull)s+)'                                         # 0
        r'|(%(strings)s%(dull)s*)'                             # 1
        r'|(?<=%(preregex1)s)'
            r'(%(space)s*(?:%(newline)s%(space)s*)*)'          # 2
            r'(%(regex)s)'                                     # 3
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 4
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(preregex2)s)'
            r'(%(space)s*(?:(%(newline)s)%(space)s*)*)'        # 5, 6
            r'(%(regex)s)'                                     # 7
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 8
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(id_literal_close)s)'
            r'(%(space)s*(?:%(newline)s%(space)s*)+)'          # 9
            r'(?=%(id_literal_open)s)'
        r'|(?<=%(id_literal)s)(%(space)s+)(?=%(id_literal)s)'  # 10
        r'|(?<=\+)(%(space)s+)(?=\+)'                          # 11
        r'|(?<=-)(%(space)s+)(?=-)'                            # 12
        r'|(%(space)s+)'                                       # 13
        r'|((?:%(newline)s%(space)s*)+)'                       # 14
    ) % locals()).sub

    # print space_sub_banged.__self__.pattern

    keep = _re.compile((
        r'%(space_chars)s+|%(space_comment_nobang)s+|%(newline)s+'
        r'|(%(bang_comment)s+)'
    ) % locals()).sub
    keeper = lambda m: m.groups()[0] or ''

    # print keep.__self__.pattern

    def space_subber_banged(match):
        """ Substitution callback """
        # pylint: disable = too-many-return-statements

        groups = match.groups()
        if groups[0]:
            return groups[0]
        elif groups[1]:
            return groups[1]
        elif groups[3]:
            return "%s%s%s%s" % (
                keep(keeper, groups[2]),
                groups[3],
                keep(keeper, groups[4] or ''),
                groups[4] and '\n' or '',
            )
        elif groups[7]:
            return "%s%s%s%s%s" % (
                keep(keeper, groups[5]),
                groups[6] and '\n' or '',
                groups[7],
                keep(keeper, groups[8] or ''),
                groups[8] and '\n' or '',
            )
        elif groups[9]:
            return keep(keeper, groups[9]) + '\n'
        elif groups[10] or groups[11] or groups[12]:
            return keep(keeper, groups[10] or groups[11] or groups[12]) or ' '
        else:
            return keep(keeper, groups[13] or groups[14])

    def jsmin(script, keep_bang_comments=False):
        r"""
        Minify javascript based on `jsmin.c by Douglas Crockford`_\.

        Instead of parsing the stream char by char, it uses a regular
        expression approach which minifies the whole script with one big
        substitution regex.

        .. _jsmin.c by Douglas Crockford:
           http://www.crockford.com/javascript/jsmin.c

        :Parameters:
          `script` : ``str``
            Script to minify

          `keep_bang_comments` : ``bool``
            Keep comments starting with an exclamation mark? (``/*!...*/``)

        :Return: Minified script
        :Rtype: ``str``
        """
        # pylint: disable = redefined-outer-name

        if keep_bang_comments:
            return space_sub_banged(
                space_subber_banged, '\n%s\n' % script
            ).strip()
        else:
            return space_sub_simple(
                space_subber_simple, '\n%s\n' % script
            ).strip()

    return jsmin

Example 188

Project: rjsmin
Source File: rjsmin.py
View license
def _make_jsmin(python_only=False):
    """
    Generate JS minifier based on `jsmin.c by Douglas Crockford`_

    .. _jsmin.c by Douglas Crockford:
       http://www.crockford.com/javascript/jsmin.c

    :Parameters:
      `python_only` : ``bool``
        Use only the python variant. If true, the c extension is not even
        tried to be loaded.

    :Return: Minifier
    :Rtype: ``callable``
    """
    # pylint: disable = unused-variable
    # pylint: disable = too-many-locals

    if not python_only:
        try:
            import _rjsmin
        except ImportError:
            pass
        else:
            return _rjsmin.jsmin
    try:
        xrange
    except NameError:
        xrange = range  # pylint: disable = redefined-builtin

    space_chars = r'[\000-\011\013\014\016-\040]'

    line_comment = r'(?://[^\r\n]*)'
    space_comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
    space_comment_nobang = r'(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*\*+)*/)'
    bang_comment = r'(?:/\*![^*]*\*+(?:[^/*][^*]*\*+)*/)'

    string1 = \
        r'(?:\047[^\047\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)'
    string2 = r'(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^"\\\r\n]*)*")'
    strings = r'(?:%s|%s)' % (string1, string2)

    charclass = r'(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\])'
    nospecial = r'[^/\\\[\r\n]'
    regex = r'(?:/(?![\r\n/*])%s*(?:(?:\\[^\r\n]|%s)%s*)*/)' % (
        nospecial, charclass, nospecial
    )
    space = r'(?:%s|%s)' % (space_chars, space_comment)
    newline = r'(?:%s?[\r\n])' % line_comment

    def fix_charclass(result):
        """ Fixup string of chars to fit into a regex char class """
        pos = result.find('-')
        if pos >= 0:
            result = r'%s%s-' % (result[:pos], result[pos + 1:])

        def sequentize(string):
            """
            Notate consecutive characters as sequence

            (1-4 instead of 1234)
            """
            first, last, result = None, None, []
            for char in map(ord, string):
                if last is None:
                    first = last = char
                elif last + 1 == char:
                    last = char
                else:
                    result.append((first, last))
                    first = last = char
            if last is not None:
                result.append((first, last))
            return ''.join(['%s%s%s' % (
                chr(first),
                last > first + 1 and '-' or '',
                last != first and chr(last) or ''
            ) for first, last in result])  # noqa

        return _re.sub(
            r'([\000-\040\047])',  # \047 for better portability
            lambda m: '\\%03o' % ord(m.group(1)), (
                sequentize(result)
                .replace('\\', '\\\\')
                .replace('[', '\\[')
                .replace(']', '\\]')
            )
        )

    def id_literal_(what):
        """ Make id_literal like char class """
        match = _re.compile(what).match
        result = ''.join([
            chr(c) for c in xrange(127) if not match(chr(c))
        ])
        return '[^%s]' % fix_charclass(result)

    def not_id_literal_(keep):
        """ Make negated id_literal like char class """
        match = _re.compile(id_literal_(keep)).match
        result = ''.join([
            chr(c) for c in xrange(127) if not match(chr(c))
        ])
        return r'[%s]' % fix_charclass(result)

    not_id_literal = not_id_literal_(r'[a-zA-Z0-9_$]')
    preregex1 = r'[(,=:\[!&|?{};\r\n]'
    preregex2 = r'%(not_id_literal)sreturn' % locals()

    id_literal = id_literal_(r'[a-zA-Z0-9_$]')
    id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]')
    id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047+-]')
    post_regex_off = id_literal_(r'[^\000-\040}\])?:|,;.&=+-]')

    dull = r'[^\047"/\000-\040]'

    space_sub_simple = _re.compile((
        # noqa pylint: disable = bad-continuation

        r'(%(dull)s+)'                                         # 0
        r'|(%(strings)s%(dull)s*)'                             # 1
        r'|(?<=%(preregex1)s)'
            r'%(space)s*(?:%(newline)s%(space)s*)*'
            r'(%(regex)s)'                                     # 2
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 3
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(preregex2)s)'
            r'%(space)s*(?:(%(newline)s)%(space)s*)*'          # 4
            r'(%(regex)s)'                                     # 5
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 6
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(id_literal_close)s)'
            r'%(space)s*(?:(%(newline)s)%(space)s*)+'          # 7
            r'(?=%(id_literal_open)s)'
        r'|(?<=%(id_literal)s)(%(space)s)+(?=%(id_literal)s)'  # 8
        r'|(?<=\+)(%(space)s)+(?=\+)'                          # 9
        r'|(?<=-)(%(space)s)+(?=-)'                            # 10
        r'|%(space)s+'
        r'|(?:%(newline)s%(space)s*)+'
    ) % locals()).sub

    # print space_sub_simple.__self__.pattern

    def space_subber_simple(match):
        """ Substitution callback """
        # pylint: disable = too-many-return-statements

        groups = match.groups()
        if groups[0]:
            return groups[0]
        elif groups[1]:
            return groups[1]
        elif groups[2]:
            if groups[3]:
                return groups[2] + '\n'
            return groups[2]
        elif groups[5]:
            return "%s%s%s" % (
                groups[4] and '\n' or '',
                groups[5],
                groups[6] and '\n' or '',
            )
        elif groups[7]:
            return '\n'
        elif groups[8] or groups[9] or groups[10]:
            return ' '
        else:
            return ''

    space_sub_banged = _re.compile((
        # noqa pylint: disable = bad-continuation

        r'(%(dull)s+)'                                         # 0
        r'|(%(strings)s%(dull)s*)'                             # 1
        r'|(?<=%(preregex1)s)'
            r'(%(space)s*(?:%(newline)s%(space)s*)*)'          # 2
            r'(%(regex)s)'                                     # 3
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 4
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(preregex2)s)'
            r'(%(space)s*(?:(%(newline)s)%(space)s*)*)'        # 5, 6
            r'(%(regex)s)'                                     # 7
            r'(%(space)s*(?:%(newline)s%(space)s*)+'           # 8
                r'(?=%(post_regex_off)s))?'
        r'|(?<=%(id_literal_close)s)'
            r'(%(space)s*(?:%(newline)s%(space)s*)+)'          # 9
            r'(?=%(id_literal_open)s)'
        r'|(?<=%(id_literal)s)(%(space)s+)(?=%(id_literal)s)'  # 10
        r'|(?<=\+)(%(space)s+)(?=\+)'                          # 11
        r'|(?<=-)(%(space)s+)(?=-)'                            # 12
        r'|(%(space)s+)'                                       # 13
        r'|((?:%(newline)s%(space)s*)+)'                       # 14
    ) % locals()).sub

    # print space_sub_banged.__self__.pattern

    keep = _re.compile((
        r'%(space_chars)s+|%(space_comment_nobang)s+|%(newline)s+'
        r'|(%(bang_comment)s+)'
    ) % locals()).sub
    keeper = lambda m: m.groups()[0] or ''

    # print keep.__self__.pattern

    def space_subber_banged(match):
        """ Substitution callback """
        # pylint: disable = too-many-return-statements

        groups = match.groups()
        if groups[0]:
            return groups[0]
        elif groups[1]:
            return groups[1]
        elif groups[3]:
            return "%s%s%s%s" % (
                keep(keeper, groups[2]),
                groups[3],
                keep(keeper, groups[4] or ''),
                groups[4] and '\n' or '',
            )
        elif groups[7]:
            return "%s%s%s%s%s" % (
                keep(keeper, groups[5]),
                groups[6] and '\n' or '',
                groups[7],
                keep(keeper, groups[8] or ''),
                groups[8] and '\n' or '',
            )
        elif groups[9]:
            return keep(keeper, groups[9]) + '\n'
        elif groups[10] or groups[11] or groups[12]:
            return keep(keeper, groups[10] or groups[11] or groups[12]) or ' '
        else:
            return keep(keeper, groups[13] or groups[14])

    def jsmin(script, keep_bang_comments=False):
        r"""
        Minify javascript based on `jsmin.c by Douglas Crockford`_\.

        Instead of parsing the stream char by char, it uses a regular
        expression approach which minifies the whole script with one big
        substitution regex.

        .. _jsmin.c by Douglas Crockford:
           http://www.crockford.com/javascript/jsmin.c

        :Parameters:
          `script` : ``str``
            Script to minify

          `keep_bang_comments` : ``bool``
            Keep comments starting with an exclamation mark? (``/*!...*/``)

        :Return: Minified script
        :Rtype: ``str``
        """
        # pylint: disable = redefined-outer-name

        if keep_bang_comments:
            return space_sub_banged(
                space_subber_banged, '\n%s\n' % script
            ).strip()
        else:
            return space_sub_simple(
                space_subber_simple, '\n%s\n' % script
            ).strip()

    return jsmin

Example 189

Project: Roboragi
Source File: AnimeBot.py
View license
def process_comment(comment, is_edit=False):
    #Anime/Manga requests that are found go into separate arrays
    animeArray = []
    mangaArray = []
    lnArray = []

    #ignores all "code" markup (i.e. anything between backticks)
    comment.body = re.sub(r"\`(?s)(.*?)\`", "", comment.body)
    
    #This checks for requests. First up we check all known tags for the !stats request
    if re.search('({!stats.*?}|{{!stats.*?}}|<!stats.*?>|<<!stats.*?>>)', comment.body, re.S) is not None:
        username = re.search('[uU]\/([A-Za-z0-9_-]+?)(>|}|$)', comment.body, re.S)
        subreddit = re.search('[rR]\/([A-Za-z0-9_]+?)(>|}|$)', comment.body, re.S)

        if username:
            commentReply = CommentBuilder.buildStatsComment(username=username.group(1))
        elif subreddit:
            commentReply = CommentBuilder.buildStatsComment(subreddit=subreddit.group(1))
        else:
            commentReply = CommentBuilder.buildStatsComment()
    else:
        
        #The basic algorithm here is:
        #If it's an expanded request, build a reply using the data in the braces, clear the arrays, add the reply to the relevant array and ignore everything else.
        #If it's a normal request, build a reply using the data in the braces, add the reply to the relevant array.

        #Counts the number of expanded results vs total results. If it's not just a single expanded result, they all get turned into normal requests.
        numOfRequest = 0
        numOfExpandedRequest = 0
        forceNormal = False

        for match in re.finditer("\{{2}([^}]*)\}{2}|\<{2}([^>]*)\>{2}", comment.body, re.S):
            numOfRequest += 1
            numOfExpandedRequest += 1
            
        for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))|(?<=(?<!\<)\<)([^\<\>]*)(?=\>(?!\>))", comment.body, re.S):
            numOfRequest += 1

        if (numOfExpandedRequest >= 1) and (numOfRequest > 1):
            forceNormal = True

        #Expanded Anime
        for match in re.finditer("\{{2}([^}]*)\}{2}", comment.body, re.S):
            reply = ''

            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildAnimeReply(match.group(1), False, comment)
            else:
                reply = Search.buildAnimeReply(match.group(1), True, comment)                    

            if (reply is not None):
                animeArray.append(reply)

        #Normal Anime  
        for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))", comment.body, re.S):
            reply = Search.buildAnimeReply(match.group(1), False, comment)
            
            if (reply is not None):
                animeArray.append(reply)

        #Expanded Manga
        #NORMAL EXPANDED
        for match in re.finditer("\<{2}([^>]*)\>{2}(?!(:|\>))", comment.body, re.S):
            reply = ''
            
            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildMangaReply(match.group(1), False, comment)
            else:
                reply = Search.buildMangaReply(match.group(1), True, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #AUTHOR SEARCH EXPANDED
        for match in re.finditer("\<{2}([^>]*)\>{2}:\(([^)]+)\)", comment.body, re.S):
            reply = ''
            
            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)
            else:
                reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), True, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #Normal Manga
        #NORMAL
        for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]+)\>(?!(:|\>))", comment.body, re.S):
            reply = Search.buildMangaReply(match.group(1), False, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #AUTHOR SEARCH
        for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]*)\>:\(([^)]+)\)", comment.body, re.S):
            reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #Expanded LN
        for match in re.finditer("\]{2}([^]]*)\[{2}", comment.body, re.S):
            reply = ''

            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildLightNovelReply(match.group(1), False, comment)
            else:
                reply = Search.buildLightNovelReply(match.group(1), True, comment)                    

            if (reply is not None):
                lnArray.append(reply)

        #Normal LN  
        for match in re.finditer("(?<=(?<!\])\])([^\]\[]*)(?=\[(?!\[))", comment.body, re.S):
            reply = Search.buildLightNovelReply(match.group(1), False, comment)
            
            if (reply is not None):
                lnArray.append(reply)
            
        #Here is where we create the final reply to be posted

        #The final comment reply. We add stuff to this progressively.
        commentReply = ''

        #Basically just to keep track of people posting the same title multiple times (e.g. {Nisekoi}{Nisekoi}{Nisekoi})
        postedAnimeTitles = []
        postedMangaTitles = []
        postedLNTitles = []

        #Adding all the anime to the final comment. If there's manga too we split up all the paragraphs and indent them in Reddit markup by adding a '>', then recombine them
        for i, animeReply in enumerate(animeArray):
            if not (i is 0):
                commentReply += '\n\n'

            if not (animeReply['title'] in postedAnimeTitles):
                postedAnimeTitles.append(animeReply['title'])
                commentReply += animeReply['comment']
            

        if mangaArray:
            commentReply += '\n\n'

        #Adding all the manga to the final comment
        for i, mangaReply in enumerate(mangaArray):
            if not (i is 0):
                commentReply += '\n\n'
            
            if not (mangaReply['title'] in postedMangaTitles):
                postedMangaTitles.append(mangaReply['title'])
                commentReply += mangaReply['comment']

        if lnArray:
            commentReply += '\n\n'

        #Adding all the manga to the final comment
        for i, lnReply in enumerate(lnArray):
            if not (i is 0):
                commentReply += '\n\n'
            
            if not (lnReply['title'] in postedLNTitles):
                postedLNTitles.append(lnReply['title'])
                commentReply += lnReply['comment']

        #If there are more than 10 requests, shorten them all 
        if not (commentReply is '') and (len(animeArray) + len(mangaArray)+ len(lnArray) >= 10):
            commentReply = re.sub(r"\^\((.*?)\)", "", commentReply, flags=re.M)

    #If there was actually something found, add the signature and post the comment to Reddit. Then, add the comment to the "already seen" database.
    if commentReply is not '':
        '''if (comment.author.name == 'treborabc'):
            commentReply = '[No.](https://www.reddit.com/r/anime_irl/comments/4sba1n/anime_irl/d58xkha)'''
        
        commentReply += Config.getSignature(comment.permalink)

        commentReply += Reference.get_bling(comment.author.name)

        if is_edit:
            return commentReply
        else:
            try:
                comment.reply(commentReply)
                print("Comment made.\n")
            except praw.errors.Forbidden:
                print('Request from banned subreddit: ' + str(comment.subreddit) + '\n')
            except Exception:
                traceback.print_exc()

            try:
                DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, True)
            except:
                traceback.print_exc()
    else:
        try:
            if is_edit:
                return None
            else:
                DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, False)
        except:
            traceback.print_exc()

Example 190

Project: Roboragi
Source File: AnimeBot.py
View license
def process_comment(comment, is_edit=False):
    #Anime/Manga requests that are found go into separate arrays
    animeArray = []
    mangaArray = []
    lnArray = []

    #ignores all "code" markup (i.e. anything between backticks)
    comment.body = re.sub(r"\`(?s)(.*?)\`", "", comment.body)
    
    #This checks for requests. First up we check all known tags for the !stats request
    if re.search('({!stats.*?}|{{!stats.*?}}|<!stats.*?>|<<!stats.*?>>)', comment.body, re.S) is not None:
        username = re.search('[uU]\/([A-Za-z0-9_-]+?)(>|}|$)', comment.body, re.S)
        subreddit = re.search('[rR]\/([A-Za-z0-9_]+?)(>|}|$)', comment.body, re.S)

        if username:
            commentReply = CommentBuilder.buildStatsComment(username=username.group(1))
        elif subreddit:
            commentReply = CommentBuilder.buildStatsComment(subreddit=subreddit.group(1))
        else:
            commentReply = CommentBuilder.buildStatsComment()
    else:
        
        #The basic algorithm here is:
        #If it's an expanded request, build a reply using the data in the braces, clear the arrays, add the reply to the relevant array and ignore everything else.
        #If it's a normal request, build a reply using the data in the braces, add the reply to the relevant array.

        #Counts the number of expanded results vs total results. If it's not just a single expanded result, they all get turned into normal requests.
        numOfRequest = 0
        numOfExpandedRequest = 0
        forceNormal = False

        for match in re.finditer("\{{2}([^}]*)\}{2}|\<{2}([^>]*)\>{2}", comment.body, re.S):
            numOfRequest += 1
            numOfExpandedRequest += 1
            
        for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))|(?<=(?<!\<)\<)([^\<\>]*)(?=\>(?!\>))", comment.body, re.S):
            numOfRequest += 1

        if (numOfExpandedRequest >= 1) and (numOfRequest > 1):
            forceNormal = True

        #Expanded Anime
        for match in re.finditer("\{{2}([^}]*)\}{2}", comment.body, re.S):
            reply = ''

            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildAnimeReply(match.group(1), False, comment)
            else:
                reply = Search.buildAnimeReply(match.group(1), True, comment)                    

            if (reply is not None):
                animeArray.append(reply)

        #Normal Anime  
        for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))", comment.body, re.S):
            reply = Search.buildAnimeReply(match.group(1), False, comment)
            
            if (reply is not None):
                animeArray.append(reply)

        #Expanded Manga
        #NORMAL EXPANDED
        for match in re.finditer("\<{2}([^>]*)\>{2}(?!(:|\>))", comment.body, re.S):
            reply = ''
            
            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildMangaReply(match.group(1), False, comment)
            else:
                reply = Search.buildMangaReply(match.group(1), True, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #AUTHOR SEARCH EXPANDED
        for match in re.finditer("\<{2}([^>]*)\>{2}:\(([^)]+)\)", comment.body, re.S):
            reply = ''
            
            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)
            else:
                reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), True, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #Normal Manga
        #NORMAL
        for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]+)\>(?!(:|\>))", comment.body, re.S):
            reply = Search.buildMangaReply(match.group(1), False, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #AUTHOR SEARCH
        for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]*)\>:\(([^)]+)\)", comment.body, re.S):
            reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)

            if (reply is not None):
                mangaArray.append(reply)

        #Expanded LN
        for match in re.finditer("\]{2}([^]]*)\[{2}", comment.body, re.S):
            reply = ''

            if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
                reply = Search.buildLightNovelReply(match.group(1), False, comment)
            else:
                reply = Search.buildLightNovelReply(match.group(1), True, comment)                    

            if (reply is not None):
                lnArray.append(reply)

        #Normal LN  
        for match in re.finditer("(?<=(?<!\])\])([^\]\[]*)(?=\[(?!\[))", comment.body, re.S):
            reply = Search.buildLightNovelReply(match.group(1), False, comment)
            
            if (reply is not None):
                lnArray.append(reply)
            
        #Here is where we create the final reply to be posted

        #The final comment reply. We add stuff to this progressively.
        commentReply = ''

        #Basically just to keep track of people posting the same title multiple times (e.g. {Nisekoi}{Nisekoi}{Nisekoi})
        postedAnimeTitles = []
        postedMangaTitles = []
        postedLNTitles = []

        #Adding all the anime to the final comment. If there's manga too we split up all the paragraphs and indent them in Reddit markup by adding a '>', then recombine them
        for i, animeReply in enumerate(animeArray):
            if not (i is 0):
                commentReply += '\n\n'

            if not (animeReply['title'] in postedAnimeTitles):
                postedAnimeTitles.append(animeReply['title'])
                commentReply += animeReply['comment']
            

        if mangaArray:
            commentReply += '\n\n'

        #Adding all the manga to the final comment
        for i, mangaReply in enumerate(mangaArray):
            if not (i is 0):
                commentReply += '\n\n'
            
            if not (mangaReply['title'] in postedMangaTitles):
                postedMangaTitles.append(mangaReply['title'])
                commentReply += mangaReply['comment']

        if lnArray:
            commentReply += '\n\n'

        #Adding all the manga to the final comment
        for i, lnReply in enumerate(lnArray):
            if not (i is 0):
                commentReply += '\n\n'
            
            if not (lnReply['title'] in postedLNTitles):
                postedLNTitles.append(lnReply['title'])
                commentReply += lnReply['comment']

        #If there are more than 10 requests, shorten them all 
        if not (commentReply is '') and (len(animeArray) + len(mangaArray)+ len(lnArray) >= 10):
            commentReply = re.sub(r"\^\((.*?)\)", "", commentReply, flags=re.M)

    #If there was actually something found, add the signature and post the comment to Reddit. Then, add the comment to the "already seen" database.
    if commentReply is not '':
        '''if (comment.author.name == 'treborabc'):
            commentReply = '[No.](https://www.reddit.com/r/anime_irl/comments/4sba1n/anime_irl/d58xkha)'''
        
        commentReply += Config.getSignature(comment.permalink)

        commentReply += Reference.get_bling(comment.author.name)

        if is_edit:
            return commentReply
        else:
            try:
                comment.reply(commentReply)
                print("Comment made.\n")
            except praw.errors.Forbidden:
                print('Request from banned subreddit: ' + str(comment.subreddit) + '\n')
            except Exception:
                traceback.print_exc()

            try:
                DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, True)
            except:
                traceback.print_exc()
    else:
        try:
            if is_edit:
                return None
            else:
                DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, False)
        except:
            traceback.print_exc()

Example 191

Project: txt2ipa
Source File: ipaconvert.py
View license
def ipa(text): # Generates narrow transcription of Tamil texts 

    text=" " + text + " "
#    text = """ %s """ % text

    # Move Punctuations  
    repl = lambda m: " " + m.group(1) + " "        
    text = re.sub("([\\,\\\,\.\!\?\"\'\"\(\)])", repl, text)
    # text = re.sub("/(?<=[\w])([\\,\.\!\?\"\'\"\(\)])/",repl,text)
    # text = re.sub("/([\\,\.\!\?\"\'\"\(\)])(?=[\w])/",repl,text)

    # The comments below refer to the implementation of transcription rules as described in the
    # book - Panniru Thirumurai Olipeyarppu by Punal K Murugaiyan

    # Dipthongs 

    text = text.replace("ai","ay") # 5.3.3.1 ii Palatal Approximant ... # ai

    text = text.replace("au","av") # au - dipthong replacement

    # Grantha 

    text = text.replace("j","ʤ")
    text = text.replace("h","ɦ")
    text = text.replace("S","ʂ")
    text = text.replace("srI","ʂrI")

    # Mey 

    # Vallinam 

    # pa 

    text = re.sub(r"(?<=[aAiIuUeEoO])p(?=[aAiIuUeEoO])","β",text) # 5.3.1.6 iii Voiced Bilabial Fricative
    text = re.sub(r"(?<=[yrlvZL])p","β",text) # 5.3.1.6 iii

    text = re.sub(r"(?<=[GJnVmN])p","b",text) #5.3.1.6 ii voiced bilabial plosive

    # 5.3.1.6 i no replacement 

    # ta 

    text = re.sub(r"(?<=[aAiIuUeEoOyrlvZL])t(?=[aAiIuUeEoO])","ð",text) # 5.3.1.5 iii Voiced dental Fricative

    text = re.sub(r"(?<=[nV])t","d̪",text) # 5.3.1.5 ii Voiced dental plosive

    text = re.sub(r"t","t̪",text) # 5.3.1.5 i Voiceless dental plosive

    # Ra 

    text = text.replace("XX","t̺t̺ʳ") # 5.3.1.4 ii & ii find correct name

    text = re.sub(r"(?<=V)X","d̺ʳ",text) # 5.3.1.4 iii

    # 5.3.1.4 iv & v implemented in idaiyinam section


    # Ta  

    text = re.sub(r"(?<=[aAiIuUeEoO])T(?=[aAiIuUeEoO])","ɽ",text) # 5.3.1.3 iii Retroflex Flap

    text = re.sub(r"(?<=[N])T","ɖ",text) # 5.3.1.3 ii Voiced Retroflex Plosive | VT ?

    text = text.replace("T","ʈ") # 5.3.1.3 i Voiceless Retroflex Plosive

    # ca 

    text = re.sub(r"(?<=[aAiIuUeEoOl])c(?=[aAiIuUeEoO])","s",text) # 5.3.1.2 iii voiceless alveolar fricatives 
    
    repl = lambda m: m.group(1) + "s"   
    text = re.sub(r"(\s)c",repl,text) # 5.3.1.2 iii

    text = re.sub(r"(V)c",repl,text)

    text = re.sub(r"(?<=[J])c","ʤ",text) # 5.3.1.2 ii Voiced Post Alveolar affricate - Symbol Changed : d͡ʒ

    text = re.sub(r"c","ʧ",text) # 5.3.1.2 i Voicless Post Alveolar Affrivate - Symbol Changed : t͡ʃ



    # ka 

    text = re.sub(r"Gk(?=[iI])","ŋʲgʲ",text) # 5.3.2.1 ii Palatized Velar Nasal

    text = text.replace("Gk","ŋg") # 5.3.2.1 Velar Nasal

    text = re.sub(r"(?<=[aAiIuUeEoO])k(?=[iI])","ç",text) # 5.3.1.1 viii voiceless palatal fricative

    #yrZlLv assumed above. Missing in definiation : eykiya -> eyçiya  aarkiya -> aarçiya....

    text = re.sub(r"(?<=r)k(?=[aAuUeEoO])","ɣ",text) # 5.3.1.1 Vii Voiced Velar Fricative

    text = re.sub(r"(?<=[aAiIuUeEoO])k(?=[aAuUeEoO])","x",text) # 5.3.1.1 vi Voicless Velar Fricative

    text = re.sub(r"(?<=[ylvZL])k(?=[aAuUeEoO])","x",text) # above

    text = re.sub(r"ykk","jcc",text) # 5.3.1.1 v voiceless palatal plosive

    text = re.sub(r"jkk","jcc",text) # above

    text = re.sub(r"(?<=[rylvZLGVNaAiIuUeEoO])k(?=[iI])","gʲ",text) # 5.3.1.1 iv Voiced Palatized Velar Plosive

    text = re.sub(r"(?<=[NVmn])k(?=[aAuUeEoO])","g",text) # 5.3.1.1 iii voiced velar plosive

    text = re.sub(r"(?<=k)k(?=[iI])","kʲ",text) # 5.3.1.1 ii Voiceless velar plosive

    # 5.3.1.1 i no relacement #


    # Idaiyinam 

    text = text.replace("Z","ɻ") #5.3.3.6  Retroflex Approximant

    text = re.sub(r"(?<=[aAiIuUeEoO])L(?=[aAiIuUeEoO])","ɭʼ",text)  #5.3.3.5 i Lateral Approximant - Ejective

    text = text.replace("L","ɭ") # 5.3.3.5 ii Lateral Approximant

    # 5.3.3.4 no change 

    text = re.sub(r"(?<=[aAiIuUeEoO])[rX](?=[aAiIuUeEoO])","ɾ",text) # 5.3.3.3 i Alveolar Tap

    # 5.3.3.3 ii - pure consonant r - no replacement 

    text = re.sub(r"X(?!=[aAiIuUeEoO])","r",text) # 5.3.3.3 ii Trill

    text = re.sub(r"(?<=[aAiIuUeEoO])v(?=[aAiIuUeEoO])","ʋ",text) # 5.3.3.2 ii labio-dental approximant
    text = re.sub(r"(\s)v(?=[aAiIuUeEoO])",lambda m: m.group(1)+"ʋ",text) # 5.3.3.2 ii
    text = text.replace("vv","ʊ̯ʋ") # 5.3.3.2 i near-close near-back rounded vowel - part of a dipthong
    text = text.replace("v","ʋ")

    text = re.sub(r"yy","jɪ̯",text) # 5.3.3.1 i near-close near-front unrounded vowel - part of a dipthong
    text = re.sub(r"y","ɪ̯",text) # 5.3.3.1 i near-close near-front unrounded vowel - part of a dipthong

    # Mellinam 

    # 5.3.2.6 no replacement 

    text = re.sub(r"[Vn]","n̺",text) # 5.3.2.4 Alveolar Nasal (Check Actual name in Wikipedia)

    text = text.replace("n̺d̪","n̪d̪") # 5.3.2.5 Dental Nasal

    text = re.sub(r"(?<=[aAiIuUeEoO])N(?=[aAiIuUeEoO])","ɳʼ",text) # 5.3.2.3 ii Retroflex Nasal Ejective

    text = text.replace("N","ɳ") # 5.3.2.3 Retroflex Nasal

    text = text.replace("J","ɲ") # 5.3.2.3 Palatal Nasal

    text = re.sub(r"GG(?=[iI])","ŋʲŋʲ",text) # Assumed based on above

    text = text.replace("GG","ŋŋ") # Assumed based on above

    text = text.replace("G","ŋ") # Assumed based on above

    # Uyir 

    # Seperate Pure Vowel Combinations

    text = re.sub(r"([aAiIuUeEoO])([aAiIuUeEoO])",
            lambda m: m.group(1)+"_"+m.group(2),text)

    #return text

    # Long O 

    text=re.sub(r"o(\s)",lambda m: "o·"+m.group(1),text) # 5.2.5.2 v 

    text=re.sub(r"(\s)o(?!·)",lambda m: m.group(1)+"ʷoː",text) # 5.2.5.2 iii 
    text=re.sub(r"_o(?!·)","ʷoː",text) # 5.2.5.2 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·])o(?![ː·])","oː",text) # 5.2.5.2 i

    # Short o 

    text=re.sub(r"(\s)O(?!·)",lambda m: m.group(1)+"ʷo̞",text) # 5.2.5.1 iii 
    text=re.sub(r"_O(?!·)","ʷo̞",text) # 5.2.5.1 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞])O(?![ː·])","o̞",text) # 5.2.5.1 i

    # Adding extra symbol for Retroflex Consonants 

    retroflex = ["ɽ","ɖ","ʈ","ɳ","ɭ","ɻ"]

    for rf in retroflex:
          text = re.sub("/̞(?=" + rf + ")","̞˞",text)
	      
    # Long e 

    text=re.sub(r"e(\s)",lambda m: "e·"+m.group(1),text) # 5.2.4.2 v 

    text=re.sub(r"(\s)e(?!·)",lambda m: m.group(1)+"ʲeː",text) # 5.2.4.2 iii 
    text=re.sub(r"_e(?!·)","ʲeː",text) # 5.2.4.2 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞])e(?![ː·])","eː",text) # 5.2.5.2 i


    # short e 

    text=re.sub(r"(\s)E(?!·)",lambda m: m.group(1)+"ʲɛ̝",text) # 5.2.4.1 iii 
    text=re.sub(r"_E(?!·)","ʲɛ̝",text) # 5.2.4.1 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])E(?![ː·])","ɛ̝",text) # 5.2.5.4 i

    # Adding extra symbol for Retroflex Consonants 

    for rf in retroflex:
          text = re.sub("/̝(?=" + rf + ")","̝˞",text)

    # short u 

    text = re.sub(r"(?<!u)(\S)(?<![bʋpmβ])u",lambda m: m.group(1)+"ɨ",text) # 5.2.3.1 v

    text=re.sub(r"(\s)u(?!·)",lambda m: m.group(1)+"ʷʊ",text) # 5.2.3.1 iii 

    text=re.sub(r"_u(?!·)","ʷʊ",text) # 5.2.3.1 iii - puththi_Ottum

    text = re.sub(r"(?<!u\S)([bʋpmβ])u",lambda m: m.group(1)+"ʉ̩",text) # 5.2.3.1  Vii

    text = re.sub(r"([bʋpβm])u",lambda m: m.group(1)+"ʉ̩",text) # 5.2.3.1  Vii

    text = re.sub(r"(?<![bʋpβm])u(\s)",lambda m: "ɨ"+m.group(1),text) # 5.2.3.1 v
    
    repl = lambda m: m.group(1) + m.group(2) + "ʊ"
    
    text= re.sub(r"(\s)(\S)(ɨ|ʉ̩)",repl,text) # 5.2.5.2 i

    text= re.sub(r"(U)(\S{1,2})ɨ",repl,text) # 5.2.5.2 i 

    text= re.sub(r"(ʊ)(\S{1,2})ɨ",repl,text)

    text= re.sub(r"(ʊ)(\S{1,2})ʉ̩",repl,text)

    text = re.sub(r"(?<![bʋβpm])ʊ(\s)",lambda m: "ɨ"+m.group(1),text) # 5.2.3.1 v

    text = re.sub(r"(?<=[bʋβpm])ʊ(\s)",lambda m: "ʉ̩"+m.group(1),text)

    for rf in retroflex:
          text = re.sub(r"ʊ(?=" + rf + ")","ʊ˞",text)
	      
    for rf in retroflex:
          text = re.sub(r"ʉ̩(?=" + rf + ")","ʉ̩˞",text)
	      
    for rf in retroflex:
          text = re.sub(r"ɨ(?=" + rf + ")","ɨ˞",text) 

    # 
     
    text = re.sub(r"\S(<=!u)\Su","ɨ",text) # 5.2.3.1 v
	      
    # Long u 

    text=re.sub(r"U(\s)",lambda m: "u·"+m.group(1),text) # 5.2.3.2 v 

    text=re.sub(r"(\s)U(?!·)",lambda m: m.group(1)+"ʷuː",text) # 5.2.3.2 iii 
    text=re.sub(r"_U(?!·)","ʷuː",text) # 5.2.3.2 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])U(?![ː·])","uː",text) # 5.2.3.2 i

    # short i 

    text=re.sub(r"i(\s)",lambda m: "ɪ·"+m.group(1),text) # 5.2.2.1 iii 

    text=re.sub(r"(\s)i(?!·)",lambda m: m.group(1)+"ʲɪ",text) # 5.2.4.2 iii 
    text=re.sub(r"_i(?!·)","ʲɪ",text) # 5.2.4.2 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])i(?![ː·])","ɪ",text) # 5.2.5.2 i

    for rf in retroflex:
          text = re.sub(r"ɪ(?=" + rf + ")","ɪ˞",text)

    # Long i 

    text=re.sub(r"I(\s)",lambda m: "i·"+m.group(1),text) # 5.2.2.2 v 

    text=re.sub(r"(\s)I(?!·)",lambda m: m.group(1)+"ʲiː",text) # 5.2.2.2 iii 
    text=re.sub(r"_I(?!·)","ʲiː",text) # 5.2.2.2 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])I(?![ː·])","iː",text) # 5.2.2.2 i

    # Long A 

    text=re.sub(r"(\s)A(?!·)",lambda m: m.group(1)+"ˀɑː",text) # 5.2.1.2 iii 
    text=re.sub(r"_A(?!·)","ˀɑː",text) # 5.2.1.2 iii - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])A(?![ː·])","ɑː",text) # 5.2.1.2 i

    # short a 

    # Transcription of Abbreviation Ignored 

    text=re.sub(r"(\s)a(?!·)",lambda m: m.group(1)+"ˀʌ",text) # 5.2.1.1 vi 
    text=re.sub(r"_a(?!·)","ˀʌ",text) # 5.2.1.1 vi - puththi_Ottum

    text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])a(?![ː·])","ʌ",text) # 5.2.1.1 i

    text=re.sub(r"ʌ(\s)",lambda m: "ə"+m.group(1),text) # 5.2.1.1 iii

    for rf in retroflex:
          text = re.sub(r"ʌ(?=" + rf + ")","ʌ˞",text)
          
    # Aytham 

    text = text.replace("K","x")
    text = text.replace("xt̪","xð")
    text = text.replace("xk","xx")
          
        
    # Adding extra symbol for Retroflex Consonants Common of U & O 

    # Regex won't accept (?=[Multiple Unicode Chars]) so separating each charcter 

    for rf in retroflex:
          text = re.sub(r"ː(?=" + rf + ")","˞ː",text)

    # Fixing kaaZBbu kaLaip kaLaippu bugs 

    text = text.replace("βp","pp")
    text = text.replace("β ","p ")
    text = re.sub(r"ʧ([ \n])s",lambda m: "ʧ"+m.group(1)+"ʧ",text) # ac samayam -> ac camayam \\ Check with Newlines ac \n samayam

    # New IPA Convensions 

    text = text.replace("ː","ː")
    text = text.replace("·","ˑ") #
    text = text.replace("ʧ","tʃ")
    text = text.replace("ʤ","dʒ")

    #ɨɭ (i- + Refelex -> <u> <Retroflex>

    text = text.lstrip()

    return text

Example 192

Project: twistranet
Source File: twistranet_project.py
View license
def twistranet_project():
    """
    Copies the contents of the project_template directory to a new directory
    specified as an argument to the command line.
    """
    # That is just to check that we can import the TN package.
    # If something goes wrong here, just add a --pythonpath option.     
    parser = OptionParser(
        usage = "usage: %prog [options] [<template>] <project_path>\n"
            "  where project_name is the name of the directory that will be created for your site,\n"
            "  <path> is the (optional) path where you want to install it.\n"
    )
    parser.add_option("-n", "--no-bootstrap",
        action="store_false", dest="bootstrap", default=True,
        help="Don't bootstrap project immediately. Use this if you want to review your settings before bootstraping.",
    )
    parser.add_option("-d", "--develmode",
        action="store_true", dest="develmode", default=False,
        help="Use this option if you want to work on sources in your project. eg: static files",
    )
    (options, args) = parser.parse_args()

    # Check template and path args
    if len(args) < 1:
        parser.error("project_path must be specified.")
    elif len(args) == 1:
        project_path = args[0]
        project_template = 'default'
    elif len(args) == 2:
        project_path = args[1]
        project_template = args[0]
    
    # Check if we can import TN.
    import twistranet
    
    # We decompose the given path to split installdir and project_name
    project_path = os.path.abspath(project_path)
    if os.path.lexists(project_path):
        parser.error("Directory '%s' already exists. "
            "Please specify a non-existing directory." % (project_path, ))
    project_name = os.path.split(project_path)[1]
    
    # Check if project_name is correct
    try:
        eval("%s is True" % project_name)
    except SyntaxError:
        parser.error("Directory '%s' must be a valid python identifier." % project_name)
    except:
        pass
    # Ensure the given directory name doesn't clash with an existing Python
    # package/module.
    try:
        __import__(project_name)
    except ImportError:
        pass
    except ValueError:
        pass # It's ok to install in the same directory ;)
    else:
        parser.error("'%s' conflicts with the name of an existing "
            "Python module and cannot be used as a project name. "
            "Please try another name." % project_name)

    # Build the project up copying over the twistranet project_template
    twist_package = __import__('twistranet')
    twist_package_path = os.path.dirname(os.path.abspath(twist_package.__file__))
    template_dir = os.path.join(twist_package_path, "project_templates", "default")
    if not os.path.isdir(template_dir):
        parser.error("Template '%s' is invalid." % project_template)
    try:
        shutil.copytree(template_dir, project_path, ignore=shutil.ignore_patterns(*IGNORE_PATTERNS))
    except AttributeError:
        print "shutil.copytree is likely not to have the 'ignore' attribute available.\n"
        shutil.copytree(template_dir, project_path)
    
    replaceable_files = [(os.path.join(project_path, "local_settings.py")),]

    # project variables replaced in the project files
    replacement = {
        "SECRET_KEY.*$":        "SECRET_KEY = '%s'" % (uuid1(), ),
        "__INSTANCE_NAME__":    '%s' % project_name,
    }

    # If project_template <> default, we copy the project_template-specific files as well
    if project_template != "default":
        source_root = os.path.join(twist_package_path, "project_templates", project_template)
        if not os.path.isdir(source_root):
            source_root = os.path.abspath(os.path.join(os.path.curdir, project_template))
        if not os.path.isdir(source_root):
            raise ValueError("Invalid template directory: '%s'" % source_root)
        dest_root = project_path
        for root, dirs, files in os.walk(source_root):
            # Ugly wat to deal with dotted dirs
            if '/.' in root:
                continue
            relative_root = root[len(source_root) + 1:]
            
            for d in dirs:
                if d.startswith('.'):
                    continue
                dest_dir = os.path.join(dest_root, relative_root, d)
                if not os.path.isdir(dest_dir):
                    os.mkdir(dest_dir)
                    
            for fname in files:
                # Ignore doted files, and rename if it contains any replacement string
                if fname.startswith('.'):
                    continue
                # if a file is named with __INSTANCE_NAME__.ext rename it with project_name
                dname = fname
                for regex, repl in replacement.items():
                    dname = re.sub(regex, repl, dname)

                # Actually copy
                dest_file = os.path.join(dest_root, relative_root, dname)
                shutil.copy(
                    os.path.join(source_root, root, fname),
                    dest_file,
                )
                # in project files some files must be changed (as done for local_settings, see bellow)
                replaceable_files.append(dest_file)


    # replace some settings.
    for replaceable_path in replaceable_files:
        with open(replaceable_path, "r") as f:
            data = f.read()
        with open(replaceable_path, "w") as f:
            for regex, repl in replacement.items():
                data = re.sub(regex, repl, data)
            f.write(data)
            f.close()

    # we append project_path to sys.path, used for bootstrap and for devel mode configuration (-d).
    sys.path.insert(0, project_path)        # Here is how we're gonna find the 'settings' module from here.
    # XXX NOT VERY DJANGOISH TO USE JUST 'settings' HERE !
    os.environ["DJANGO_SETTINGS_MODULE"] = 'settings'
    os.environ["TWISTRANET_NOMAIL"] = "1"   # Disable emails
    import settings
    
    # update settings.TWISTRANET_STATIC_PATH in devel mode
    # to use theme from product itself
    if options.develmode:
        from django.utils.importlib import import_module
        theme_app = import_module(settings.TWISTRANET_THEME_APP)
        theme_app_dir = os.path.split(theme_app.__file__)[0]
        DEVEL_TWISTRANET_STATIC_PATH = os.path.abspath(os.path.join(theme_app_dir, 'static'))
        DEVEL_TWISTRANET_LOCALE_PATHS = "("
        for twapp in ('core', 'notifier', 'search', 'sharing', 'tagging', 'twistapp', ):
            DEVEL_TWISTRANET_LOCALE_PATHS += "r'%s', " %os.path.abspath(os.path.join(twist_package_path, twapp, 'locale'))
        DEVEL_TWISTRANET_LOCALE_PATHS += ")"
        settings_path = os.path.join(project_path, "settings.py")
        f = open(settings_path, "r")
        data = f.read()
        f.close()
        f = open(settings_path, "w")
        #  XXX TODO : change with a replace
        data += '''
# ADDED FOR DEVEL MODE ONLY

TWISTRANET_STATIC_PATH = r"%(static_path)s"
STATICFILES_DIRS = ( r"%(static_path)s",)
COMPRESS_ROOT = r"%(static_path)s"
LOCALE_PATHS = %(locale_paths)s

        ''' % { 'static_path': DEVEL_TWISTRANET_STATIC_PATH ,
                'locale_paths': DEVEL_TWISTRANET_LOCALE_PATHS,
              }


        f.write(data)
        f.close()
        # fix settings for the first server start
        settings.TWISTRANET_STATIC_PATH = DEVEL_TWISTRANET_STATIC_PATH
        settings.STATICFILES_DIRS = (DEVEL_TWISTRANET_STATIC_PATH ,)
        settings.COMPRESS_ROOT = DEVEL_TWISTRANET_STATIC_PATH

    # As we use a standard sqlite configuration, we can boostrap quite safely just now.
    # then we start the server
    if options.bootstrap:
        from django.core.management import call_command
        from django import conf
        # update static files,
        # excepted in devel mode
        if not options.develmode:
            call_command('twistranet_update')
        call_command('twistranet_bootstrap')
        
        # Now we can start the server!
        os.environ["TWISTRANET_NOMAIL"] = ""    # Re-enable emails
        call_command("runserver", "0.0.0.0:8000", use_reloader = False,  )

Example 193

Project: OpenUpgrade
Source File: escpos.py
View license
    def receipt(self,xml):
        """
        Prints an xml based receipt definition
        """

        def strclean(string):
            if not string:
                string = ''
            string = string.strip()
            string = re.sub('\s+',' ',string)
            return string

        def format_value(value, decimals=3, width=0, decimals_separator='.', thousands_separator=',', autoint=False, symbol='', position='after'):
            decimals = max(0,int(decimals))
            width    = max(0,int(width))
            value    = float(value)

            if autoint and math.floor(value) == value:
                decimals = 0
            if width == 0:
                width = ''

            if thousands_separator:
                formatstr = "{:"+str(width)+",."+str(decimals)+"f}"
            else:
                formatstr = "{:"+str(width)+"."+str(decimals)+"f}"


            ret = formatstr.format(value)
            ret = ret.replace(',','COMMA')
            ret = ret.replace('.','DOT')
            ret = ret.replace('COMMA',thousands_separator)
            ret = ret.replace('DOT',decimals_separator)

            if symbol:
                if position == 'after':
                    ret = ret + symbol
                else:
                    ret = symbol + ret
            return ret

        def print_elem(stylestack, serializer, elem, indent=0):

            elem_styles = {
                'h1': {'bold': 'on', 'size':'double'},
                'h2': {'size':'double'},
                'h3': {'bold': 'on', 'size':'double-height'},
                'h4': {'size': 'double-height'},
                'h5': {'bold': 'on'},
                'em': {'font': 'b'},
                'b':  {'bold': 'on'},
            }

            stylestack.push()
            if elem.tag in elem_styles:
                stylestack.set(elem_styles[elem.tag])
            stylestack.set(elem.attrib)

            if elem.tag in ('p','div','section','article','receipt','header','footer','li','h1','h2','h3','h4','h5'):
                serializer.start_block(stylestack)
                serializer.text(elem.text)
                for child in elem:
                    print_elem(stylestack,serializer,child)
                    serializer.start_inline(stylestack)
                    serializer.text(child.tail)
                    serializer.end_entity()
                serializer.end_entity()

            elif elem.tag in ('span','em','b','left','right'):
                serializer.start_inline(stylestack)
                serializer.text(elem.text)
                for child in elem:
                    print_elem(stylestack,serializer,child)
                    serializer.start_inline(stylestack)
                    serializer.text(child.tail)
                    serializer.end_entity()
                serializer.end_entity()

            elif elem.tag == 'value':
                serializer.start_inline(stylestack)
                serializer.pre(format_value( 
                                              elem.text,
                                              decimals=stylestack.get('value-decimals'),
                                              width=stylestack.get('value-width'),
                                              decimals_separator=stylestack.get('value-decimals-separator'),
                                              thousands_separator=stylestack.get('value-thousands-separator'),
                                              autoint=(stylestack.get('value-autoint') == 'on'),
                                              symbol=stylestack.get('value-symbol'),
                                              position=stylestack.get('value-symbol-position') 
                                            ))
                serializer.end_entity()

            elif elem.tag == 'line':
                width = stylestack.get('width')
                if stylestack.get('size') in ('double', 'double-width'):
                    width = width / 2

                lineserializer = XmlLineSerializer(stylestack.get('indent')+indent,stylestack.get('tabwidth'),width,stylestack.get('line-ratio'))
                serializer.start_block(stylestack)
                for child in elem:
                    if child.tag == 'left':
                        print_elem(stylestack,lineserializer,child,indent=indent)
                    elif child.tag == 'right':
                        lineserializer.start_right()
                        print_elem(stylestack,lineserializer,child,indent=indent)
                serializer.pre(lineserializer.get_line())
                serializer.end_entity()

            elif elem.tag == 'ul':
                serializer.start_block(stylestack)
                bullet = stylestack.get('bullet')
                for child in elem:
                    if child.tag == 'li':
                        serializer.style(stylestack)
                        serializer.raw(' ' * indent * stylestack.get('tabwidth') + bullet)
                    print_elem(stylestack,serializer,child,indent=indent+1)
                serializer.end_entity()

            elif elem.tag == 'ol':
                cwidth = len(str(len(elem))) + 2
                i = 1
                serializer.start_block(stylestack)
                for child in elem:
                    if child.tag == 'li':
                        serializer.style(stylestack)
                        serializer.raw(' ' * indent * stylestack.get('tabwidth') + ' ' + (str(i)+')').ljust(cwidth))
                        i = i + 1
                    print_elem(stylestack,serializer,child,indent=indent+1)
                serializer.end_entity()

            elif elem.tag == 'pre':
                serializer.start_block(stylestack)
                serializer.pre(elem.text)
                serializer.end_entity()

            elif elem.tag == 'hr':
                width = stylestack.get('width')
                if stylestack.get('size') in ('double', 'double-width'):
                    width = width / 2
                serializer.start_block(stylestack)
                serializer.text('-'*width)
                serializer.end_entity()

            elif elem.tag == 'br':
                serializer.linebreak()

            elif elem.tag == 'img':
                if 'src' in elem.attrib and 'data:' in elem.attrib['src']:
                    self.print_base64_image(elem.attrib['src'])

            elif elem.tag == 'barcode' and 'encoding' in elem.attrib:
                serializer.start_block(stylestack)
                self.barcode(strclean(elem.text),elem.attrib['encoding'])
                serializer.end_entity()

            elif elem.tag == 'cut':
                self.cut()
            elif elem.tag == 'partialcut':
                self.cut(mode='part')
            elif elem.tag == 'cashdraw':
                self.cashdraw(2)
                self.cashdraw(5)

            stylestack.pop()

        try:
            stylestack      = StyleStack() 
            serializer      = XmlSerializer(self)
            root            = ET.fromstring(xml.encode('utf-8'))

            self._raw(stylestack.to_escpos())

            print_elem(stylestack,serializer,root)

            if 'open-cashdrawer' in root.attrib and root.attrib['open-cashdrawer'] == 'true':
                self.cashdraw(2)
                self.cashdraw(5)
            if not 'cut' in root.attrib or root.attrib['cut'] == 'true' :
                self.cut()

        except Exception as e:
            errmsg = str(e)+'\n'+'-'*48+'\n'+traceback.format_exc() + '-'*48+'\n'
            self.text(errmsg)
            self.cut()

            raise e

Example 194

Project: OpenUpgrade
Source File: translate.py
View license
def trans_generate(lang, modules, cr):
    dbname = cr.dbname

    registry = openerp.registry(dbname)
    trans_obj = registry['ir.translation']
    model_data_obj = registry['ir.model.data']
    uid = 1

    query = 'SELECT name, model, res_id, module' \
            '  FROM ir_model_data'

    query_models = """SELECT m.id, m.model, imd.module
            FROM ir_model AS m, ir_model_data AS imd
            WHERE m.id = imd.res_id AND imd.model = 'ir.model' """

    if 'all_installed' in modules:
        query += ' WHERE module IN ( SELECT name FROM ir_module_module WHERE state = \'installed\') '
        query_models += " AND imd.module in ( SELECT name FROM ir_module_module WHERE state = 'installed') "
    query_param = None
    if 'all' not in modules:
        query += ' WHERE module IN %s'
        query_models += ' AND imd.module in %s'
        query_param = (tuple(modules),)
    else:
        query += ' WHERE module != %s'
        query_models += ' AND imd.module != %s'
        query_param = ('__export__',)
    query += ' ORDER BY module, model, name'
    query_models += ' ORDER BY module, model'

    cr.execute(query, query_param)

    _to_translate = set()
    def push_translation(module, type, name, id, source, comments=None):
        # empty and one-letter terms are ignored, they probably are not meant to be
        # translated, and would be very hard to translate anyway.
        sanitized_term = (source or '').strip()
        try:
            # verify the minimal size without eventual xml tags
            # wrap to make sure html content like '<a>b</a><c>d</c>' is accepted by lxml
            wrapped = "<div>%s</div>" % sanitized_term
            node = etree.fromstring(wrapped)
            sanitized_term = etree.tostring(node, encoding='UTF-8', method='text')
        except etree.ParseError:
            pass
        # remove non-alphanumeric chars
        sanitized_term = re.sub(r'\W+', '', sanitized_term)
        if not sanitized_term or len(sanitized_term) <= 1:
            return

        tnx = (module, source, name, id, type, tuple(comments or ()))
        _to_translate.add(tnx)

    def push(mod, type, name, res_id, term):
        term = (term or '').strip()
        if len(term) > 2 or term in ENGLISH_SMALL_WORDS:
            push_translation(mod, type, name, res_id, term)

    def get_root_view(xml_id):
        view = model_data_obj.xmlid_to_object(cr, uid, xml_id)
        if view:
            while view.mode != 'primary':
                view = view.inherit_id
        xml_id = view.get_external_id(cr, uid).get(view.id, xml_id)
        return xml_id

    for (xml_name,model,res_id,module) in cr.fetchall():
        module = encode(module)
        model = encode(model)
        xml_name = "%s.%s" % (module, encode(xml_name))

        if model not in registry:
            _logger.error("Unable to find object %r", model)
            continue

        Model = registry[model]
        if not Model._translate:
            # explicitly disabled
            continue

        obj = Model.browse(cr, uid, res_id)
        if not obj.exists():
            _logger.warning("Unable to find object %r with id %d", model, res_id)
            continue

        if model=='ir.model.fields':
            try:
                field_name = encode(obj.name)
            except AttributeError, exc:
                _logger.error("name error in %s: %s", xml_name, str(exc))
                continue
            field_model = registry.get(obj.model)
            if (field_model is None or not field_model._translate or
                    field_name not in field_model._fields):
                continue
            field_def = field_model._fields[field_name]

            if hasattr(field_def, 'selection') and isinstance(field_def.selection, (list, tuple)):
                name = "%s,%s" % (encode(obj.model), field_name)
                for dummy, val in field_def.selection:
                    push_translation(module, 'selection', name, 0, encode(val))

        elif model=='ir.actions.report.xml':
            name = encode(obj.report_name)
            fname = ""
            if obj.report_rml:
                fname = obj.report_rml
                parse_func = trans_parse_rml
                report_type = "report"
            elif obj.report_xsl:
                continue
            if fname and obj.report_type in ('pdf', 'xsl'):
                try:
                    report_file = misc.file_open(fname)
                    try:
                        d = etree.parse(report_file)
                        for t in parse_func(d.iter()):
                            push_translation(module, report_type, name, 0, t)
                    finally:
                        report_file.close()
                except (IOError, etree.XMLSyntaxError):
                    _logger.exception("couldn't export translation for report %s %s %s", name, report_type, fname)

        for field_name, field_def in obj._fields.iteritems():
            if getattr(field_def, 'translate', None):
                name = model + "," + field_name
                try:
                    value = obj[field_name] or ''
                except Exception:
                    continue
                for term in set(field_def.get_trans_terms(value)):
                    push_translation(module, 'model', name, xml_name, encode(term))

        # End of data for ir.model.data query results

    cr.execute(query_models, query_param)

    def push_constraint_msg(module, term_type, model, msg):
        if not hasattr(msg, '__call__'):
            push_translation(encode(module), term_type, encode(model), 0, encode(msg))

    def push_local_constraints(module, model, cons_type='sql_constraints'):
        """Climb up the class hierarchy and ignore inherited constraints
           from other modules"""
        term_type = 'sql_constraint' if cons_type == 'sql_constraints' else 'constraint'
        msg_pos = 2 if cons_type == 'sql_constraints' else 1
        for cls in model.__class__.__mro__:
            if getattr(cls, '_module', None) != module:
                continue
            constraints = getattr(cls, '_local_' + cons_type, [])
            for constraint in constraints:
                push_constraint_msg(module, term_type, model._name, constraint[msg_pos])
            
    for (_, model, module) in cr.fetchall():
        if model not in registry:
            _logger.error("Unable to find object %r", model)
            continue

        model_obj = registry[model]

        if model_obj._constraints:
            push_local_constraints(module, model_obj, 'constraints')

        if model_obj._sql_constraints:
            push_local_constraints(module, model_obj, 'sql_constraints')

    installed_modules = map(
        lambda m: m['name'],
        registry['ir.module.module'].search_read(cr, uid, [('state', '=', 'installed')], fields=['name']))

    path_list = [(path, True) for path in openerp.modules.module.ad_paths]
    # Also scan these non-addon paths
    for bin_path in ['osv', 'report', 'modules', 'service', 'tools']:
        path_list.append((os.path.join(config.config['root_path'], bin_path), True))

    # non-recursive scan for individual files in root directory but without
    # scanning subdirectories that may contain addons
    path_list.append((config.config['root_path'], False))
    _logger.debug("Scanning modules at paths: %s", path_list)

    def get_module_from_path(path):
        for (mp, rec) in path_list:
            mp = os.path.join(mp, '')
            if rec and path.startswith(mp) and os.path.dirname(path) != mp:
                path = path[len(mp):]
                return path.split(os.path.sep)[0]
        return 'base' # files that are not in a module are considered as being in 'base' module

    def verified_module_filepaths(fname, path, root):
        fabsolutepath = join(root, fname)
        frelativepath = fabsolutepath[len(path):]
        display_path = "addons%s" % frelativepath
        module = get_module_from_path(fabsolutepath)
        if ('all' in modules or module in modules) and module in installed_modules:
            if os.path.sep != '/':
                display_path = display_path.replace(os.path.sep, '/')
            return module, fabsolutepath, frelativepath, display_path
        return None, None, None, None

    def babel_extract_terms(fname, path, root, extract_method="python", trans_type='code',
                               extra_comments=None, extract_keywords={'_': None}):
        module, fabsolutepath, _, display_path = verified_module_filepaths(fname, path, root)
        extra_comments = extra_comments or []
        if not module: return
        src_file = open(fabsolutepath, 'r')
        try:
            for extracted in extract.extract(extract_method, src_file,
                                             keywords=extract_keywords):
                # Babel 0.9.6 yields lineno, message, comments
                # Babel 1.3 yields lineno, message, comments, context
                lineno, message, comments = extracted[:3]
                push_translation(module, trans_type, display_path, lineno,
                                 encode(message), comments + extra_comments)
        except Exception:
            _logger.exception("Failed to extract terms from %s", fabsolutepath)
        finally:
            src_file.close()

    for (path, recursive) in path_list:
        _logger.debug("Scanning files of modules at %s", path)
        for root, dummy, files in osutil.walksymlinks(path):
            for fname in fnmatch.filter(files, '*.py'):
                babel_extract_terms(fname, path, root)
            # mako provides a babel extractor: http://docs.makotemplates.org/en/latest/usage.html#babel
            for fname in fnmatch.filter(files, '*.mako'):
                babel_extract_terms(fname, path, root, 'mako', trans_type='report')
            # Javascript source files in the static/src/js directory, rest is ignored (libs)
            if fnmatch.fnmatch(root, '*/static/src/js*'):
                for fname in fnmatch.filter(files, '*.js'):
                    babel_extract_terms(fname, path, root, 'javascript',
                                        extra_comments=[WEB_TRANSLATION_COMMENT],
                                        extract_keywords={'_t': None, '_lt': None})
            # QWeb template files
            if fnmatch.fnmatch(root, '*/static/src/xml*'):
                for fname in fnmatch.filter(files, '*.xml'):
                    babel_extract_terms(fname, path, root, 'openerp.tools.translate:babel_extract_qweb',
                                        extra_comments=[WEB_TRANSLATION_COMMENT])
            if not recursive:
                # due to topdown, first iteration is in first level
                break

    out = []
    # translate strings marked as to be translated
    for module, source, name, id, type, comments in sorted(_to_translate):
        trans = '' if not lang else trans_obj._get_source(cr, uid, name, type, lang, source)
        out.append((module, type, name, id, source, encode(trans) or '', comments))
    return out

Example 195

Project: openstates
Source File: bills.py
View license
    def scrape_bill_type(self, chamber, session, bill_type, type_abbr,
            committee_abbr_regex=get_committee_name_regex()):

        if chamber == 'upper':
            chamber_name = 'SENATE'
        else:
            chamber_name = 'ASSEMBLY'

        bills = self.session.query(CABill).filter_by(
            session_year=session).filter_by(
            measure_type=type_abbr)

        for bill in bills:
            bill_session = session
            if bill.session_num != '0':
                bill_session += ' Special Session %s' % bill.session_num

            bill_id = bill.short_bill_id

            fsbill = Bill(bill_session, chamber, bill_id, '')

            # # Construct session for web query, going from '20092010' to '0910'
            # source_session = session[2:4] + session[6:8]

            # # Turn 'AB 10' into 'ab_10'
            # source_num = "%s_%s" % (bill.measure_type.lower(),
            #                         bill.measure_num)

            # Construct a fake source url
            source_url = ('http://leginfo.legislature.ca.gov/faces/'
                          'billNavClient.xhtml?bill_id=%s') % bill.bill_id

            fsbill.add_source(source_url)
            fsbill.add_version(bill_id, source_url, 'text/html')

            title = ''
            type_ = ['bill']
            subject = ''
            all_titles = set()

            # Get digest test (aka "summary") from latest version.
            if bill.versions:
                version = bill.versions[-1]
                nsmap = version.xml.nsmap
                xpath = '//caml:DigestText/xhtml:p'
                els = version.xml.xpath(xpath, namespaces=nsmap)
                chunks = []
                for el in els:
                    t = etree_text_content(el)
                    t = re.sub(r'\s+', ' ', t)
                    t = re.sub(r'\)(\S)', lambda m: ') %s' % m.group(1), t)
                    chunks.append(t)
                summary = '\n\n'.join(chunks)

            for version in bill.versions:
                if not version.bill_xml:
                    continue

                # CA is inconsistent in that some bills have a short title
                # that is longer, more descriptive than title.
                if bill.measure_type in ('AB', 'SB'):
                    impact_clause = clean_title(version.title)
                    title = clean_title(version.short_title)
                else:
                    impact_clause = None
                    if len(version.title) < len(version.short_title) and \
                            not version.title.lower().startswith('an act'):
                        title = clean_title(version.short_title)
                    else:
                        title = clean_title(version.title)

                if title:
                    all_titles.add(title)

                type_ = [bill_type]

                if version.appropriation == 'Yes':
                    type_.append('appropriation')
                if version.fiscal_committee == 'Yes':
                    type_.append('fiscal committee')
                if version.local_program == 'Yes':
                    type_.append('local program')
                if version.urgency == 'Yes':
                    type_.append('urgency')
                if version.taxlevy == 'Yes':
                    type_.append('tax levy')

                if version.subject:
                    subject = clean_title(version.subject)

            if not title:
                self.warning("Couldn't find title for %s, skipping" % bill_id)
                continue

            fsbill['title'] = title
            fsbill['summary'] = summary
            fsbill['type'] = type_
            fsbill['subjects'] = filter(None, [subject])
            fsbill['impact_clause'] = impact_clause

            # We don't want the current title in alternate_titles
            all_titles.remove(title)

            fsbill['alternate_titles'] = list(all_titles)

            for author in version.authors:
                if author.house == chamber_name:
                    fsbill.add_sponsor(SPONSOR_TYPES[author.contribution],
                                       author.name,
                                       official_type=author.contribution)

            seen_actions = set()
            for action in bill.actions:
                if not action.action:
                    # NULL action text seems to be an error on CA's part,
                    # unless it has some meaning I'm missing
                    continue
                actor = action.actor or chamber
                actor = actor.strip()
                match = re.match(r'(Assembly|Senate)($| \(Floor)', actor)
                if match:
                    actor = {'Assembly': 'lower',
                             'Senate': 'upper'}[match.group(1)]
                elif actor.startswith('Governor'):
                    actor = 'other'
                else:
                    def replacer(matchobj):
                        if matchobj:
                            return {'Assembly': 'lower',
                                    'Senate': 'upper'}[matchobj.group()]
                        else:
                            return matchobj.group()

                    actor = re.sub(r'^(Assembly|Senate)', replacer, actor)

                type_ = []

                act_str = action.action
                act_str = re.sub(r'\s+', ' ', act_str)

                attrs = self.categorizer.categorize(act_str)

                # Add in the committee strings of the related committees, if any.
                kwargs = attrs
                matched_abbrs = committee_abbr_regex.findall(action.action)

                if re.search(r'Com[s]?. on', action.action) and not matched_abbrs:
                    msg = 'Failed to extract committee abbr from %r.'
                    self.logger.warning(msg % action.action)

                if matched_abbrs:
                    committees = []
                    for abbr in matched_abbrs:
                        try:
                            name = self.committee_abbr_to_name(chamber, abbr)
                            committees.append(name)
                        except KeyError:
                            msg = ('Mapping contains no committee name for '
                                   'abbreviation %r. Action text was %r.')
                            args = (abbr, action.action)
                            raise KeyError(msg % args)

                    committees = filter(None, committees)
                    kwargs['committees'] = committees

                    code = re.search(r'C[SXZ]\d+', actor)
                    if code is not None:
                        code = code.group()
                        kwargs['actor_info'] = {'committee_code': code}

                    assert len(committees) == len(matched_abbrs)
                    for committee, abbr in zip(committees, matched_abbrs):
                        act_str = act_str.replace('Coms. on ', '')
                        act_str = act_str.replace('Com. on ' + abbr, committee)
                        act_str = act_str.replace(abbr, committee)
                        if not act_str.endswith('.'):
                            act_str = act_str + '.'

                # Determine which chamber the action originated from.
                changed = False
                for committee_chamber in ['upper', 'lower', 'joint']:
                    if actor.startswith(committee_chamber):
                        actor = committee_chamber
                        changed = True
                        break
                if not changed:
                    actor = 'other'

                if actor != action.actor:
                    actor_info = kwargs.get('actor_info', {})
                    actor_info['details'] = action.actor
                    kwargs['actor_info'] = actor_info

                # Add strings for related legislators, if any.
                rgx = '(?:senator|assembly[mwp][^ .,:;]+)\s+[^ .,:;]+'
                legislators = re.findall(rgx, action.action, re.I)
                if legislators:
                    kwargs['legislators'] = legislators

                date = action.action_date
                date = self._tz.localize(date)
                date = date.date()
                if (actor, act_str, date) in seen_actions:
                    continue
                fsbill.add_action(actor, act_str, date, **kwargs)
                seen_actions.add((actor, act_str, date))

            for vote in bill.votes:
                if vote.vote_result == '(PASS)':
                    result = True
                else:
                    result = False

                full_loc = vote.location.description
                first_part = full_loc.split(' ')[0].lower()
                if first_part in ['asm', 'assembly']:
                    vote_chamber = 'lower'
                    vote_location = ' '.join(full_loc.split(' ')[1:])
                elif first_part.startswith('sen'):
                    vote_chamber = 'upper'
                    vote_location = ' '.join(full_loc.split(' ')[1:])
                else:
                    raise ScrapeError("Bad location: %s" % full_loc)

                if vote.motion:
                    motion = vote.motion.motion_text or ''
                else:
                    motion = ''

                if "Third Reading" in motion or "3rd Reading" in motion:
                    vtype = 'passage'
                elif "Do Pass" in motion:
                    vtype = 'passage'
                else:
                    vtype = 'other'

                motion = motion.strip()

                # Why did it take until 2.7 to get a flags argument on re.sub?
                motion = re.compile(r'(\w+)( Extraordinary)? Session$',
                                    re.IGNORECASE).sub('', motion)
                motion = re.compile(r'^(Senate|Assembly) ',
                                    re.IGNORECASE).sub('', motion)
                motion = re.sub(r'^(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.?  ',
                                '', motion)
                motion = re.sub(r' \(\w+\)$', '', motion)
                motion = re.sub(r'(SCR|SB|AB|AJR|ACR)\s?\d+ \w+\.?$',
                                '', motion)
                motion = re.sub(r'(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.? '
                                r'Urgency Clause$',
                                '(Urgency Clause)', motion)
                motion = re.sub(r'\s+', ' ', motion)

                if not motion:
                    self.warning("Got blank motion on vote for %s" % bill_id)
                    continue

                fsvote = Vote(vote_chamber,
                              self._tz.localize(vote.vote_date_time),
                              motion,
                              result,
                              int(vote.ayes),
                              int(vote.noes),
                              int(vote.abstain),
                              threshold=vote.threshold,
                              type_=vtype)

                if vote_location != 'Floor':
                    fsvote['committee'] = vote_location

                for record in vote.votes:
                    if record.vote_code == 'AYE':
                        fsvote.yes(record.legislator_name)
                    elif record.vote_code.startswith('NO'):
                        fsvote.no(record.legislator_name)
                    else:
                        fsvote.other(record.legislator_name)

                for s in ('yes', 'no', 'other'):
                    # Kill dupe votes.
                    key = s + '_votes'
                    fsvote[key] = list(set(fsvote[key]))

                # In a small percentage of bills, the integer vote counts
                # are inaccurate, so let's ignore them.
                for k in ('yes', 'no', 'other'):
                    fsvote[k + '_count'] = len(fsvote[k + '_votes'])

                fsbill.add_vote(fsvote)

            self.save_bill(fsbill)
            self.session.expire_all()

Example 196

Project: openstates
Source File: bills.py
View license
    def parse_bill(self, chamber, session, bill_id, url):
        page = self.lxmlize(url)

        short_bill_id = re.sub(r'(H|S)([JC])R', r'\1\2', bill_id)
        version_link_node = self.get_node(
            page,
            '//a[contains(@href, "{bill_id}/bill.doc") or contains(@href,'
            '"{bill_id}/bill.pdf")]'.format(bill_id=short_bill_id))

        if version_link_node is None:
            # Bill withdrawn
            self.logger.warning('Bill withdrawn.')
            return
        else:
            source_url = version_link_node.attrib['href']

            if source_url.endswith('.doc'):
                mimetype='application/msword'
            elif source_url.endswith('.pdf'):
                mimetype='application/pdf'

        if self._is_post_2016:
            title_texts = self.get_nodes(
                page,
                '//div[@class="StandardText leftDivMargin"]/text()')
            title_texts = filter(None, [text.strip() for text in title_texts])\
                [1:]
            title_texts = [s for s in title_texts if s != ',']
            title = ' '.join(title_texts)

            actions = self.get_nodes(
                page,
                '//div[@class="StandardText leftDivMargin"]/'
                'div[@class="StandardText"][last()]//text()[normalize-space()]')
        else:
            pars = version_link_node.xpath("following-sibling::p")

            if len(pars) == 2:
                title = pars[0].xpath("string()")
                action_p = pars[1]
            else:
                title = pars[0].getprevious().tail
                if not title:
                    self.warning('walking backwards to get bill title, error prone!')
                    title = pars[0].getprevious().getprevious()
                    while not title.tail:
                        title = title.getprevious()
                    title = title.tail
                    self.warning('got title the dangerous way: %s' % title)
                action_p = pars[0]

            title = re.sub(ur'[\s\xa0]+', ' ', title).strip()
            actions = action_p.xpath("string()").split("\n")

        if 'CR' in bill_id:
            bill_type = 'concurrent resolution'
        elif 'JR' in bill_id:
            bill_type = 'joint resolution'
        elif 'R' in bill_id:
            bill_type = 'resolution'
        else:
            bill_type = 'bill'

        bill = Bill(session, chamber, bill_id, title, type=bill_type)
        bill['subjects'] = self._subjects[bill_id]
        bill.add_source(url)

        bill.add_version("Most Recent Version",
            source_url,
            mimetype=mimetype)

        for link in page.xpath("//a[contains(@href, 'legislator/')]"):
            bill.add_sponsor('primary', link.text.strip())

        for line in actions:
            line_actions = line.strip().split(';')

            for index, action in enumerate(line_actions):
                action = action.strip()
                if not action:
                    continue

                action_date_text = line.split('-')[0].strip()
                if self._is_post_2016:
                    action_date_string = action_date_text.replace(',', '')
                else:
                    action_date_string = '{} {}'.format(action_date_text,
                        session[0:4])

                # This patch is super hacky, but allows us to better
                # capture actions that screw up the formatting such as
                # veto document links.
                try:
                    action_date = datetime.datetime.strptime(
                        action_date_string, '%b %d %Y')
                    cached_action_date = action_date
                    used_cached_action_date = False
                except:
                    action_date = cached_action_date
                    used_cached_action_date = True

                # Separate out the date if first action on the line.
                if index == 0 and not used_cached_action_date:
                    action = '-'.join(action.split('-')[1:]).strip()
                    if not action:
                        continue

                if action.endswith('House') or action.endswith('(H)'):
                    actor = 'lower'
                elif action.endswith('Senate') or action.endswith('(S)'):
                    actor = 'upper'
                else:
                    actor = chamber

                atype = []
                if 'introduced in' in action:
                    atype.append('bill:introduced')
                    if 'to ' in action:
                        atype.append('committee:referred')
                elif 'signed by Governor' in action:
                    atype.append('governor:signed')
                elif 'vetoed' in action:
                    atype.append('governor:vetoed')

                    # Get the accompanying veto message document. There
                    # should only be one.
                    veto_document_link = self.get_node(page,
                        '//div[@class="StandardText leftDivMargin"]/'
                        'div[@class="StandardText"][last()]/a[contains(@href,'
                        '"veto.pdf")]')

                    if veto_document_link is not None:
                        bill.add_document("Veto Message",
                            veto_document_link.attrib['href'])
                elif re.match(r'^to [A-Z]', action):
                    atype.append('committee:referred')
                elif action == 'adopted by voice vote':
                    atype.append('bill:passed')

                if '1st reading' in action:
                    atype.append('bill:reading:1')
                if '3rd reading' in action:
                    atype.append('bill:reading:3')
                    if 'passed' in action:
                        atype.append('bill:passed')
                if '2nd reading' in action:
                    atype.append('bill:reading:2')

                if 'R' in bill_id and 'adopted by voice vote' in action:
                    atype.append('bill:passed')

                amendment_re = (r'floor amendments?( \([a-z\d\-]+\))*'
                                r'( and \([a-z\d\-]+\))? filed')
                if re.search(amendment_re, action):
                    atype.append('amendment:introduced')

                if not atype:
                    atype = ['other']

                # Capitalize the first letter of the action for nicer
                # display. capitalize() won't work for this because it
                # lowercases all other letters.
                action = (action[0].upper() + action[1:])

                if action:
                    bill.add_action(actor, action, action_date, type=atype)

        try:
            votes_link = page.xpath(
                "//a[contains(@href, 'vote_history.pdf')]")[0]
            bill.add_document("Vote History",
                votes_link.attrib['href'])
        except IndexError:
            # No votes
            self.logger.warning(u'No votes found for {}'.format(title))
            pass

        # Ugly Hack Alert!
        # find actions before introduction date and subtract 1 from the year
        # if the date is after introduction
        intro_date = None
        for i, action in enumerate(bill['actions']):
            if 'bill:introduced' in action['type']:
                intro_date = action['date']
                break
            for action in bill['actions'][:i]:
                if action['date'] > intro_date:
                    action['date'] = action['date'].replace(year=action['date'].year-1)
                    self.debug('corrected year for %s', action['action'])

        self.save_bill(bill)

Example 197

Project: openstates
Source File: bills.py
View license
    def scrape_bills(self, session, year_abr):
        #Main Bill information
        main_bill_csv = self.access_to_csv('MainBill')

        # keep a dictionary of bills (mapping bill_id to Bill obj)
        bill_dict = {}

        for rec in main_bill_csv:
            bill_type = rec["BillType"].strip()
            bill_number = int(rec["BillNumber"])
            bill_id = bill_type + str(bill_number)
            title = rec["Synopsis"]
            if bill_type[0] == 'A':
                chamber = "lower"
            else:
                chamber = "upper"

            # some bills have a blank title.. just skip it
            if not title:
                continue

            bill = Bill(str(session), chamber, bill_id, title,
                        type=self._bill_types[bill_type[1:]])
            if rec['IdenticalBillNumber'].strip():
                bill.add_companion(rec['IdenticalBillNumber'].split()[0])

            # TODO: last session info is in there too
            bill_dict[bill_id] = bill

        #Sponsors
        bill_sponsors_csv = self.access_to_csv('BillSpon')

        for rec in bill_sponsors_csv:
            bill_type = rec["BillType"].strip()
            bill_number = int(rec["BillNumber"])
            bill_id = bill_type + str(bill_number)
            if bill_id not in bill_dict:
                self.warning('unknown bill %s in sponsor database' % bill_id)
                continue
            bill = bill_dict[bill_id]
            name = rec["Sponsor"]
            sponsor_type = rec["Type"]
            if sponsor_type == 'P':
                sponsor_type = "primary"
            else:
                sponsor_type = "cosponsor"
            bill.add_sponsor(sponsor_type, name)


        #Documents
        bill_document_csv = self.access_to_csv('BillWP')

        for rec in bill_document_csv:
            bill_type = rec["BillType"].strip()
            bill_number = int(rec["BillNumber"])
            bill_id = bill_type + str(bill_number)
            if bill_id not in bill_dict:
                self.warning('unknown bill %s in document database' % bill_id)
                continue
            bill = bill_dict[bill_id]
            document = rec["Document"]
            document = document.split('\\')
            document = document[-2] + "/" + document[-1]
            year = str(year_abr) + str((year_abr + 1))

            #doc_url = "ftp://www.njleg.state.nj.us/%s/%s" % (year, document)
            htm_url = 'http://www.njleg.state.nj.us/%s/Bills/%s' % (year_abr,
                document.replace('.DOC', '.HTM'))

            # name document based _doctype
            try:
                doc_name = self._doctypes[rec['DocType']]
            except KeyError:
                raise Exception('unknown doctype %s on %s' %
                                (rec['DocType'], bill_id))
            if rec['Comment']:
                doc_name += ' ' + rec['Comment']

            if rec['DocType'] in self._version_types:
                # Clean HTMX links.
                if htm_url.endswith('HTMX'):
                    htm_url = re.sub('X$', '', htm_url)

                if htm_url.endswith('HTM'):
                    mimetype = 'text/html'
                elif htm_url.endswith('wpd'):
                    mimetype = 'application/vnd.wordperfect'
                bill.add_version(doc_name, htm_url, mimetype=mimetype)
            else:
                bill.add_document(doc_name, htm_url)

        # Votes
        next_year = int(year_abr)+1
        vote_info_list = ['A%s' % year_abr,
                          'A%s' % next_year,
                          'S%s' % year_abr,
                          'S%s' % next_year,
                          'CA%s-%s' % (year_abr, next_year),
                          'CS%s-%s' % (year_abr, next_year),
                         ]

        for filename in vote_info_list:
            s_vote_url = 'ftp://www.njleg.state.nj.us/votes/%s.zip' % filename
            try:
                s_vote_zip, resp = self.urlretrieve(s_vote_url)
            except scrapelib.FTPError:
                self.warning('could not find %s' % s_vote_url)
                continue
            zipedfile = zipfile.ZipFile(s_vote_zip)
            for vfile in ["%s.txt" % (filename), "%sEnd.txt" % (filename)]:
                try:
                    vote_file = zipedfile.open(vfile, 'U')
                except KeyError:
                    #
                    # Right, so, 2011 we have an "End" file with more
                    # vote data than was in the original dump.
                    #
                    self.warning("No such file: %s" % (vfile))
                    continue

                vdict_file = csv.DictReader(vote_file)

                votes = {}
                if filename.startswith('A') or filename.startswith('CA'):
                    chamber = "lower"
                else:
                    chamber = "upper"

                if filename.startswith('C'):
                    vote_file_type = 'committee'
                else:
                    vote_file_type = 'chamber'

                for rec in vdict_file:

                    if vote_file_type == 'chamber':
                        bill_id = rec["Bill"].strip()
                        leg = rec["Full_Name"]

                        date = rec["Session_Date"]
                        action = rec["Action"]
                        leg_vote = rec["Legislator_Vote"]
                    else:
                        bill_id = '%s%s' % (rec['Bill_Type'], rec['Bill_Number'])
                        leg = rec['Name']
                        # drop time portion
                        date = rec['Agenda_Date'].split()[0]
                        # make motion readable
                        action = self._com_vote_motions[rec['BillAction']]
                        # first char (Y/N) use [0:1] to ignore ''
                        leg_vote = rec['LegislatorVote'][0:1]

                    date = datetime.strptime(date, "%m/%d/%Y")
                    vote_id = '_'.join((bill_id, chamber, action))
                    vote_id = vote_id.replace(" ", "_")

                    if vote_id not in votes:
                        votes[vote_id] = Vote(chamber, date, action, None, None,
                                              None, None, bill_id=bill_id)
                    if vote_file_type == 'committee':
                        votes[vote_id]['committee'] = self._committees[
                            rec['Committee_House']]

                    if leg_vote == "Y":
                        votes[vote_id].yes(leg)
                    elif leg_vote == "N":
                        votes[vote_id].no(leg)
                    else:
                        votes[vote_id].other(leg)

            # remove temp file
            os.remove(s_vote_zip)

            #Counts yes/no/other votes and saves overall vote
            for vote in votes.itervalues():
                vote_yes_count = len(vote["yes_votes"])
                vote_no_count = len(vote["no_votes"])
                vote_other_count = len(vote["other_votes"])
                vote["yes_count"] = vote_yes_count
                vote["no_count"] = vote_no_count
                vote["other_count"] = vote_other_count

                # Veto override.
                if vote['motion'] == 'OVERRIDE':
                    # Per the NJ leg's glossary, a veto override requires
                    # 2/3ds of each chamber. 27 in the senate, 54 in the house.
                    # http://www.njleg.state.nj.us/legislativepub/glossary.asp
                    vote['passed'] = False
                    if vote['chamber'] == 'lower':
                        if vote_yes_count >= 54:
                            vote['passed'] = True
                    elif vote['chamber'] == 'upper':
                        if vote_yes_count >= 27:
                            vote['passed'] = True

                # Regular vote.
                elif vote_yes_count > vote_no_count:
                    vote["passed"] = True
                else:
                    vote["passed"] = False
                vote_bill_id = vote["bill_id"]
                bill = bill_dict[vote_bill_id]
                bill.add_vote(vote)

        #Actions
        bill_action_csv = self.access_to_csv('BillHist')
        actor_map = {'A': 'lower', 'G': 'executive', 'S': 'upper'}

        for rec in bill_action_csv:
            bill_type = rec["BillType"].strip()
            bill_number = int(rec["BillNumber"])
            bill_id = bill_type + str(bill_number)
            if bill_id not in bill_dict:
                self.warning('unknown bill %s in action database' % bill_id)
                continue
            bill = bill_dict[bill_id]
            action = rec["Action"]
            date = rec["DateAction"]
            date = datetime.strptime(date, "%m/%d/%y %H:%M:%S")
            actor = actor_map[rec["House"]]
            comment = rec["Comment"]
            action, atype = self.categorize_action(action, bill_id)
            if comment:
                action += (' ' + comment)
            bill.add_action(actor, action, date, type=atype)

        # Subjects
        subject_csv = self.access_to_csv('BillSubj')
        for rec in subject_csv:
            bill_id = rec['BillType'].strip() + str(int(rec['BillNumber']))
            if bill_id not in bill_dict:
                self.warning('unknown bill %s in subject database' % bill_id)
                continue
            bill = bill_dict.get(bill_id)
            if bill:
                bill.setdefault('subjects', []).append(rec['SubjectKey'])
            else:
                self.warning('invalid bill id in BillSubj: %s' % bill_id)

        phony_bill_count = 0
        # save all bills at the end
        for bill in bill_dict.itervalues():
            # add sources
            if not bill['actions'] and not bill['versions']:
                self.warning('probable phony bill detected %s',
                             bill['bill_id'])
                phony_bill_count += 1
            else:
                bill.add_source('http://www.njleg.state.nj.us/downloads.asp')
                self.save_bill(bill)

        if phony_bill_count:
            self.warning('%s total phony bills detected', phony_bill_count)

Example 198

Project: tcollector
Source File: sysload.py
View license
def main():
    """top main loop"""

    collection_interval=DEFAULT_COLLECTION_INTERVAL
    collect_every_cpu=True
    if(sysload_conf):
        config = sysload_conf.get_config()
        collection_interval=config['collection_interval']
        collect_every_cpu=config['collect_every_cpu']

    global signal_received

    signal.signal(signal.SIGTERM, handlesignal)
    signal.signal(signal.SIGINT, handlesignal)

    try:
        if platform.system() == "FreeBSD":
            if collect_every_cpu:
                p_top = subprocess.Popen(
                    ["top", "-S", "-P", "-n", "-s"+str(collection_interval), "-dinfinity", "0"],
                    stdout=subprocess.PIPE,
                )
            else:
                p_top = subprocess.Popen(
                    ["top", "-S", "-n", "-s"+str(collection_interval), "-dinfinity", "0"],
                    stdout=subprocess.PIPE,
                )            
        else:
            if collect_every_cpu:
                p_top = subprocess.Popen(
                    ["mpstat", "-P", "ALL", str(collection_interval)],
                    stdout=subprocess.PIPE,
                )
            else:
                p_top = subprocess.Popen(
                    ["mpstat", str(collection_interval)],
                    stdout=subprocess.PIPE,
                )
    except OSError, e:
        if e.errno == errno.ENOENT:
            # it makes no sense to run this collector here
            sys.exit(13) # we signal tcollector to not run us
        raise

    timestamp = 0

    while signal_received is None:
        try:
            line = p_top.stdout.readline()
        except (IOError, OSError), e:
            if e.errno in (errno.EINTR, errno.EAGAIN):
                break
            raise

        if not line:
            # end of the program, die
            break

        # CPU: --> CPU all:  : FreeBSD, to match the all CPU
        # %( [uni][a-z]+,?)? : FreeBSD, so that top output matches mpstat output
        # AM                 : Linux, mpstat output depending on locale
        # PM                 : Linux, mpstat output depending on locale
        # .* load            : FreeBSD, to correctly match load averages
        # ,                  : FreeBSD, to correctly match processes: Mem: ARC: and Swap:
        fields = re.sub("CPU:", "CPU all:", re.sub(r"%( [uni][a-z]+,?)?| AM | PM |.* load |,", "", line)).split()
        if len(fields) <= 0:
            continue

        if (((fields[0] == "CPU") or (re.match("[0-9][0-9]:[0-9][0-9]:[0-9][0-9]",fields[0]))) and ((collect_every_cpu and re.match("[0-9]+:?",fields[1])) or ((not collect_every_cpu) and re.match("all:?",fields[1])))):
            if((fields[1] == "all") or (fields[1] == "0")):
                timestamp = int(time.time())
            cpuid=fields[1].replace(":","")
            cpuuser=fields[2]
            cpunice=fields[3]
            cpusystem=fields[4]
            cpuinterrupt=fields[6]
            cpuidle=fields[-1]
            print ("cpu.usr %s %s cpu=%s" % (timestamp, float(cpuuser), cpuid))
            print ("cpu.nice %s %s cpu=%s" % (timestamp, float(cpunice), cpuid))
            print ("cpu.sys %s %s cpu=%s" % (timestamp, float(cpusystem), cpuid))
            print ("cpu.irq %s %s cpu=%s" % (timestamp, float(cpuinterrupt), cpuid))
            print ("cpu.idle %s %s cpu=%s" % (timestamp, float(cpuidle), cpuid))
        
        elif(fields[0] == "averages:"):
            timestamp = int(time.time())
            print ("load.1m %s %s" % (timestamp, fields[1]))
            print ("load.5m %s %s" % (timestamp, fields[2]))
            print ("load.15m %s %s" % (timestamp, fields[3]))

        elif (re.match("[0-9]+ processes:",line)):
            starting=0
            running=0
            sleeping=0
            stopped=0
            zombie=0
            waiting=0
            lock=0
            for i in range(len(fields)):
                if(fields[i] == "starting"):
                    starting=fields[i-1]
                if(fields[i] == "running"):
                    running=fields[i-1]
                if(fields[i] == "sleeping"):
                    sleeping=fields[i-1]
                if(fields[i] == "stopped"):
                    stopped=fields[i-1]
                if(fields[i] == "zombie"):
                    zombie=fields[i-1]
                if(fields[i] == "waiting"):
                    waiting=fields[i-1]
                if(fields[i] == "lock"):
                    lock=fields[i-1]
            print ("ps.all %s %s" % (timestamp, fields[0]))
            print ("ps.start %s %s" % (timestamp, starting))
            print ("ps.run %s %s" % (timestamp, running))
            print ("ps.sleep %s %s" % (timestamp, sleeping))
            print ("ps.stop %s %s" % (timestamp, stopped))
            print ("ps.zomb %s %s" % (timestamp, zombie))
            print ("ps.wait %s %s" % (timestamp, waiting))
            print ("ps.lock %s %s" % (timestamp, lock))

        elif(fields[0] == "Mem:"):
            active=0
            inact=0
            wired=0
            cache=0
            buf=0
            free=0
            for i in range(len(fields)):
                if(fields[i] == "Active"):
                    active=convert_to_bytes(fields[i-1])
                if(fields[i] == "Inact"):
                    inact=convert_to_bytes(fields[i-1])
                if(fields[i] == "Wired"):
                    wired=convert_to_bytes(fields[i-1])
                if(fields[i] == "Cache"):
                    cache=convert_to_bytes(fields[i-1])
                if(fields[i] == "Buf"):
                    buf=convert_to_bytes(fields[i-1])
                if(fields[i] == "Free"):
                    free=convert_to_bytes(fields[i-1])
            print ("mem.active %s %s" % (timestamp, active))
            print ("mem.inact %s %s" % (timestamp, inact))
            print ("mem.wired %s %s" % (timestamp, wired))
            print ("mem.cache %s %s" % (timestamp, cache))
            print ("mem.buf %s %s" % (timestamp, buf))
            print ("mem.free %s %s" % (timestamp, free))

        elif(fields[0] == "ARC:"):
            total=0
            mru=0
            mfu=0
            anon=0
            header=0
            other=0
            for i in range(len(fields)):
                if(fields[i] == "Total"):
                    total=convert_to_bytes(fields[i-1])
                if(fields[i] == "MRU"):
                    mru=convert_to_bytes(fields[i-1])
                if(fields[i] == "MFU"):
                    mfu=convert_to_bytes(fields[i-1])
                if(fields[i] == "Anon"):
                    anon=convert_to_bytes(fields[i-1])
                if(fields[i] == "Header"):
                    header=convert_to_bytes(fields[i-1])
                if(fields[i] == "Other"):
                    other=convert_to_bytes(fields[i-1])
            print ("arc.total %s %s" % (timestamp, total))
            print ("arc.mru %s %s" % (timestamp, mru))
            print ("arc.mfu %s %s" % (timestamp, mfu))
            print ("arc.anon %s %s" % (timestamp, anon))
            print ("arc.header %s %s" % (timestamp, header))
            print ("arc.other %s %s" % (timestamp, other))

        elif(fields[0] == "Swap:"):
            total=0
            free=0
            inuse=0
            inps=0
            outps=0
            for i in range(len(fields)):
                if(fields[i] == "Total"):
                    total=convert_to_bytes(fields[i-1])
                if(fields[i] == "Free"):
                    free=convert_to_bytes(fields[i-1])
                if(fields[i] == "Inuse"):
                    inuse=convert_to_bytes(fields[i-1])
                if(fields[i] == "In"):
                    inps=convert_to_bytes(fields[i-1])/collection_interval
                if(fields[i] == "Out"):
                    outps=convert_to_bytes(fields[i-1])/collection_interval
            print ("swap.total %s %s" % (timestamp, total))
            print ("swap.free %s %s" % (timestamp, free))
            print ("swap.inuse %s %s" % (timestamp, inuse))
            print ("swap.inps %s %s" % (timestamp, inps))
            print ("swap.outps %s %s" % (timestamp, outps))

        sys.stdout.flush()

    if signal_received is None:
        signal_received = signal.SIGTERM
    try:
        os.kill(p_top.pid, signal_received)
    except Exception:
        pass
    p_top.wait()

Example 199

Project: nycbuildings
Source File: convert.py
View license
def convert(buildingsFile, osmOut):
    with open(buildingsFile) as f:
        buildings = json.load(f)
    buildingShapes = []
    buildingIdx = index.Index()
    for building in buildings:
        shape = asShape(building['geometry'])
        buildingShapes.append(shape)
        buildingIdx.add(len(buildingShapes) - 1, shape.bounds)

    # Generates a new osm id.
    osmIds = dict(node = -1, way = -1, rel = -1)
    def newOsmId(type):
        osmIds[type] = osmIds[type] - 1
        return osmIds[type]

    ## Formats multi part house numbers
    def formatHousenumber(p):
        def suffix(part1, part2, hyphen_type=None):
            part1 = stripZeroes(part1)
            if not part2:
                return str(part1)
            part2 = stripZeroes(part2)
            if hyphen_type == 'U': # unit numbers
                return part1 + '-' + part2
            if len(part2) == 1 and part2.isalpha(): # single letter extensions
                return part1 + part2
            return part1 + ' ' + part2 # All others
        def stripZeroes(addr): # strip leading zeroes from numbers
            if addr.isdigit():
                addr = str(int(addr))
            if '-' in addr:
                try:
                    addr2 = addr.split('-')
                    if len(addr2) == 2:
                        addr = str(int(addr2[0])) + '-' + str(int(addr2[1])).zfill(2)
                except:
                    pass
            return addr
        number = suffix(p['HOUSE_NUMB'], p['HOUSE_NU_1'], p['HYPHEN_TYP'])
        return number

    # Converts an address
    def convertAddress(address):
        result = dict()
        if all (k in address for k in ('HOUSE_NUMB', 'STREET_NAM')):
            if address['HOUSE_NUMB']:
                result['addr:housenumber'] = formatHousenumber(address)
            if address['STREET_NAM']:
                streetname = address['STREET_NAM'].title()
                streetname = streetname.replace('F D R ', 'FDR ')
                # Expand Service Road
                # See https://github.com/osmlab/nycbuildings/issues/30
                streetname = re.sub(r"(.*)\bSr\b(.*)", r"\1Service Road\2", streetname)
                # Expand cardinal directions on Service Roads
                streetname = re.sub(r"(.*\bService Road\s)\bN\b(.*)", r"\1North\2", streetname)
                streetname = re.sub(r"(.*\bService Road\s)\bE\b(.*)", r"\1East\2", streetname)
                streetname = re.sub(r"(.*\bService Road\s)\bS\b(.*)", r"\1South\2", streetname)
                streetname = re.sub(r"(.*\bService Road\s)\bW\b(.*)", r"\1West\2", streetname)
                # Expand Expressway on Service Roads
                streetname = re.sub(r"(.*)Expwy\s\bN\b(.*)", r"\1Expressway North\2", streetname)
                streetname = re.sub(r"(.*)Expwy\s\bE\b(.*)", r"\1Expressway East\2", streetname)
                streetname = re.sub(r"(.*)Expwy\s\bS\b(.*)", r"\1Expressway South\2", streetname)
                streetname = re.sub(r"(.*)Expwy\s\bW\b(.*)", r"\1Expressway West\2", streetname)
                streetname = re.sub(r"(.*)Expwy(.*)", r"\1Expressway\2", streetname)
                # Add ordinal suffixes to numerals
                streetname = re.sub(r"(.*)(\d*11)\s+(.*)", r"\1\2th \3", streetname)
                streetname = re.sub(r"(.*)(\d*12)\s+(.*)", r"\1\2th \3", streetname)
                streetname = re.sub(r"(.*)(\d*13)\s+(.*)", r"\1\2th \3", streetname)
                streetname = re.sub(r"(.*)(\d*1)\s+(.*)", r"\1\2st \3", streetname)
                streetname = re.sub(r"(.*)(\d*2)\s+(.*)", r"\1\2nd \3", streetname)
                streetname = re.sub(r"(.*)(\d*3)\s+(.*)", r"\1\2rd \3", streetname)
                streetname = re.sub(r"(.*)(\d+)\s+(.*)", r"\1\2th \3", streetname)
                # Expand 'Ft' -> 'Fort'
                if streetname[0:3] == 'Ft ': streetname = 'Fort ' + streetname[3:]
                # Expand 'St ' -> 'Saint'
                if streetname[0:3] == 'St ': streetname = 'Saint ' + streetname[3:]
                # Expand 'Rev ' -> 'Reverend '
                if streetname[0:4] == 'Rev ': streetname = 'Reverend ' + streetname[3:]
                # Expand middlename ' St John' fix
                streetname = streetname.replace('St John', 'Saint John')
                # Middle name expansions
                streetname = streetname.replace(' St ', ' Street ')
                streetname = streetname.replace(' Rd ', ' Road ')
                streetname = streetname.replace(' Blvd ', ' Boulevard ')
                result['addr:street'] = streetname
            if address['ZIPCODE']:
                result['addr:postcode'] = str(int(address['ZIPCODE']))
        return result

    # Appends new node or returns existing if exists.
    nodes = {}
    def appendNewNode(coords, osmXml):
        rlon = int(float(coords[0]*10**7))
        rlat = int(float(coords[1]*10**7))
        if (rlon, rlat) in nodes:
            return nodes[(rlon, rlat)]
        node = etree.Element('node', visible = 'true', id = str(newOsmId('node')))
        node.set('lon', str(Decimal(coords[0])*Decimal(1)))
        node.set('lat', str(Decimal(coords[1])*Decimal(1)))
        nodes[(rlon, rlat)] = node
        osmXml.append(node)
        return node

    def appendNewWay(coords, intersects, osmXml):
        way = etree.Element('way', visible='true', id=str(newOsmId('way')))
        firstNid = 0
        for i, coord in enumerate(coords):
            if i == 0: continue # the first and last coordinate are the same
            node = appendNewNode(coord, osmXml)
            if i == 1: firstNid = node.get('id')
            way.append(etree.Element('nd', ref=node.get('id')))
            
            # Check each way segment for intersecting nodes
            int_nodes = {}
            try:
                line = LineString([coord, coords[i+1]])
            except IndexError:
                line = LineString([coord, coords[1]])
            for idx, c in enumerate(intersects):
                if line.buffer(0.000001).contains(Point(c[0], c[1])) and c not in coords:
                    t_node = appendNewNode(c, osmXml)
                    for n in way.iter('nd'):
                        if n.get('ref') == t_node.get('id'):
                            break
                    else:
                        int_nodes[t_node.get('id')] = Point(c).distance(Point(coord))
            for n in sorted(int_nodes, key=lambda key: int_nodes[key]): # add intersecting nodes in order
                way.append(etree.Element('nd', ref=n))
            
        way.append(etree.Element('nd', ref=firstNid)) # close way
        osmXml.append(way)
        return way

    # Appends an address to a given node or way.
    def appendAddress(address, element):
        for k, v in convertAddress(address['properties']).iteritems():
            element.append(etree.Element('tag', k=k, v=v))

    # Appends a building to a given OSM xml document.
    def appendBuilding(building, shape, address, osmXml):
        # Check for intersecting buildings
        intersects = []
        for i in buildingIdx.intersection(shape.bounds):
            try:
                for c in buildingShapes[i].exterior.coords:
                    if Point(c[0], c[1]).buffer(0.000001).intersects(shape):
                        intersects.append(c)
            except AttributeError:
                for c in buildingShapes[i][0].exterior.coords:
                    if Point(c[0], c[1]).buffer(0.000001).intersects(shape):
                        intersects.append(c)

        # Export building, create multipolygon if there are interior shapes.
        interiors = []
        try:
            way = appendNewWay(list(shape.exterior.coords), intersects, osmXml)
            for interior in shape.interiors:
                interiors.append(appendNewWay(list(interior.coords), [], osmXml))
        except AttributeError:
            way = appendNewWay(list(shape[0].exterior.coords), intersects, osmXml)
            for interior in shape[0].interiors:
                interiors.append(appendNewWay(list(interior.coords), [], osmXml))
        if len(interiors) > 0:
            relation = etree.Element('relation', visible='true', id=str(newOsmId('way')))
            relation.append(etree.Element('member', type='way', role='outer', ref=way.get('id')))
            for interior in interiors:
                relation.append(etree.Element('member', type='way', role='inner', ref=interior.get('id')))
            relation.append(etree.Element('tag', k='type', v='multipolygon'))
            osmXml.append(relation)
            way = relation
        way.append(etree.Element('tag', k='building', v='yes'))
        if 'HEIGHT_ROO' in building['properties']:
            height = round(((building['properties']['HEIGHT_ROO'] * 12) * 0.0254), 1)
            if height > 0:
                way.append(etree.Element('tag', k='height', v=str(height)))
        if 'BIN' in building['properties']:
            way.append(etree.Element('tag', k='nycdoitt:bin', v=str(building['properties']['BIN'])))
        if address: appendAddress(address, way)

    # Export buildings & addresses. Only export address with building if there is exactly
    # one address per building. Export remaining addresses as individual nodes.
    allAddresses = []
    osmXml = etree.Element('osm', version='0.6', generator='[email protected]')
    for i in range(0, len(buildings)):

        # Filter out special addresses categories A and B
        buildingAddresses = []
        for address in buildings[i]['properties']['addresses']:
            if address['properties']['SPECIAL_CO'] not in ['A', 'B']:
                buildingAddresses.append(address)
        address = None
        if len(buildingAddresses) == 1:
            address = buildingAddresses[0]
        else:
            allAddresses.extend(buildingAddresses)

        if int(buildings[i]['properties']['HEIGHT_ROO']) == 0:
            if shape.area > 1e-09:
                appendBuilding(buildings[i], buildingShapes[i], address, osmXml)
        else:
            appendBuilding(buildings[i], buildingShapes[i], address, osmXml)

    # Export any addresses that aren't the only address for a building.
    if (len(allAddresses) > 0):
        for address in allAddresses:
            node = appendNewNode(address['geometry']['coordinates'], osmXml)
            appendAddress(address, node)

    with open(osmOut, 'w') as outFile:
        outFile.writelines(tostring(osmXml, pretty_print=True, xml_declaration=True, encoding='UTF-8'))
        print 'Exported ' + osmOut

Example 200

View license
def ss_get_mysql_stats(options):
    # Process connection options and connect to MySQL.
    global cache_dir, poll_time, chk_options
        
    # Connect to MySQL.
    host = options.host
    user = options.user
    passwd = options.password
    port = options.port
    heartbeat = options.heartbeat
    
    db = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, cursorclass=MySQLdb.cursors.DictCursor)
    cursor = db.cursor()
    
    sanitized_host = host.replace(':', '').replace('/', '_')
    sanitized_host = sanitized_host + '_' + str(port)
    cache_file = os.path.join(cache_dir, '%s-mysql_graphite_stats.txt' % (sanitized_host))
    log_debug('Cache file is %s' % (cache_file))
    
    # First, check the cache.
    fp = None
    if not options.nocache:
        with open(cache_file, 'a+') as fp:
            try:
                fcntl.flock(fp, fcntl.LOCK_SH) # LOCK_SH
                try:
                    lines = open(cache_file).readlines()
                except Exception:
                    lines = []
                if os.path.getsize(cache_file) > 0 and \
                        os.path.getctime(cache_file) + (poll_time/2) > int(time.time()) and \
                        len(lines) > 0:
                    # The cache file is good to use
                    log_debug('Using the cache file')
                    return lines[0]
                else:
                    log_debug('The cache file seems too small or stale')
                    try:
                        # Escalate the lock to exclusive, so we can write to it.
                        fcntl.flock(fp, fcntl.LOCK_EX) # LOCK_EX
                        try:
                            lines = open(cache_file).readlines()
                        except Exception:
                            lines = []
                        if os.path.getsize(cache_file) > 0 and \
                                os.path.getctime(cache_file) + (poll_time/2) > int(time.time()) and \
                                len(lines) > 0:
                            log_debug("Using the cache file")
                            return lines[0]
                        f.truncate(0)
                    except Exception:
                        pass
            except IOError:
                log_debug("Couldn't lock the cache file, ignoring it.")
                fp = None
    else:
        log_debug("Couldn't open cache file")
        fp = None
        
    # Set up variables
    status = { # Holds the result of SHOW STATUS, SHOW INNODB STATUS, etc
        # Define some indexes so they don't cause errors with += operations
        'relay_log_space'           : None,
        'binary_log_space'          : None,
        'current_transactions'      : 0,
        'locked_transactions'       : 0,
        'active_transactions'       : 0,
        'innodb_locked_tables'      : 0,
        'innodb_tables_in_use'      : 0,
        'innodb_lock_structs'       : 0,
        'innodb_lock_wait_secs'     : 0,
        'innodb_sem_waits'          : 0,
        'innodb_set_wait_time_ms'   : 0,
        # Values for the 'state' column from SHOW PROCESSLIST (converted to
        # lowercase, with spaces replaced by underscores)
        'State_closing_tables'      : None,
        'State_copying_to_tmp_table': None,
        'State_end'                 : None,
        'State_freeing_items'       : None,
        'State_init'                : None,
        'State_login'               : None,
        'State_preparing'           : None,
        'State_reading_from_net'    : None,
        'State_sending_data'        : None,
        'State_sorting_result'      : None,
        'State_statistics'          : None,
        'State_updating'            : None,
        'State_writing_to_net'      : None,
        'State_none'                : None,
        'State_other'               : None, # Everything not listed above
    }
    
    # Get SHOW STATUS
    cursor.execute("SHOW /*!50002 GLOBAL */ STATUS")
    result = cursor.fetchall()
    for row in result:
        row = dict_change_key_case(row, case='lower')
        status[row.get('variable_name')] = row.get('value')

    # Get SHOW VARIABLES
    cursor.execute('SHOW VARIABLES')
    result = cursor.fetchall()
    for row in result:
        row = dict_change_key_case(row, case='lower')
        status[row.get('variable_name')] = row.get('value')
     
    # Get SHOW SLAVE STATUS 
    if chk_options.get('slave') and not options.no_replication_client:
        cursor.execute('SHOW SLAVE STATUS')
        result = cursor.fetchall()
        slave_status_row_gotten = 0
        for row in result:
            slave_status_row_gotten += 1
            # Must lowercase keys because different MySQL versions have different
            # lettercase.
            row = dict_change_key_case(row, case='lower')
            status['relay_log_space'] = row.get('relay_log_space')
            status['slave_lag'] = row.get('seconds_behind_master')
            
            if len(heartbeat) > 0:
                cursor.execute(
                    'SELECT MAX(GREATEST(0, UNIX_TIMESTAMP() - UNIX_TIMESTAMP(ts) - 1)) AS delay FROM %s' % (heartbeat)
                )
                result2 = cursor.fetchall()
                slave_delay_rows_gotten = 0
                for row2 in result2:
                    slave_delay_rows_gotten += 1
                    if type(row2) == dict and 'delay' in row2.keys():
                        status['slave_lag'] = row2.get('delay')
                    else:
                        log_debug("Couldn't get slave lag from %s" % (heartbeat))
                        
                if slave_delay_rows_gotten == 0:
                    log_debug('Got nothing from heartbeat query')
            
            # Scale slave_running and slave_stopped relative to the slave lag.
            status['slave_running'] = status.get('slave_lag') if row.get('slave_sql_running') == 'Yes' else 0
            status['slave_stopped'] = 0 if row.get('slave_sql_running') == 'Yes' else status.get('slave_lag')
        
        if slave_status_row_gotten == 0:
            log_debug('Got nothing from SHOW SLAVE STATUS')
    
    # Get SHOW MASTER STATUS
    if chk_options.get('master') and not options.no_super and status.get('log_bin') == 'ON':
        binlogs = []
        cursor.execute('SHOW MASTER LOGS')
        result = cursor.fetchall()
        for row in result:
            row = dict_change_key_case(row, case='lower')
            # Older versions of MySQL may not have the File_size column in the
            # results of the command.  Zero-size files indicate the user is
            # deleting binlogs manually from disk (bad user! bad!).
            if 'file_size' in row.keys() and row.get('file_size') > 0:
                binlogs.append(row.get('file_size'))
                
        if len(binlogs) > 0:
            status['binary_log_space'] = sum(binlogs)

    # Get SHOW PROCESSLIST and aggregate it by state
    if chk_options.get('procs'):
        cursor.execute('SHOW PROCESSLIST')
        result = cursor.fetchall()
        for row in result:
            state = row.get('state')
            if state is None:
                state = 'NULL'
            if state == '':
                state = 'none'
                
            # MySQL 5.5 replaces the 'Locked' state with a variety of "Waiting for
            # X lock" types of statuses.  Wrap these all back into "Locked" because
            # we don't really care about the type of locking it is.
            state = re.sub('^(Table lock|Waiting for .*lock)$', 'Locked', state)
            state = state.replace(' ', '_')
            if 'State_%s' % (state) in status.keys():
                increment(status, 'State_%s' % (state), 1)
            else:
                increment(status, 'State_other', 1)

    # Get SHOW INNODB STATUS and extract the desired metrics from it
    if chk_options.get('innodb') and status.get('have_innodb') == 'YES':
        cursor.execute('SHOW /*!50000 ENGINE*/ INNODB STATUS')
        result = cursor.fetchall()
        istatus_text = result[0].get('Status')
        istatus_vals = get_innodb_array(istatus_text)
        
        if chk_options.get('get_qrt') and status.get('have_response_time_distribution') == 'YES':
            log_debug('Getting query time histogram')
            i = 0
            cursor.execute(
                '''
                SELECT `count`, total * 1000000 AS total
                FROM INFORMATION_SCHEMA.QUERY_RESPONSE_TIME
                WHERE `time` <> 'TOO LONG'
                '''
            )
            result = cursor.fetchall()
            for row in result:
                if i > 13:
                    # It's possible that the number of rows returned isn't 14.
                    # Don't add extra status counters.
                    break
                count_key = 'Query_time_count_%02d' % (i)
                total_key = 'Query_time_total_%02d' % (i)
                status[count_key] = row['count']
                status[total_key] = row['total']
                i += 1
            # It's also possible that the number of rows returned is too few.
            # Don't leave any status counters unassigned; it will break graphs.
            while i <= 13:
                count_key = 'Query_time_count_%02d' % (i)
                total_key = 'Query_time_total_%02d' % (i)
                status[count_key] = 0
                status[total_key] = 0
                i += 1
        else:
            log_debug('Not getting time histogram because it is not enabled')        
            
        # Override values from InnoDB parsing with values from SHOW STATUS,
        # because InnoDB status might not have everything and the SHOW STATUS is
        # to be preferred where possible.
        
        overrides = {
            'Innodb_buffer_pool_pages_data'  : 'database_pages',
            'Innodb_buffer_pool_pages_dirty' : 'modified_pages',
            'Innodb_buffer_pool_pages_free'  : 'free_pages',
            'Innodb_buffer_pool_pages_total' : 'pool_size',
            'Innodb_data_fsyncs'             : 'file_fsyncs',
            'Innodb_data_pending_reads'      : 'pending_normal_aio_reads',
            'Innodb_data_pending_writes'     : 'pending_normal_aio_writes',
            'Innodb_os_log_pending_fsyncs'   : 'pending_log_flushes',
            'Innodb_pages_created'           : 'pages_created',
            'Innodb_pages_read'              : 'pages_read',
            'Innodb_pages_written'           : 'pages_written',
            'Innodb_rows_deleted'            : 'rows_deleted',
            'Innodb_rows_inserted'           : 'rows_inserted',
            'Innodb_rows_read'               : 'rows_read',
            'Innodb_rows_updated'            : 'rows_updated',
        }
        
        # If the SHOW STATUS value exists, override...
        for k,v in overrides.items():
            if k in status.keys():
                log_debug('Override %s' % (k))
                istatus_vals[v] = status[k]
                
        # Now copy the values into $status.
        for k in istatus_vals.keys():
            status[k] = istatus_vals[k]
            
    # Make table_open_cache backwards-compatible (issue 63).
    if 'table_open_cache' in status.keys():
        status['table_cache'] = status.get('table_open_cache')
        
    # Compute how much of the key buffer is used and unflushed (issue 127).
    status['Key_buf_bytes_used'] = big_sub(status.get('key_buffer_size'), big_multiply(status.get('Key_blocks_unused'), status.get('key_cache_block_size')))
    status['Key_buf_bytes_unflushed'] = big_multiply(status.get('Key_blocks_not_flushed'), status.get('key_cache_block_size'))
    
    if 'unflushed_log' in status.keys() and status.get('unflushed_log'):
        # TODO: I'm not sure what the deal is here; need to debug this.  But the
        # unflushed log bytes spikes a lot sometimes and it's impossible for it to
        # be more than the log buffer.
        log_debug('Unflushed log: %s' % (status.get('unflushed_log')))
        status['unflushed_log'] = max(status.get('unflushed_log'), status.get('innodb_log_buffer_size'))
        
    keys = [
        'Key_read_requests',
        'Key_reads',
        'Key_write_requests',
        'Key_writes',
        'history_list',
        'innodb_transactions',
        'read_views',
        'current_transactions',
        'locked_transactions',
        'active_transactions',
        'pool_size',
        'free_pages',
        'database_pages',
        'modified_pages',
        'pages_read',
        'pages_created',
        'pages_written',
        'file_fsyncs',
        'file_reads',
        'file_writes',
        'log_writes',
        'pending_aio_log_ios',
        'pending_aio_sync_ios',
        'pending_buf_pool_flushes',
        'pending_chkp_writes',
        'pending_ibuf_aio_reads',
        'pending_log_flushes',
        'pending_log_writes',
        'pending_normal_aio_reads',
        'pending_normal_aio_writes',
        'ibuf_inserts',
        'ibuf_merged',
        'ibuf_merges',
        'spin_waits',
        'spin_rounds',
        'os_waits',
        'rows_inserted',
        'rows_updated',
        'rows_deleted',
        'rows_read',
        'Table_locks_waited',
        'Table_locks_immediate',
        'Slow_queries',
        'Open_files',
        'Open_tables',
        'Opened_tables',
        'innodb_open_files',
        'open_files_limit',
        'table_cache',
        'Aborted_clients',
        'Aborted_connects',
        'Max_used_connections',
        'Slow_launch_threads',
        'Threads_cached',
        'Threads_connected',
        'Threads_created',
        'Threads_running',
        'max_connections',
        'thread_cache_size',
        'Connections',
        'slave_running',
        'slave_stopped',
        'Slave_retried_transactions',
        'slave_lag',
        'Slave_open_temp_tables',
        'Qcache_free_blocks',
        'Qcache_free_memory',
        'Qcache_hits',
        'Qcache_inserts',
        'Qcache_lowmem_prunes',
        'Qcache_not_cached',
        'Qcache_queries_in_cache',
        'Qcache_total_blocks',
        'query_cache_size',
        'Questions',
        'Com_update',
        'Com_insert',
        'Com_select',
        'Com_delete',
        'Com_replace',
        'Com_load',
        'Com_update_multi',
        'Com_insert_select',
        'Com_delete_multi',
        'Com_replace_select',
        'Select_full_join',
        'Select_full_range_join',
        'Select_range',
        'Select_range_check',
        'Select_scan',
        'Sort_merge_passes',
        'Sort_range',
        'Sort_rows',
        'Sort_scan',
        'Created_tmp_tables',
        'Created_tmp_disk_tables',
        'Created_tmp_files',
        'Bytes_sent',
        'Bytes_received',
        'innodb_log_buffer_size',
        'unflushed_log',
        'log_bytes_flushed',
        'log_bytes_written',
        'relay_log_space',
        'binlog_cache_size',
        'Binlog_cache_disk_use',
        'Binlog_cache_use',
        'binary_log_space',
        'innodb_locked_tables',
        'innodb_lock_structs',
        'State_closing_tables',
        'State_copying_to_tmp_table',
        'State_end',
        'State_freeing_items',
        'State_init',
        'State_locked',
        'State_login',
        'State_preparing',
        'State_reading_from_net',
        'State_sending_data',
        'State_sorting_result',
        'State_statistics',
        'State_updating',
        'State_writing_to_net',
        'State_none',
        'State_other',
        'Handler_commit',
        'Handler_delete',
        'Handler_discover',
        'Handler_prepare',
        'Handler_read_first',
        'Handler_read_key',
        'Handler_read_next',
        'Handler_read_prev',
        'Handler_read_rnd',
        'Handler_read_rnd_next',
        'Handler_rollback',
        'Handler_savepoint',
        'Handler_savepoint_rollback',
        'Handler_update',
        'Handler_write',
        'innodb_tables_in_use',
        'innodb_lock_wait_secs',
        'hash_index_cells_total',
        'hash_index_cells_used',
        'total_mem_alloc',
        'additional_pool_alloc',
        'uncheckpointed_bytes',
        'ibuf_used_cells',
        'ibuf_free_cells',
        'ibuf_cell_count',
        'adaptive_hash_memory',
        'page_hash_memory',
        'dictionary_cache_memory',
        'file_system_memory',
        'lock_system_memory',
        'recovery_system_memory',
        'thread_hash_memory',
        'innodb_sem_waits',
        'innodb_sem_wait_time_ms',
        'Key_buf_bytes_unflushed',
        'Key_buf_bytes_used',
        'key_buffer_size',
        'Innodb_row_lock_time',
        'Innodb_row_lock_waits',
        'Query_time_count_00',
        'Query_time_count_01',
        'Query_time_count_02',
        'Query_time_count_03',
        'Query_time_count_04',
        'Query_time_count_05',
        'Query_time_count_06',
        'Query_time_count_07',
        'Query_time_count_08',
        'Query_time_count_09',
        'Query_time_count_10',
        'Query_time_count_11',
        'Query_time_count_12',
        'Query_time_count_13',
        'Query_time_total_00',
        'Query_time_total_01',
        'Query_time_total_02',
        'Query_time_total_03',
        'Query_time_total_04',
        'Query_time_total_05',
        'Query_time_total_06',
        'Query_time_total_07',
        'Query_time_total_08',
        'Query_time_total_09',
        'Query_time_total_10',
        'Query_time_total_11',
        'Query_time_total_12',
        'Query_time_total_13',
    ] 
    
    # Return the output.
    output = []
    for k in keys:
        # If the value isn't defined, return -1 which is lower than (most graphs')
        # minimum value of 0, so it'll be regarded as a missing value.
        val = status.get(k) if status.get(k) is not None else -1
        output.append('%s:%s' % (k, str(val)))
        
    result = ' '.join(output)
    if fp is not None:
        with open(cache_file, 'w+') as fp:
            fp.write('%s\n' % result)
    db.close()
    return result