Here are the examples of the python api re.sub taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
200 Examples
0
Example 151
View license def do_POST(self, group_name, lines, ip_address, username=''):
forum_id = self.get_forum(group_name)
prefix = settings.phpbb_table_prefix
# patch by Andreas Wegmann <[email protected]> to fix the handling of unusual encodings of messages
lines = mime_decode_header(re.sub(q_quote_multiline, "=?\\1?Q?\\2\\3?=", lines))
body = self.get_message_body(lines)
author, email = from_regexp.search(lines, 0).groups()
subject = subject_regexp.search(lines, 0).groups()[0].strip()
# get the authentication information now
if username != '':
stmt = """
SELECT
user_id
FROM
%susers
WHERE
username='%s'""" % (prefix, username)
num_rows = self.cursor.execute(stmt)
if num_rows == 0:
poster_id = -1
else:
poster_id = self.cursor.fetchone()[0]
post_username = ''
else:
poster_id = -1
post_username = author
if lines.find('References') != -1:
# get the 'modifystamp' value from the parent (if any)
references = references_regexp.search(lines, 0).groups()
parent_id, void = references[-1].strip().split('@')
stmt = """
SELECT
topic_id
FROM
%sposts
WHERE
post_id=%s
GROUP BY
post_id""" % (prefix, parent_id)
num_rows = self.cursor.execute(stmt)
if num_rows == 0:
return None
thread_id = self.cursor.fetchone()[0]
else:
# create a new topic
stmt = """
INSERT INTO
%stopics
(
forum_id,
topic_title,
topic_poster,
topic_time,
topic_status,
topic_vote,
topic_type
) VALUES (
%s,
'%s',
%s,
UNIX_TIMESTAMP(),
0,
0,
0
)""" % (prefix, forum_id, self.quote_string(subject), poster_id)
self.cursor.execute(stmt)
thread_id = self.cursor.insert_id()
stmt = """
INSERT INTO
%sposts
(
topic_id,
forum_id,
poster_id,
post_time,
poster_ip,
post_username,
enable_bbcode,
enable_html,
enable_smilies,
enable_sig
) VALUES (
%s,
%s,
%s,
UNIX_TIMESTAMP(),
'%s',
'%s',
1,
0,
1,
0
)""" % (prefix, thread_id, forum_id, poster_id, self.encode_ip(ip_address), post_username)
self.cursor.execute(stmt)
new_id = self.cursor.insert_id()
if not new_id:
return None
else:
# insert into the '*posts_text' table
stmt = """
INSERT INTO
%sposts_text
(
post_id,
bbcode_uid,
post_subject,
post_text
) VALUES (
%s,
'%s',
'%s',
'%s'
)""" % (prefix, new_id, self.make_bbcode_uid(), self.quote_string(subject), self.quote_string(body))
if not self.cursor.execute(stmt):
# delete from 'topics' and 'posts' tables before returning...
stmt = """
DELETE FROM
%stopics
WHERE
topic_id=%s""" % (prefix, thread_id)
self.cursor.execute(stmt)
stmt = """
DELETE FROM
%sposts
WHERE
post_id=%s""" % (prefix, new_id)
self.cursor.execute(stmt)
return None
else:
if lines.find('References') != -1:
# update the total number of posts in the forum
stmt = """
UPDATE
%sforums
SET
forum_posts=forum_posts+1,
forum_last_post_id=%s
WHERE
forum_id=%s
""" % (settings.phpbb_table_prefix, new_id, forum_id)
self.cursor.execute(stmt)
else:
# update the total number of topics and posts in the forum
stmt = """
UPDATE
%sforums
SET
forum_topics=forum_topics+1,
forum_posts=forum_posts+1,
forum_last_post_id=%s
WHERE
forum_id=%s
""" % (settings.phpbb_table_prefix, new_id, forum_id)
self.cursor.execute(stmt)
# update the user's post count, if this is indeed a real user
if poster_id != -1:
stmt = """
UPDATE
%susers
SET
user_posts=user_posts+1
WHERE
user_id=%s""" % (prefix, poster_id)
self.cursor.execute(stmt)
# setup last post on the topic thread (Patricio Anguita <[email protected]>)
stmt = """
UPDATE
%stopics
SET
topic_replies=topic_replies+1,
topic_last_post_id=%s
WHERE
topic_id=%s""" % (prefix, new_id, thread_id)
self.cursor.execute(stmt)
# if this is the first post on the thread.. (Patricio Anguita <[email protected]>)
if lines.find('References') == -1:
stmt = """
UPDATE
%stopics
SET
topic_first_post_id=%s
WHERE
topic_id=%s AND
topic_first_post_id=0""" % (prefix, new_id, thread_id)
self.cursor.execute(stmt)
return 1
0
Example 152
View license def do_POST(self, group_name, lines, ip_address, username=''):
forum_id = self.get_forum(group_name)
prefix = settings.phpbb_table_prefix
nuke_prefix = settings.nuke_table_prefix
# patch by Andreas Wegmann <[email protected]> to fix the handling of unusual encodings of messages
lines = mime_decode_header(re.sub(q_quote_multiline, "=?\\1?Q?\\2\\3?=", lines))
body = self.get_message_body(lines)
author, email = from_regexp.search(lines, 0).groups()
subject = subject_regexp.search(lines, 0).groups()[0].strip()
# get the authentication information now
if username != '':
stmt = """
SELECT
user_id
FROM
%susers
WHERE
username='%s'""" % (nuke_prefix, username)
num_rows = self.cursor.execute(stmt)
if num_rows == 0:
poster_id = -1
else:
poster_id = self.cursor.fetchone()[0]
post_username = ''
else:
poster_id = -1
post_username = author
if lines.find('References') != -1:
# get the 'modifystamp' value from the parent (if any)
references = references_regexp.search(lines, 0).groups()
parent_id, void = references[-1].strip().split('@')
stmt = """
SELECT
topic_id
FROM
%sposts
WHERE
post_id=%s
GROUP BY
post_id""" % (prefix, parent_id)
num_rows = self.cursor.execute(stmt)
if num_rows == 0:
return None
thread_id = self.cursor.fetchone()[0]
else:
# create a new topic
stmt = """
INSERT INTO
%stopics
(
forum_id,
topic_title,
topic_poster,
topic_time,
topic_status,
topic_vote,
topic_type
) VALUES (
%s,
'%s',
%s,
UNIX_TIMESTAMP(),
0,
0,
0
)""" % (prefix, forum_id, self.quote_string(subject), poster_id)
self.cursor.execute(stmt)
thread_id = self.cursor.insert_id()
stmt = """
INSERT INTO
%sposts
(
topic_id,
forum_id,
poster_id,
post_time,
poster_ip,
post_username,
enable_bbcode,
enable_html,
enable_smilies,
enable_sig
) VALUES (
%s,
%s,
%s,
UNIX_TIMESTAMP(),
'%s',
'%s',
1,
0,
1,
0
)""" % (prefix, thread_id, forum_id, poster_id, self.encode_ip(ip_address), post_username)
self.cursor.execute(stmt)
new_id = self.cursor.insert_id()
if not new_id:
return None
else:
# insert into the '*posts_text' table
stmt = """
INSERT INTO
%sposts_text
(
post_id,
bbcode_uid,
post_subject,
post_text
) VALUES (
%s,
'%s',
'%s',
'%s'
)""" % (prefix, new_id, self.make_bbcode_uid(), self.quote_string(subject), self.quote_string(body))
if not self.cursor.execute(stmt):
# delete from 'topics' and 'posts' tables before returning...
stmt = """
DELETE FROM
%stopics
WHERE
topic_id=%s""" % (prefix, thread_id)
self.cursor.execute(stmt)
stmt = """
DELETE FROM
%sposts
WHERE
post_id=%s""" % (prefix, new_id)
self.cursor.execute(stmt)
return None
else:
if lines.find('References') != -1:
# update the total number of posts in the forum
stmt = """
UPDATE
%sforums
SET
forum_posts=forum_posts+1,
forum_last_post_id=%s
WHERE
forum_id=%s
""" % (settings.phpbb_table_prefix, new_id, forum_id)
self.cursor.execute(stmt)
else:
# update the total number of topics and posts in the forum
stmt = """
UPDATE
%sforums
SET
forum_topics=forum_topics+1,
forum_posts=forum_posts+1,
forum_last_post_id=%s
WHERE
forum_id=%s
""" % (settings.phpbb_table_prefix, new_id, forum_id)
self.cursor.execute(stmt)
# update the user's post count, if this is indeed a real user
if poster_id != -1:
stmt = """
UPDATE
%susers
SET
user_posts=user_posts+1
WHERE
user_id=%s""" % (nuke_prefix, poster_id)
self.cursor.execute(stmt)
# setup last post on the topic thread (Patricio Anguita <[email protected]>)
stmt = """
UPDATE
%stopics
SET
topic_replies=topic_replies+1,
topic_last_post_id=%s
WHERE
topic_id=%s""" % (prefix, new_id, thread_id)
self.cursor.execute(stmt)
# if this is the first post on the thread.. (Patricio Anguita <[email protected]>)
if lines.find('References') == -1:
stmt = """
UPDATE
%stopics
SET
topic_first_post_id=%s
WHERE
topic_id=%s AND
topic_first_post_id=0""" % (prefix, new_id, thread_id)
self.cursor.execute(stmt)
return 1
0
Example 153
View license def preprocess_pmathml(self, xml):
r"""
Pre-process presentation MathML from ASCIIMathML to make it more
acceptable for SnuggleTeX, and also to accomodate some sympy
conventions (eg hat(i) for \hat{i}).
This method would be a good spot to look for an integral and convert
it, if possible...
"""
if type(xml) == str or type(xml) == unicode:
xml = etree.fromstring(xml) # TODO: wrap in try
xml = self.fix_greek_in_mathml(xml) # convert greek utf letters to greek spelled out in ascii
def gettag(expr):
return re.sub('{http://[^}]+}', '', expr.tag)
def fix_pmathml(xml):
"""
f and g are processed as functions by asciimathml, eg "f-2" turns
into "<mrow><mi>f</mi><mo>-</mo></mrow><mn>2</mn>" this is
really terrible for turning into cmathml. undo this here.
"""
for k in xml:
tag = gettag(k)
if tag == 'mrow':
if len(k) == 2:
if gettag(k[0]) == 'mi' and k[0].text in ['f', 'g'] and gettag(k[1]) == 'mo':
idx = xml.index(k)
xml.insert(idx, deepcopy(k[0])) # drop the <mrow> container
xml.insert(idx + 1, deepcopy(k[1]))
xml.remove(k)
fix_pmathml(k)
fix_pmathml(xml)
def fix_hat(xml):
"""
hat i is turned into <mover><mi>i</mi><mo>^</mo></mover> ; mangle
this into <mi>hat(f)</mi> hat i also somtimes turned into
<mover><mrow> <mi>j</mi> </mrow><mo>^</mo></mover>
"""
for k in xml:
tag = gettag(k)
if tag == 'mover':
if len(k) == 2:
if gettag(k[0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
newk = etree.Element('mi')
newk.text = 'hat(%s)' % k[0].text
xml.replace(k, newk)
if gettag(k[0]) == 'mrow' and gettag(k[0][0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
newk = etree.Element('mi')
newk.text = 'hat(%s)' % k[0][0].text
xml.replace(k, newk)
fix_hat(k)
fix_hat(xml)
def flatten_pmathml(xml):
"""
Give the text version of certain PMathML elements
Sometimes MathML will be given with each letter separated (it
doesn't know if its implicit multiplication or what). From an xml
node, find the (text only) variable name it represents. So it takes
<mrow>
<mi>m</mi>
<mi>a</mi>
<mi>x</mi>
</mrow>
and returns 'max', for easier use later on.
"""
tag = gettag(xml)
if tag == 'mn':
return xml.text
elif tag == 'mi':
return xml.text
elif tag == 'mrow':
return ''.join([flatten_pmathml(y) for y in xml])
raise Exception('[flatten_pmathml] unknown tag %s' % tag)
def fix_mathvariant(parent):
"""
Fix certain kinds of math variants
Literally replace <mstyle mathvariant="script"><mi>N</mi></mstyle>
with 'scriptN'. There have been problems using script_N or script(N)
"""
for child in parent:
if (gettag(child) == 'mstyle' and child.get('mathvariant') == 'script'):
newchild = etree.Element('mi')
newchild.text = 'script%s' % flatten_pmathml(child[0])
parent.replace(child, newchild)
fix_mathvariant(child)
fix_mathvariant(xml)
# find "tagged" superscripts
# they have the character \u200b in the superscript
# replace them with a__b so snuggle doesn't get confused
def fix_superscripts(xml):
""" Look for and replace sup elements with 'X__Y' or 'X_Y__Z'
In the javascript, variables with '__X' in them had an invisible
character inserted into the sup (to distinguish from powers)
E.g. normal:
<msubsup>
<mi>a</mi>
<mi>b</mi>
<mi>c</mi>
</msubsup>
to be interpreted '(a_b)^c' (nothing done by this method)
And modified:
<msubsup>
<mi>b</mi>
<mi>x</mi>
<mrow>
<mo>​</mo>
<mi>d</mi>
</mrow>
</msubsup>
to be interpreted 'a_b__c'
also:
<msup>
<mi>x</mi>
<mrow>
<mo>​</mo>
<mi>B</mi>
</mrow>
</msup>
to be 'x__B'
"""
for k in xml:
tag = gettag(k)
# match things like the last example--
# the second item in msub is an mrow with the first
# character equal to \u200b
if (
tag == 'msup' and
len(k) == 2 and gettag(k[1]) == 'mrow' and
gettag(k[1][0]) == 'mo' and k[1][0].text == u'\u200b' # whew
):
# replace the msup with 'X__Y'
k[1].remove(k[1][0])
newk = etree.Element('mi')
newk.text = '%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]))
xml.replace(k, newk)
# match things like the middle example-
# the third item in msubsup is an mrow with the first
# character equal to \u200b
if (
tag == 'msubsup' and
len(k) == 3 and gettag(k[2]) == 'mrow' and
gettag(k[2][0]) == 'mo' and k[2][0].text == u'\u200b' # whew
):
# replace the msubsup with 'X_Y__Z'
k[2].remove(k[2][0])
newk = etree.Element('mi')
newk.text = '%s_%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]), flatten_pmathml(k[2]))
xml.replace(k, newk)
fix_superscripts(k)
fix_superscripts(xml)
def fix_msubsup(parent):
"""
Snuggle returns an error when it sees an <msubsup> replace such
elements with an <msup>, except the first element is of
the form a_b. I.e. map a_b^c => (a_b)^c
"""
for child in parent:
# fix msubsup
if (gettag(child) == 'msubsup' and len(child) == 3):
newchild = etree.Element('msup')
newbase = etree.Element('mi')
newbase.text = '%s_%s' % (flatten_pmathml(child[0]), flatten_pmathml(child[1]))
newexp = child[2]
newchild.append(newbase)
newchild.append(newexp)
parent.replace(child, newchild)
fix_msubsup(child)
fix_msubsup(xml)
self.xml = xml # pylint: disable=attribute-defined-outside-init
return self.xml
0
Example 154
View license def preprocess_pmathml(self, xml):
r"""
Pre-process presentation MathML from ASCIIMathML to make it more
acceptable for SnuggleTeX, and also to accomodate some sympy
conventions (eg hat(i) for \hat{i}).
This method would be a good spot to look for an integral and convert
it, if possible...
"""
if type(xml) == str or type(xml) == unicode:
xml = etree.fromstring(xml) # TODO: wrap in try
xml = self.fix_greek_in_mathml(xml) # convert greek utf letters to greek spelled out in ascii
def gettag(expr):
return re.sub('{http://[^}]+}', '', expr.tag)
def fix_pmathml(xml):
"""
f and g are processed as functions by asciimathml, eg "f-2" turns
into "<mrow><mi>f</mi><mo>-</mo></mrow><mn>2</mn>" this is
really terrible for turning into cmathml. undo this here.
"""
for k in xml:
tag = gettag(k)
if tag == 'mrow':
if len(k) == 2:
if gettag(k[0]) == 'mi' and k[0].text in ['f', 'g'] and gettag(k[1]) == 'mo':
idx = xml.index(k)
xml.insert(idx, deepcopy(k[0])) # drop the <mrow> container
xml.insert(idx + 1, deepcopy(k[1]))
xml.remove(k)
fix_pmathml(k)
fix_pmathml(xml)
def fix_hat(xml):
"""
hat i is turned into <mover><mi>i</mi><mo>^</mo></mover> ; mangle
this into <mi>hat(f)</mi> hat i also somtimes turned into
<mover><mrow> <mi>j</mi> </mrow><mo>^</mo></mover>
"""
for k in xml:
tag = gettag(k)
if tag == 'mover':
if len(k) == 2:
if gettag(k[0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
newk = etree.Element('mi')
newk.text = 'hat(%s)' % k[0].text
xml.replace(k, newk)
if gettag(k[0]) == 'mrow' and gettag(k[0][0]) == 'mi' and gettag(k[1]) == 'mo' and str(k[1].text) == '^':
newk = etree.Element('mi')
newk.text = 'hat(%s)' % k[0][0].text
xml.replace(k, newk)
fix_hat(k)
fix_hat(xml)
def flatten_pmathml(xml):
"""
Give the text version of certain PMathML elements
Sometimes MathML will be given with each letter separated (it
doesn't know if its implicit multiplication or what). From an xml
node, find the (text only) variable name it represents. So it takes
<mrow>
<mi>m</mi>
<mi>a</mi>
<mi>x</mi>
</mrow>
and returns 'max', for easier use later on.
"""
tag = gettag(xml)
if tag == 'mn':
return xml.text
elif tag == 'mi':
return xml.text
elif tag == 'mrow':
return ''.join([flatten_pmathml(y) for y in xml])
raise Exception('[flatten_pmathml] unknown tag %s' % tag)
def fix_mathvariant(parent):
"""
Fix certain kinds of math variants
Literally replace <mstyle mathvariant="script"><mi>N</mi></mstyle>
with 'scriptN'. There have been problems using script_N or script(N)
"""
for child in parent:
if (gettag(child) == 'mstyle' and child.get('mathvariant') == 'script'):
newchild = etree.Element('mi')
newchild.text = 'script%s' % flatten_pmathml(child[0])
parent.replace(child, newchild)
fix_mathvariant(child)
fix_mathvariant(xml)
# find "tagged" superscripts
# they have the character \u200b in the superscript
# replace them with a__b so snuggle doesn't get confused
def fix_superscripts(xml):
""" Look for and replace sup elements with 'X__Y' or 'X_Y__Z'
In the javascript, variables with '__X' in them had an invisible
character inserted into the sup (to distinguish from powers)
E.g. normal:
<msubsup>
<mi>a</mi>
<mi>b</mi>
<mi>c</mi>
</msubsup>
to be interpreted '(a_b)^c' (nothing done by this method)
And modified:
<msubsup>
<mi>b</mi>
<mi>x</mi>
<mrow>
<mo>​</mo>
<mi>d</mi>
</mrow>
</msubsup>
to be interpreted 'a_b__c'
also:
<msup>
<mi>x</mi>
<mrow>
<mo>​</mo>
<mi>B</mi>
</mrow>
</msup>
to be 'x__B'
"""
for k in xml:
tag = gettag(k)
# match things like the last example--
# the second item in msub is an mrow with the first
# character equal to \u200b
if (
tag == 'msup' and
len(k) == 2 and gettag(k[1]) == 'mrow' and
gettag(k[1][0]) == 'mo' and k[1][0].text == u'\u200b' # whew
):
# replace the msup with 'X__Y'
k[1].remove(k[1][0])
newk = etree.Element('mi')
newk.text = '%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]))
xml.replace(k, newk)
# match things like the middle example-
# the third item in msubsup is an mrow with the first
# character equal to \u200b
if (
tag == 'msubsup' and
len(k) == 3 and gettag(k[2]) == 'mrow' and
gettag(k[2][0]) == 'mo' and k[2][0].text == u'\u200b' # whew
):
# replace the msubsup with 'X_Y__Z'
k[2].remove(k[2][0])
newk = etree.Element('mi')
newk.text = '%s_%s__%s' % (flatten_pmathml(k[0]), flatten_pmathml(k[1]), flatten_pmathml(k[2]))
xml.replace(k, newk)
fix_superscripts(k)
fix_superscripts(xml)
def fix_msubsup(parent):
"""
Snuggle returns an error when it sees an <msubsup> replace such
elements with an <msup>, except the first element is of
the form a_b. I.e. map a_b^c => (a_b)^c
"""
for child in parent:
# fix msubsup
if (gettag(child) == 'msubsup' and len(child) == 3):
newchild = etree.Element('msup')
newbase = etree.Element('mi')
newbase.text = '%s_%s' % (flatten_pmathml(child[0]), flatten_pmathml(child[1]))
newexp = child[2]
newchild.append(newbase)
newchild.append(newexp)
parent.replace(child, newchild)
fix_msubsup(child)
fix_msubsup(xml)
self.xml = xml # pylint: disable=attribute-defined-outside-init
return self.xml
0
Example 155
View licensedef deploy_latest_app_ami(app_name, env_name):
global aws
aws = AWS.from_config(config)
aws.connect()
lb_group_ids=[aws.get_security_group_id('riker-load-balancer')]
inst_group_ids=[aws.get_security_group_id('riker-instance')]
app = App(env_name, app_name)
health_check_target = app.config.get('health_check', 'TCP:80')
name = re.sub('[^A-Za-z0-9\-]', '-', app.name)
app_image = LatestAppImage(app).get()
print '-----> Connecting to ELB'
elb_conn = boto.connect_elb()
log('info', 'Load balancer', show_header=True)
load_balancer_name = name
try:
elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
lb = elb_result[0]
log('info', 'Found {}'.format(load_balancer_name))
except boto.exception.BotoServerError:
log('info', 'Not found, creating load balancer')
listeners = [(80, 80, 'HTTP', 'HTTP')]
lb = elb_conn.create_load_balancer(name=load_balancer_name,
zones=None,
complex_listeners=listeners,
security_groups=lb_group_ids,
subnets=[aws.subnet_id])
hc = HealthCheck(target=health_check_target)
lb.configure_health_check(hc)
cda = ConnectionDrainingAttribute()
cda.enabled = True
cda.timeout = 300
elb_conn.modify_lb_attribute(load_balancer_name=load_balancer_name,
attribute='connectionDraining',
value=cda)
print '-----> Connecting to AutoScale'
as_conn = boto.connect_autoscale()
log('info', 'Launch configuration', show_header=True)
launch_config_name = "{}-{}".format(name, app_image.tags['deploy-id'])
lc_result = as_conn.get_all_launch_configurations(names=[launch_config_name])
if len(lc_result) == 0:
log('info', 'Not found, creating LaunchConfiguration')
lc = LaunchConfiguration(name=launch_config_name,
image_id=app_image.id,
key_name=aws.key_pair_name,
security_groups=inst_group_ids,
instance_type=aws.instance_type)
as_conn.create_launch_configuration(lc)
else:
log('info', 'Found {}'.format(launch_config_name))
lc = lc_result[0]
existing_group = None
deploy_id = int(app_image.tags['deploy-id'] or 0)
log('info', 'Getting previous auto-scaling group', show_header=True)
for did in xrange(deploy_id-1, 0, -1):
existing_group_name = "{}-{}".format(name, did)
log('info', '{} ?'.format(existing_group_name))
ag_result = as_conn.get_all_groups(names=[existing_group_name])
if len(ag_result) > 0:
existing_group = ag_result[0]
log('info', 'Found {}'.format(existing_group.name))
break
else:
log('info', 'No')
if existing_group is not None:
existing_healthy_instances = [inst for inst in existing_group.instances if inst.lifecycle_state == 'InService' and inst.health_status == 'Healthy']
existing_healthy_instance_count = len(existing_healthy_instances)
desired_capacity = existing_group.desired_capacity
min_size = existing_group.min_size
max_size = existing_group.max_size
if existing_healthy_instance_count == 0 and desired_capacity == 0:
print '-----> WARNING: existing auto-scaling group {} has no healthy instances and a desired capacity of 0. New auto-scaling group will launch 1 instance.'.format(existing_group)
desired_capacity = 1
min_size = 1
max_size = max_size if max_size > 0 else 1
else:
existing_healthy_instance_count = 0
desired_capacity = 1
min_size = 1
max_size = 1
log('info', '{} existing instance(s) found'.format(existing_healthy_instance_count), show_header=True)
log('info', 'Existing auto-scale properties: desired_capacity={}, min_size={}, max_size={}'.format(desired_capacity, min_size, max_size))
log('info', 'Auto-scaling group', show_header=True)
group_name = "{}-{}".format(name, app_image.tags['deploy-id'])
ag_result = as_conn.get_all_groups(names=[group_name])
if len(ag_result) == 0:
log('info', 'Not found, creating autoscale group')
ag = AutoScalingGroup(name=group_name,
load_balancers=[load_balancer_name], launch_config=lc,
desired_capacity=desired_capacity, min_size=min_size, max_size=max_size,
health_check_type='ELB', health_check_period='300',
vpc_zone_identifier=aws.subnet_id)
as_conn.create_auto_scaling_group(ag)
else:
log('info', 'Found {}'.format(group_name))
ag = ag_result[0]
ag.desired_capacity = desired_capacity
ag.max_size = max_size
ag.min_size = min_size
ag.launch_config_name = launch_config_name
ag.update()
log('info', 'Waiting for new instances to become healthy', show_header=True)
all_healthy = False
for i in xrange(60):
if i > 0:
print ' ---'
time.sleep(10)
elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
lb = elb_result[0]
lb_insts = lb.get_instance_health()
print ' Load-balancer instances: {}'.format(lb_insts)
# NOTE: re-get auto-scaling group to get updated instance info.
ag = as_conn.get_all_groups(names=[group_name])[0]
ag_insts = [inst for inst in ag.instances]
log('info', 'Auto-scaling group Instances: {}'.format(ag_insts))
if len(ag_insts) < desired_capacity:
not_yet_launched_count = desired_capacity - len(ag_insts)
log('info', '{} new instance(s) not yet launched'.format(not_yet_launched_count))
continue
ag_inst_ids = set(inst.instance_id for inst in ag_insts)
lb_inst_ids = set(inst.instance_id for inst in lb_insts)
asg_insts_not_in_lb = ag_inst_ids.difference(lb_inst_ids)
if len(asg_insts_not_in_lb) > 0:
log('info', '{} new instance(s) not yet in load balancer'.format(len(asg_insts_not_in_lb)))
continue
new_lb_insts = [inst for inst in lb_insts if inst.instance_id in ag_inst_ids]
healthy_new_lb_insts = [inst for inst in new_lb_insts if inst.state == 'InService']
all_healthy = len(healthy_new_lb_insts) == len(ag_insts)
log('info', '{} new instance(s) are healthy'.format(len(healthy_new_lb_insts)))
diff = existing_healthy_instance_count - len(healthy_new_lb_insts)
if existing_group is not None and diff >= 0:
change = False
if existing_group.desired_capacity != diff:
existing_group.desired_capacity = diff
change = True
if existing_group.max_size != diff:
existing_group.max_size = diff
change = True
if diff < existing_group.min_size:
existing_group.min_size = diff
change = True
if change:
existing_group.update()
log('info', 'Change previous auto-scale group {} properties: desired_capacity={}, min_size={}, max_size={}'.format(existing_group, existing_group.desired_capacity, existing_group.min_size, existing_group.max_size))
if all_healthy:
log('info', 'All new instances healthy!', show_header=True)
healthy_lb_inst_ids = [inst.instance_id for inst in lb_insts if inst.state == 'InService']
previous_healthy_inst_ids = [inst.instance_id for inst in existing_healthy_instances] if existing_group else []
not_yet_out_of_service = set(previous_healthy_inst_ids).intersection(healthy_lb_inst_ids)
if len(not_yet_out_of_service) > 0:
log('info', 'Waiting to remove previous instances ({}) from load balancer'.format(not_yet_out_of_service))
else:
log('info', 'All previous instances ({}) have been removed from load balancer'.format(previous_healthy_inst_ids), show_header=True)
if all_healthy and len(not_yet_out_of_service) == 0:
break
else:
raise Exception("Timeout")
elb_result = elb_conn.get_all_load_balancers(load_balancer_names=[load_balancer_name])
lb = elb_result[0]
lb_insts = [inst for inst in lb.get_instance_health() if inst.state == 'InService']
print '-----> Deployed {} instance(s) of {} to {}'.format(lb_insts, app.name, lb.dns_name)
print '-----> DONE!'
0
Example 156
View license @auth
def _POST(self, *param, **params):
host_id = self.chk_hostby1(param)
if host_id is None: return web.notfound()
model = findbyhost1(self.orm, host_id)
if not validates_guest_import(self):
return web.badrequest(self.view.alert)
uuid = self.input.uuid
kvc = KaresansuiVirtConnection()
try:
# Storage Pool
#inactive_pool = kvc.list_inactive_storage_pool()
inactive_pool = []
active_pool = kvc.list_active_storage_pool()
pools = inactive_pool + active_pool
pools.sort()
export = []
for pool_name in pools:
pool = kvc.search_kvn_storage_pools(pool_name)
path = pool[0].get_info()["target"]["path"]
if os.path.exists(path):
for _afile in glob.glob("%s/*/info.dat" % (path,)):
e_param = ExportConfigParam()
e_param.load_xml_config(_afile)
if e_param.get_uuid() != uuid:
continue
e_name = e_param.get_domain()
_dir = os.path.dirname(_afile)
param = ConfigParam(e_name)
path = "%s/%s.xml" % (_dir, e_name)
if os.path.isfile(path) is False:
self.logger.error('Export corrupt data.(file not found) - path=%s' % path)
return web.internalerror()
param.load_xml_config(path)
if e_name != param.get_domain_name():
self.logger.error('Export corrupt data.(The name does not match) - info=%s, xml=%s' \
% (e_name, param.get_name()))
return web.internalerror()
_dir = os.path.dirname(_afile)
title = e_param.get_title()
if title != "":
title = re.sub("[\r\n]","",title)
if title == "":
title = _('untitled')
created = e_param.get_created()
if created != "":
created_str = time.strftime("%Y/%m/%d %H:%M:%S", \
time.localtime(float(created)))
else:
created_str = _("N/A")
export.append({"info" : {"dir" : _dir,
"pool" : pool_name,
"uuid" : e_param.get_uuid(),
"name" : e_param.get_domain(),
"created" : int(created),
"created_str" : created_str,
"title" : title,
"database" : {"name" : e_param.database["name"],
"tags" : e_param.database["tags"],
"attribute" : e_param.database["attribute"],
"notebook" : {"title" : e_param.database["notebook"]["title"],
"value" : e_param.database["notebook"]["value"],
},
"uniq_key" : e_param.database["uniq_key"],
"hypervisor" : e_param.database["hypervisor"],
"icon" : e_param.database["icon"],
},
},
"xml" : {"on_reboot" : param.get_behavior('on_reboot'),
"on_poweroff" : param.get_behavior('on_poweroff'),
"on_crash" : param.get_behavior('on_crash'),
"boot_dev" : param.get_boot_dev(),
#"bootloader" : param.get_bootloader(),
#"commandline" : param.get_commandline(),
#"current_snapshot" : param.get_current_snapshot(),
'disk' : param.get_disk(),
"domain_name" : param.get_domain_name(),
"domain_type" : param.get_domain_type(),
"features_acpi" : param.get_features_acpi(),
"features_apic" : param.get_features_apic(),
"features_pae" : param.get_features_pae(),
#"initrd" : param.get_initrd(),
"interface" : param.get_interface(),
#"kernel" : param.get_kernel(),
"max_memory" : param.get_max_memory(),
'max_vcpus' : param.get_max_vcpus(),
"max_vcpus_limit" : param.get_max_vcpus_limit(),
"memory" : param.get_memory(),
"uuid" : param.get_uuid(),
"vcpus" : param.get_vcpus(),
"vcpus_limit" : param.get_vcpus_limit(),
"graphics_autoport" : param.get_graphics_autoport(),
"keymap" : param.get_graphics_keymap(),
"graphics_listen" : param.get_graphics_listen(),
"graphics_passwd" : param.get_graphics_passwd(),
"graphics_port" : param.get_graphics_port(),
},
"pool" : pool[0].get_info(),
})
if len(export) != 1:
self.logger.info("Export does not exist. - uuid=%s" % self.view.uuid)
return web.badrequest()
else:
export = export[0]
finally:
kvc.close()
# Pool running?
if export['pool']['is_active'] is False:
return web.badrequest("The destination, the storage pool is not running.")
dest_domname = export['xml']['domain_name']
dest_uniqkey = export['info']['database']['uniq_key']
# Same guest OS is already running.
if m_findby1uniquekey(self.orm, dest_uniqkey) is not None:
self.logger.info(_("guest '%s' already exists. (DB) - %s") % (dest_domname, dest_uniqkey))
return web.badrequest(_("guest '%s' already exists.") % dest_domname)
dest_dir = "%s/%s" % (export['pool']['target']['path'], export['xml']['domain_name'])
if os.path.exists(dest_dir) is True:
self.logger.info(_("guest '%s' already exists. (FS) - %s") % (dest_domname, dest_dir))
return web.badrequest(_("guest '%s' already exists.") % dest_domname)
# disk check
try:
src_disk = "%s/%s/images/%s.img" \
% (export["info"]["dir"], export["info"]["name"], export["info"]["name"])
if os.path.exists(src_disk):
s_size = os.path.getsize(src_disk) / (1024 * 1024) # a unit 'MB'
if chk_create_disk(export["info"]["dir"], s_size) is False:
partition = get_partition_info(export["info"]["dir"], header=False)
return web.badrequest(
_("No space available to create disk image in '%s' partition.") \
% partition[5][0])
except:
pass
extra_uniq_key = string_from_uuid(generate_uuid())
options = {}
options["exportuuid"] = export["info"]["uuid"]
options["destuuid"] = extra_uniq_key
options["quiet"] = None
# Database Notebook
try:
_notebook = n_new(
export["info"]["database"]["notebook"]["title"],
export["info"]["database"]["notebook"]["value"],
)
except:
_notebook = None
# Database Tag
_tags = []
try:
tag_array = comma_split(export["info"]["database"]["tags"])
tag_array = uniq_sort(tag_array)
for x in tag_array:
if t_count(self.orm, x) == 0:
_tags.append(t_new(x))
else:
_tags.append(t_name(self.orm, x))
except:
_tags.append(t_new(""))
parent = m_findby1(self.orm, host_id)
dest_guest = m_new(created_user=self.me,
modified_user=self.me,
uniq_key=extra_uniq_key,
name=export["info"]["database"]["name"],
attribute=int(export["info"]["database"]["attribute"]),
hypervisor=int(export["info"]["database"]["hypervisor"]),
notebook=_notebook,
tags=_tags,
icon=export["info"]["database"]["icon"],
is_deleted=False,
parent=parent,
)
ret = regist_guest(self,
_guest=dest_guest,
icon_filename=export["info"]["database"]["icon"],
cmd=VIRT_COMMAND_IMPORT_GUEST,
options=options,
cmdname=['Import Guest', 'Import Guest'],
rollback_options={"name" : export["xml"]["domain_name"]},
is_create=False
)
if ret is True:
return web.accepted()
else:
return False
0
Example 157
View license @auth
def _POST(self, *param, **params):
host_id = self.chk_hostby1(param)
if host_id is None: return web.notfound()
model = findbyhost1(self.orm, host_id)
if not validates_guest_import(self):
return web.badrequest(self.view.alert)
uuid = self.input.uuid
kvc = KaresansuiVirtConnection()
try:
# Storage Pool
#inactive_pool = kvc.list_inactive_storage_pool()
inactive_pool = []
active_pool = kvc.list_active_storage_pool()
pools = inactive_pool + active_pool
pools.sort()
export = []
for pool_name in pools:
pool = kvc.search_kvn_storage_pools(pool_name)
path = pool[0].get_info()["target"]["path"]
if os.path.exists(path):
for _afile in glob.glob("%s/*/info.dat" % (path,)):
e_param = ExportConfigParam()
e_param.load_xml_config(_afile)
if e_param.get_uuid() != uuid:
continue
e_name = e_param.get_domain()
_dir = os.path.dirname(_afile)
param = ConfigParam(e_name)
path = "%s/%s.xml" % (_dir, e_name)
if os.path.isfile(path) is False:
self.logger.error('Export corrupt data.(file not found) - path=%s' % path)
return web.internalerror()
param.load_xml_config(path)
if e_name != param.get_domain_name():
self.logger.error('Export corrupt data.(The name does not match) - info=%s, xml=%s' \
% (e_name, param.get_name()))
return web.internalerror()
_dir = os.path.dirname(_afile)
title = e_param.get_title()
if title != "":
title = re.sub("[\r\n]","",title)
if title == "":
title = _('untitled')
created = e_param.get_created()
if created != "":
created_str = time.strftime("%Y/%m/%d %H:%M:%S", \
time.localtime(float(created)))
else:
created_str = _("N/A")
export.append({"info" : {"dir" : _dir,
"pool" : pool_name,
"uuid" : e_param.get_uuid(),
"name" : e_param.get_domain(),
"created" : int(created),
"created_str" : created_str,
"title" : title,
"database" : {"name" : e_param.database["name"],
"tags" : e_param.database["tags"],
"attribute" : e_param.database["attribute"],
"notebook" : {"title" : e_param.database["notebook"]["title"],
"value" : e_param.database["notebook"]["value"],
},
"uniq_key" : e_param.database["uniq_key"],
"hypervisor" : e_param.database["hypervisor"],
"icon" : e_param.database["icon"],
},
},
"xml" : {"on_reboot" : param.get_behavior('on_reboot'),
"on_poweroff" : param.get_behavior('on_poweroff'),
"on_crash" : param.get_behavior('on_crash'),
"boot_dev" : param.get_boot_dev(),
#"bootloader" : param.get_bootloader(),
#"commandline" : param.get_commandline(),
#"current_snapshot" : param.get_current_snapshot(),
'disk' : param.get_disk(),
"domain_name" : param.get_domain_name(),
"domain_type" : param.get_domain_type(),
"features_acpi" : param.get_features_acpi(),
"features_apic" : param.get_features_apic(),
"features_pae" : param.get_features_pae(),
#"initrd" : param.get_initrd(),
"interface" : param.get_interface(),
#"kernel" : param.get_kernel(),
"max_memory" : param.get_max_memory(),
'max_vcpus' : param.get_max_vcpus(),
"max_vcpus_limit" : param.get_max_vcpus_limit(),
"memory" : param.get_memory(),
"uuid" : param.get_uuid(),
"vcpus" : param.get_vcpus(),
"vcpus_limit" : param.get_vcpus_limit(),
"graphics_autoport" : param.get_graphics_autoport(),
"keymap" : param.get_graphics_keymap(),
"graphics_listen" : param.get_graphics_listen(),
"graphics_passwd" : param.get_graphics_passwd(),
"graphics_port" : param.get_graphics_port(),
},
"pool" : pool[0].get_info(),
})
if len(export) != 1:
self.logger.info("Export does not exist. - uuid=%s" % self.view.uuid)
return web.badrequest()
else:
export = export[0]
finally:
kvc.close()
# Pool running?
if export['pool']['is_active'] is False:
return web.badrequest("The destination, the storage pool is not running.")
dest_domname = export['xml']['domain_name']
dest_uniqkey = export['info']['database']['uniq_key']
# Same guest OS is already running.
if m_findby1uniquekey(self.orm, dest_uniqkey) is not None:
self.logger.info(_("guest '%s' already exists. (DB) - %s") % (dest_domname, dest_uniqkey))
return web.badrequest(_("guest '%s' already exists.") % dest_domname)
dest_dir = "%s/%s" % (export['pool']['target']['path'], export['xml']['domain_name'])
if os.path.exists(dest_dir) is True:
self.logger.info(_("guest '%s' already exists. (FS) - %s") % (dest_domname, dest_dir))
return web.badrequest(_("guest '%s' already exists.") % dest_domname)
# disk check
try:
src_disk = "%s/%s/images/%s.img" \
% (export["info"]["dir"], export["info"]["name"], export["info"]["name"])
if os.path.exists(src_disk):
s_size = os.path.getsize(src_disk) / (1024 * 1024) # a unit 'MB'
if chk_create_disk(export["info"]["dir"], s_size) is False:
partition = get_partition_info(export["info"]["dir"], header=False)
return web.badrequest(
_("No space available to create disk image in '%s' partition.") \
% partition[5][0])
except:
pass
extra_uniq_key = string_from_uuid(generate_uuid())
options = {}
options["exportuuid"] = export["info"]["uuid"]
options["destuuid"] = extra_uniq_key
options["quiet"] = None
# Database Notebook
try:
_notebook = n_new(
export["info"]["database"]["notebook"]["title"],
export["info"]["database"]["notebook"]["value"],
)
except:
_notebook = None
# Database Tag
_tags = []
try:
tag_array = comma_split(export["info"]["database"]["tags"])
tag_array = uniq_sort(tag_array)
for x in tag_array:
if t_count(self.orm, x) == 0:
_tags.append(t_new(x))
else:
_tags.append(t_name(self.orm, x))
except:
_tags.append(t_new(""))
parent = m_findby1(self.orm, host_id)
dest_guest = m_new(created_user=self.me,
modified_user=self.me,
uniq_key=extra_uniq_key,
name=export["info"]["database"]["name"],
attribute=int(export["info"]["database"]["attribute"]),
hypervisor=int(export["info"]["database"]["hypervisor"]),
notebook=_notebook,
tags=_tags,
icon=export["info"]["database"]["icon"],
is_deleted=False,
parent=parent,
)
ret = regist_guest(self,
_guest=dest_guest,
icon_filename=export["info"]["database"]["icon"],
cmd=VIRT_COMMAND_IMPORT_GUEST,
options=options,
cmdname=['Import Guest', 'Import Guest'],
rollback_options={"name" : export["xml"]["domain_name"]},
is_create=False
)
if ret is True:
return web.accepted()
else:
return False
0
Example 158
View license @auth
def _POST(self, *param, **params):
host_id = self.chk_hostby1(param)
if host_id is None: return web.notfound()
if not validates_watch(self):
self.logger.debug("Set watch failed. Did not validate.")
return web.badrequest(self.view.alert)
plugin = self.input.watch_target
plugin_instance = None
type = None
type_instance = None
plugin_ds = None
libvirt_host = None
if plugin == COLLECTD_PLUGIN_CPU:
#cpu method
plugin_instance = string.atoi(self.input.logical_cpu_number) - 1
type_instance = self.input.cpu_status
type = COLLECTD_CPU_TYPE
plugin_ds = COLLECTD_CPU_DS
elif plugin == COLLECTD_PLUGIN_MEMORY:
#memory method
type_instance = self.input.memory_status
type = COLLECTD_MEMORY_TYPE
plugin_ds = COLLECTD_MEMORY_DS
elif plugin == COLLECTD_PLUGIN_DF:
#df method
type = COLLECTD_DF_TYPE
type_instance = self.input.df_target_fs
type_instance = re.sub(r'^/dev/', '', type_instance)
type_instance = re.sub(r'/', '_', type_instance)
plugin_ds = self.input.df_disk_status
elif plugin == COLLECTD_PLUGIN_INTERFACE:
#interface method
type = self.input.network_status
type_instance = self.input.network_target_interface
plugin_ds = self.input.network_direction
elif plugin == COLLECTD_PLUGIN_LIBVIRT:
#libvirt method
libvirt_host = self.input.libvirt_target_machine
if self.input.libvirt_target == "cpu":
if self.input.libvirt_vcpu_target == "total":
type = COLLECTD_LIBVIRT_TYPE['CPU_TOTAL']
else:
type = COLLECTD_LIBVIRT_TYPE['VCPU']
type_instance = self.input.libvirt_vcpu_target
plugin_ds = COLLECTD_CPU_DS
elif self.input.libvirt_target == "disk":
type = COLLECTD_LIBVIRT_TYPE['DISK_OCTETS']
type_instance = self.input.libvirt_disk_target
plugin_ds = self.input.libvirt_disk_value_type
elif self.input.libvirt_target == "network":
type = "if_" + self.input.libvirt_network_status
type_instance = self.input.libvirt_target_interface
plugin_ds = self.input.libvirt_network_direction
elif plugin == COLLECTD_PLUGIN_LOAD:
#load method
type = COLLECTD_LOAD_TYPE
plugin_ds = self.input.load_term
else:
self.logger.debug("Set watch failed. Unknown plugin type.")
return web.badrequest()
plugin_selector = create_plugin_selector(plugin_instance, type, type_instance, plugin_ds, libvirt_host)
## text
continuation_count = self.input.continuation_count
prohibition_period = self.input.prohibition_period
threshold_val1 = self.input.threshold_val1
threshold_val2 = self.input.threshold_val2
threshold_type = self.input.threshold_type
if is_param(self.input, 'warning_script'):
warning_script = self.input.warning_script
else:
warning_script = ""
if is_param(self.input, 'warning_mail_body'):
warning_mail_body = self.input.warning_mail_body
else:
warning_mail_body = ""
if is_param(self.input, 'failure_script'):
failure_script = self.input.failure_script
else:
failure_script = ""
if is_param(self.input, 'failure_mail_body'):
failure_mail_body = self.input.failure_mail_body
else:
failure_mail_body = ""
if is_param(self.input, 'okay_script'):
okay_script = self.input.okay_script
else:
okay_script = ""
if is_param(self.input, 'okay_mail_body'):
okay_mail_body = self.input.okay_mail_body
else:
okay_mail_body = ""
if is_param(self.input, 'notify_mail_to'):
notify_mail_to = self.input.notify_mail_to
else:
notify_mail_to = ""
if is_param(self.input, 'notify_mail_from'):
notify_mail_from = self.input.notify_mail_from
else:
notify_mail_from = ""
## bool
bool_input_key = ["use_percentage", "enable_warning_mail",
"enable_failure_mail", "enable_okay_mail",
"enable_warning_script", "enable_failure_script",
"enable_okay_script"]
bool_values = {}
for key in bool_input_key:
if self.input.has_key(key):
bool_values.update({key:True})
else:
bool_values.update({key:False})
if threshold_type == "max":
warning_value = create_threshold_value(min_value=None, max_value=threshold_val1)
failure_value = create_threshold_value(min_value=None, max_value=threshold_val2)
elif threshold_type == "min":
warning_value = create_threshold_value(min_value=threshold_val2, max_value=None)
failure_value = create_threshold_value(min_value=threshold_val1, max_value=None)
else:
self.logger.debug("Set watch failed. Unknown threshold type.")
return web.badrequest()
machine = m_findby1(self.orm, host_id)
if w_is_uniq_duplication(self.orm, machine, plugin, plugin_selector) is True:
self.logger.debug("Set watch failed. Duplicate watch DB.")
return web.badrequest("Set watch failed. Duplication watch")
_watch = w_new(created_user = self.me,
modified_user = self.me,
name = self.input.watch_name,
plugin = plugin,
plugin_selector = plugin_selector,
karesansui_version = get_karesansui_version(),
collectd_version = get_collectd_version(),
machine = machine,
continuation_count = continuation_count,
prohibition_period = prohibition_period,
warning_value = warning_value,
is_warning_percentage = bool_values.get("use_percentage"),
is_warning_script = bool_values.get("enable_warning_script"),
warning_script = warning_script,
is_warning_mail = bool_values.get("enable_warning_mail"),
warning_mail_body = warning_mail_body,
failure_value = failure_value,
is_failure_percentage = bool_values.get("use_percentage"),
is_failure_script = bool_values.get("enable_failure_script"),
failure_script = failure_script,
is_failure_mail = bool_values.get("enable_failure_mail"),
failure_mail_body = failure_mail_body,
is_okay_script = bool_values.get("enable_okay_script"),
okay_script = okay_script,
is_okay_mail = bool_values.get("enable_okay_mail"),
okay_mail_body = okay_mail_body,
notify_mail_to = notify_mail_to,
notify_mail_from = notify_mail_from,
is_deleted = False,
)
w_save(self.orm, _watch)
modules = ["collectdplugin"]
host = m_findbyhost1(self.orm, host_id)
extra_args = {'include':'^threshold_'}
#extra_args = {}
dop = read_conf(modules, webobj=self, machine=host, extra_args=extra_args)
if dop is False:
self.logger.debug("Set watch failed. Failed read conf.")
return web.internalerror('Internal Server Error. (Read Conf)')
params = {}
if threshold_type == "max":
params['WarningMax'] = str(threshold_val1)
params['FailureMax'] = str(threshold_val2)
elif threshold_type == "min":
params['WarningMin'] = str(threshold_val2)
params['FailureMin'] = str(threshold_val1)
params['Percentage'] = str(bool_values.get("use_percentage")).lower()
params['Persist'] = "true"
set_threshold(plugin,plugin_selector,params,dop=dop,webobj=self, host=host)
extra_args = {}
command = "/etc/init.d/collectd condrestart"
extra_args = {"post-command": command}
retval = write_conf(dop, webobj=self, machine=host, extra_args=extra_args)
if retval is False:
self.logger.debug("Set watch failed. Failed write conf.")
return web.internalerror('Internal Server Error. (Write Conf)')
return web.created(None)
0
Example 159
View license @auth
def _POST(self, *param, **params):
host_id = self.chk_hostby1(param)
if host_id is None: return web.notfound()
if not validates_watch(self):
self.logger.debug("Set watch failed. Did not validate.")
return web.badrequest(self.view.alert)
plugin = self.input.watch_target
plugin_instance = None
type = None
type_instance = None
plugin_ds = None
libvirt_host = None
if plugin == COLLECTD_PLUGIN_CPU:
#cpu method
plugin_instance = string.atoi(self.input.logical_cpu_number) - 1
type_instance = self.input.cpu_status
type = COLLECTD_CPU_TYPE
plugin_ds = COLLECTD_CPU_DS
elif plugin == COLLECTD_PLUGIN_MEMORY:
#memory method
type_instance = self.input.memory_status
type = COLLECTD_MEMORY_TYPE
plugin_ds = COLLECTD_MEMORY_DS
elif plugin == COLLECTD_PLUGIN_DF:
#df method
type = COLLECTD_DF_TYPE
type_instance = self.input.df_target_fs
type_instance = re.sub(r'^/dev/', '', type_instance)
type_instance = re.sub(r'/', '_', type_instance)
plugin_ds = self.input.df_disk_status
elif plugin == COLLECTD_PLUGIN_INTERFACE:
#interface method
type = self.input.network_status
type_instance = self.input.network_target_interface
plugin_ds = self.input.network_direction
elif plugin == COLLECTD_PLUGIN_LIBVIRT:
#libvirt method
libvirt_host = self.input.libvirt_target_machine
if self.input.libvirt_target == "cpu":
if self.input.libvirt_vcpu_target == "total":
type = COLLECTD_LIBVIRT_TYPE['CPU_TOTAL']
else:
type = COLLECTD_LIBVIRT_TYPE['VCPU']
type_instance = self.input.libvirt_vcpu_target
plugin_ds = COLLECTD_CPU_DS
elif self.input.libvirt_target == "disk":
type = COLLECTD_LIBVIRT_TYPE['DISK_OCTETS']
type_instance = self.input.libvirt_disk_target
plugin_ds = self.input.libvirt_disk_value_type
elif self.input.libvirt_target == "network":
type = "if_" + self.input.libvirt_network_status
type_instance = self.input.libvirt_target_interface
plugin_ds = self.input.libvirt_network_direction
elif plugin == COLLECTD_PLUGIN_LOAD:
#load method
type = COLLECTD_LOAD_TYPE
plugin_ds = self.input.load_term
else:
self.logger.debug("Set watch failed. Unknown plugin type.")
return web.badrequest()
plugin_selector = create_plugin_selector(plugin_instance, type, type_instance, plugin_ds, libvirt_host)
## text
continuation_count = self.input.continuation_count
prohibition_period = self.input.prohibition_period
threshold_val1 = self.input.threshold_val1
threshold_val2 = self.input.threshold_val2
threshold_type = self.input.threshold_type
if is_param(self.input, 'warning_script'):
warning_script = self.input.warning_script
else:
warning_script = ""
if is_param(self.input, 'warning_mail_body'):
warning_mail_body = self.input.warning_mail_body
else:
warning_mail_body = ""
if is_param(self.input, 'failure_script'):
failure_script = self.input.failure_script
else:
failure_script = ""
if is_param(self.input, 'failure_mail_body'):
failure_mail_body = self.input.failure_mail_body
else:
failure_mail_body = ""
if is_param(self.input, 'okay_script'):
okay_script = self.input.okay_script
else:
okay_script = ""
if is_param(self.input, 'okay_mail_body'):
okay_mail_body = self.input.okay_mail_body
else:
okay_mail_body = ""
if is_param(self.input, 'notify_mail_to'):
notify_mail_to = self.input.notify_mail_to
else:
notify_mail_to = ""
if is_param(self.input, 'notify_mail_from'):
notify_mail_from = self.input.notify_mail_from
else:
notify_mail_from = ""
## bool
bool_input_key = ["use_percentage", "enable_warning_mail",
"enable_failure_mail", "enable_okay_mail",
"enable_warning_script", "enable_failure_script",
"enable_okay_script"]
bool_values = {}
for key in bool_input_key:
if self.input.has_key(key):
bool_values.update({key:True})
else:
bool_values.update({key:False})
if threshold_type == "max":
warning_value = create_threshold_value(min_value=None, max_value=threshold_val1)
failure_value = create_threshold_value(min_value=None, max_value=threshold_val2)
elif threshold_type == "min":
warning_value = create_threshold_value(min_value=threshold_val2, max_value=None)
failure_value = create_threshold_value(min_value=threshold_val1, max_value=None)
else:
self.logger.debug("Set watch failed. Unknown threshold type.")
return web.badrequest()
machine = m_findby1(self.orm, host_id)
if w_is_uniq_duplication(self.orm, machine, plugin, plugin_selector) is True:
self.logger.debug("Set watch failed. Duplicate watch DB.")
return web.badrequest("Set watch failed. Duplication watch")
_watch = w_new(created_user = self.me,
modified_user = self.me,
name = self.input.watch_name,
plugin = plugin,
plugin_selector = plugin_selector,
karesansui_version = get_karesansui_version(),
collectd_version = get_collectd_version(),
machine = machine,
continuation_count = continuation_count,
prohibition_period = prohibition_period,
warning_value = warning_value,
is_warning_percentage = bool_values.get("use_percentage"),
is_warning_script = bool_values.get("enable_warning_script"),
warning_script = warning_script,
is_warning_mail = bool_values.get("enable_warning_mail"),
warning_mail_body = warning_mail_body,
failure_value = failure_value,
is_failure_percentage = bool_values.get("use_percentage"),
is_failure_script = bool_values.get("enable_failure_script"),
failure_script = failure_script,
is_failure_mail = bool_values.get("enable_failure_mail"),
failure_mail_body = failure_mail_body,
is_okay_script = bool_values.get("enable_okay_script"),
okay_script = okay_script,
is_okay_mail = bool_values.get("enable_okay_mail"),
okay_mail_body = okay_mail_body,
notify_mail_to = notify_mail_to,
notify_mail_from = notify_mail_from,
is_deleted = False,
)
w_save(self.orm, _watch)
modules = ["collectdplugin"]
host = m_findbyhost1(self.orm, host_id)
extra_args = {'include':'^threshold_'}
#extra_args = {}
dop = read_conf(modules, webobj=self, machine=host, extra_args=extra_args)
if dop is False:
self.logger.debug("Set watch failed. Failed read conf.")
return web.internalerror('Internal Server Error. (Read Conf)')
params = {}
if threshold_type == "max":
params['WarningMax'] = str(threshold_val1)
params['FailureMax'] = str(threshold_val2)
elif threshold_type == "min":
params['WarningMin'] = str(threshold_val2)
params['FailureMin'] = str(threshold_val1)
params['Percentage'] = str(bool_values.get("use_percentage")).lower()
params['Persist'] = "true"
set_threshold(plugin,plugin_selector,params,dop=dop,webobj=self, host=host)
extra_args = {}
command = "/etc/init.d/collectd condrestart"
extra_args = {"post-command": command}
retval = write_conf(dop, webobj=self, machine=host, extra_args=extra_args)
if retval is False:
self.logger.debug("Set watch failed. Failed write conf.")
return web.internalerror('Internal Server Error. (Write Conf)')
return web.created(None)
0
Example 160
View license def extractInfo(self):
#################/
# name - Naming table
#################/
self.sFamilyClass = 0
self.sFamilySubClass = 0
name_offset = self.seek_table("name")
format = self.read_ushort()
if format != 0:
raise Exception("Unknown name table format " + format)
numRecords = self.read_ushort()
string_data_offset = name_offset + self.read_ushort()
names = {1: '', 2: '', 3: '', 4: '', 6: ''}
K = names.keys()
nameCount = len(names)
for i in range(numRecords):
platformId = self.read_ushort()
encodingId = self.read_ushort()
languageId = self.read_ushort()
nameId = self.read_ushort()
length = self.read_ushort()
offset = self.read_ushort()
if nameId not in K:
continue
N = ''
if platformId == 3 and encodingId == 1 and languageId == 0x409: # Microsoft, Unicode, US English, PS Name
opos = self._pos
self.seek(string_data_offset + offset)
if length % 2 != 0:
raise Exception("PostScript name is UTF-16BE string of odd length")
length /= 2
N = ''
while length > 0:
char = self.read_ushort()
N += (chr(char))
length -= 1
self._pos = opos
self.seek(opos)
elif platformId == 1 and encodingId == 0 and languageId == 0: # Macintosh, Roman, English, PS Name
opos = self._pos
N = self.get_chunk(string_data_offset + offset, length)
self._pos = opos
self.seek(opos)
if N and names[nameId] == '':
names[nameId] = N
nameCount -= 1
if nameCount == 0:
break
if names[6]:
psName = names[6]
elif names[4]:
psName = re.sub(' ', '-', names[4])
elif names[1]:
psName = re.sub(' ', '-', names[1])
else:
psName = ''
if not psName:
raise Exception("Could not find PostScript font name")
self.name = psName
if names[1]:
self.familyName = names[1]
else:
self.familyName = psName
if names[2]:
self.styleName = names[2]
else:
self.styleName = 'Regular'
if names[4]:
self.fullName = names[4]
else:
self.fullName = psName
if names[3]:
self.uniqueFontID = names[3]
else:
self.uniqueFontID = psName
if names[6]:
self.fullName = names[6]
#################/
# head - Font header table
#################/
self.seek_table("head")
self.skip(18)
self.unitsPerEm = unitsPerEm = self.read_ushort()
scale = 1000 / float(unitsPerEm)
self.skip(16)
xMin = self.read_short()
yMin = self.read_short()
xMax = self.read_short()
yMax = self.read_short()
self.bbox = [(xMin * scale), (yMin * scale), (xMax * scale), (yMax * scale)]
self.skip(3 * 2)
indexToLocFormat = self.read_ushort()
glyphDataFormat = self.read_ushort()
if glyphDataFormat != 0:
raise Exception('Unknown glyph data format ' + glyphDataFormat)
#################/
# hhea metrics table
#################/
# ttf2t1 seems to use this value rather than the one in OS/2 - so put in for compatibility
if "hhea" in self.tables:
self.seek_table("hhea")
self.skip(4)
hheaAscender = self.read_short()
hheaDescender = self.read_short()
self.ascent = (hheaAscender * scale)
self.descent = (hheaDescender * scale)
#################/
# OS/2 - OS/2 and Windows metrics table
#################/
if "OS/2" in self.tables:
self.seek_table("OS/2")
version = self.read_ushort()
self.skip(2)
usWeightClass = self.read_ushort()
self.skip(2)
fsType = self.read_ushort()
if fsType == 0x0002 or (fsType & 0x0300) != 0:
self.restrictedUse = True
raise Exception('ERROR - Font file ' + self.filename + ' cannot be embedded due to copyright restrictions.')
self.skip(20)
sF = self.read_short()
self.sFamilyClass = (sF >> 8)
self.sFamilySubClass = (sF & 0xFF)
self._pos += 10 # PANOSE = 10 byte length
panose = self.fh.read(10)
self.skip(26)
sTypoAscender = self.read_short()
sTypoDescender = self.read_short()
if not self.ascent:
self.ascent = (sTypoAscender * scale)
if not self.descent:
self.descent = (sTypoDescender * scale)
if version > 1:
self.skip(16)
sCapHeight = self.read_short()
self.capHeight = (sCapHeight * scale)
else:
self.capHeight = self.ascent
else:
usWeightClass = 500
if not self.ascent:
self.ascent = (yMax * scale)
if not self.descent:
self.descent = (yMin * scale)
self.capHeight = self.ascent
self.stemV = 50 + int(pow((usWeightClass / 65.0), 2))
#################/
# post - PostScript table
#################/
self.seek_table("post")
self.skip(4)
self.italicAngle = self.read_short() + self.read_ushort() / 65536.0
self.underlinePosition = self.read_short() * scale
self.underlineThickness = self.read_short() * scale
isFixedPitch = self.read_ulong()
self.flags = 4
if self.italicAngle != 0:
self.flags = self.flags | 64
if usWeightClass >= 600:
self.flags = self.flags | 262144
if isFixedPitch:
self.flags = self.flags | 1
#################/
# hhea - Horizontal header table
#################/
self.seek_table("hhea")
self.skip(32)
metricDataFormat = self.read_ushort()
if metricDataFormat != 0:
raise Exception('Unknown horizontal metric data format ', metricDataFormat)
numberOfHMetrics = self.read_ushort()
if numberOfHMetrics == 0:
raise Exception('Number of horizontal metrics is 0')
#################/
# maxp - Maximum profile table
#################/
self.seek_table("maxp")
self.skip(4)
numGlyphs = self.read_ushort()
#################/
# cmap - Character to glyph index mapping table
#################/
cmap_offset = self.seek_table("cmap")
self.skip(2)
cmapTableCount = self.read_ushort()
unicode_cmap_offset = 0
for i in range(cmapTableCount):
platformID = self.read_ushort()
encodingID = self.read_ushort()
offset = self.read_ulong()
save_pos = self._pos
if (platformID == 3 and encodingID == 1) or platformID == 0: # Microsoft, Unicode
format = self.get_ushort(cmap_offset + offset)
if format == 4:
if not unicode_cmap_offset:
unicode_cmap_offset = cmap_offset + offset
break
self.seek(save_pos)
if not unicode_cmap_offset:
raise Exception('Font (' + self.filename + ') does not have cmap for Unicode (platform 3, encoding 1, format 4, or platform 0, any encoding, format 4)')
glyphToChar = {}
charToGlyph = {}
self.getCMAP4(unicode_cmap_offset, glyphToChar, charToGlyph)
#################/
# hmtx - Horizontal metrics table
#################/
self.getHMTX(numberOfHMetrics, numGlyphs, glyphToChar, scale)
0
Example 161
View license def extractInfo(self):
#################/
# name - Naming table
#################/
self.sFamilyClass = 0
self.sFamilySubClass = 0
name_offset = self.seek_table("name")
format = self.read_ushort()
if format != 0:
raise Exception("Unknown name table format " + format)
numRecords = self.read_ushort()
string_data_offset = name_offset + self.read_ushort()
names = {1: '', 2: '', 3: '', 4: '', 6: ''}
K = names.keys()
nameCount = len(names)
for i in range(numRecords):
platformId = self.read_ushort()
encodingId = self.read_ushort()
languageId = self.read_ushort()
nameId = self.read_ushort()
length = self.read_ushort()
offset = self.read_ushort()
if nameId not in K:
continue
N = ''
if platformId == 3 and encodingId == 1 and languageId == 0x409: # Microsoft, Unicode, US English, PS Name
opos = self._pos
self.seek(string_data_offset + offset)
if length % 2 != 0:
raise Exception("PostScript name is UTF-16BE string of odd length")
length /= 2
N = ''
while length > 0:
char = self.read_ushort()
N += (chr(char))
length -= 1
self._pos = opos
self.seek(opos)
elif platformId == 1 and encodingId == 0 and languageId == 0: # Macintosh, Roman, English, PS Name
opos = self._pos
N = self.get_chunk(string_data_offset + offset, length)
self._pos = opos
self.seek(opos)
if N and names[nameId] == '':
names[nameId] = N
nameCount -= 1
if nameCount == 0:
break
if names[6]:
psName = names[6]
elif names[4]:
psName = re.sub(' ', '-', names[4])
elif names[1]:
psName = re.sub(' ', '-', names[1])
else:
psName = ''
if not psName:
raise Exception("Could not find PostScript font name")
self.name = psName
if names[1]:
self.familyName = names[1]
else:
self.familyName = psName
if names[2]:
self.styleName = names[2]
else:
self.styleName = 'Regular'
if names[4]:
self.fullName = names[4]
else:
self.fullName = psName
if names[3]:
self.uniqueFontID = names[3]
else:
self.uniqueFontID = psName
if names[6]:
self.fullName = names[6]
#################/
# head - Font header table
#################/
self.seek_table("head")
self.skip(18)
self.unitsPerEm = unitsPerEm = self.read_ushort()
scale = 1000 / float(unitsPerEm)
self.skip(16)
xMin = self.read_short()
yMin = self.read_short()
xMax = self.read_short()
yMax = self.read_short()
self.bbox = [(xMin * scale), (yMin * scale), (xMax * scale), (yMax * scale)]
self.skip(3 * 2)
indexToLocFormat = self.read_ushort()
glyphDataFormat = self.read_ushort()
if glyphDataFormat != 0:
raise Exception('Unknown glyph data format ' + glyphDataFormat)
#################/
# hhea metrics table
#################/
# ttf2t1 seems to use this value rather than the one in OS/2 - so put in for compatibility
if "hhea" in self.tables:
self.seek_table("hhea")
self.skip(4)
hheaAscender = self.read_short()
hheaDescender = self.read_short()
self.ascent = (hheaAscender * scale)
self.descent = (hheaDescender * scale)
#################/
# OS/2 - OS/2 and Windows metrics table
#################/
if "OS/2" in self.tables:
self.seek_table("OS/2")
version = self.read_ushort()
self.skip(2)
usWeightClass = self.read_ushort()
self.skip(2)
fsType = self.read_ushort()
if fsType == 0x0002 or (fsType & 0x0300) != 0:
self.restrictedUse = True
raise Exception('ERROR - Font file ' + self.filename + ' cannot be embedded due to copyright restrictions.')
self.skip(20)
sF = self.read_short()
self.sFamilyClass = (sF >> 8)
self.sFamilySubClass = (sF & 0xFF)
self._pos += 10 # PANOSE = 10 byte length
panose = self.fh.read(10)
self.skip(26)
sTypoAscender = self.read_short()
sTypoDescender = self.read_short()
if not self.ascent:
self.ascent = (sTypoAscender * scale)
if not self.descent:
self.descent = (sTypoDescender * scale)
if version > 1:
self.skip(16)
sCapHeight = self.read_short()
self.capHeight = (sCapHeight * scale)
else:
self.capHeight = self.ascent
else:
usWeightClass = 500
if not self.ascent:
self.ascent = (yMax * scale)
if not self.descent:
self.descent = (yMin * scale)
self.capHeight = self.ascent
self.stemV = 50 + int(pow((usWeightClass / 65.0), 2))
#################/
# post - PostScript table
#################/
self.seek_table("post")
self.skip(4)
self.italicAngle = self.read_short() + self.read_ushort() / 65536.0
self.underlinePosition = self.read_short() * scale
self.underlineThickness = self.read_short() * scale
isFixedPitch = self.read_ulong()
self.flags = 4
if self.italicAngle != 0:
self.flags = self.flags | 64
if usWeightClass >= 600:
self.flags = self.flags | 262144
if isFixedPitch:
self.flags = self.flags | 1
#################/
# hhea - Horizontal header table
#################/
self.seek_table("hhea")
self.skip(32)
metricDataFormat = self.read_ushort()
if metricDataFormat != 0:
raise Exception('Unknown horizontal metric data format ', metricDataFormat)
numberOfHMetrics = self.read_ushort()
if numberOfHMetrics == 0:
raise Exception('Number of horizontal metrics is 0')
#################/
# maxp - Maximum profile table
#################/
self.seek_table("maxp")
self.skip(4)
numGlyphs = self.read_ushort()
#################/
# cmap - Character to glyph index mapping table
#################/
cmap_offset = self.seek_table("cmap")
self.skip(2)
cmapTableCount = self.read_ushort()
unicode_cmap_offset = 0
for i in range(cmapTableCount):
platformID = self.read_ushort()
encodingID = self.read_ushort()
offset = self.read_ulong()
save_pos = self._pos
if (platformID == 3 and encodingID == 1) or platformID == 0: # Microsoft, Unicode
format = self.get_ushort(cmap_offset + offset)
if format == 4:
if not unicode_cmap_offset:
unicode_cmap_offset = cmap_offset + offset
break
self.seek(save_pos)
if not unicode_cmap_offset:
raise Exception('Font (' + self.filename + ') does not have cmap for Unicode (platform 3, encoding 1, format 4, or platform 0, any encoding, format 4)')
glyphToChar = {}
charToGlyph = {}
self.getCMAP4(unicode_cmap_offset, glyphToChar, charToGlyph)
#################/
# hmtx - Horizontal metrics table
#################/
self.getHMTX(numberOfHMetrics, numGlyphs, glyphToChar, scale)
0
Example 162
View license def on_enter(self, *args):
'''When the user press enter and wants to run a command
'''
self.unbind(on_subprocess_done=self.on_enter)
# if there is a command running, it's necessary to stop it first
if self.command_status == 'started':
self.kill_process()
# restart this call after killing the running process
self.bind(on_subprocess_done=self.on_enter)
return
txtinput_command_line = self.txtinput_command_line
add_to_cache = self.add_to_cache
command_history = self.command_history
def remove_command_interaction_widgets(*args):
'''command finished: remove widget responsible for interaction
'''
parent.remove_widget(self.interact_layout)
self.interact_layout = None
# enable running a new command
try:
parent.add_widget(self.txtinput_command_line)
except:
self._initialize(0)
self._focus(txtinput_command_line, True)
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
def run_cmd(*args):
'''Run the command
'''
# this is run inside a thread so take care, avoid gui ops
try:
_posix = True
if sys.platform[0] == 'w':
_posix = False
cmd = command
if PY2 and isinstance(cmd, unicode):
cmd = command.encode(get_fs_encoding())
if not self.shell:
cmd = shlex.split(cmd, posix=_posix)
for i in range(len(cmd)):
cmd[i] = cmd[i].replace('\x01', ' ')
map(lambda s: s.decode(get_fs_encoding()), cmd)
except Exception as err:
cmd = ''
self.add_to_cache(''.join((str(err), ' <', command, ' >\n')))
if len(cmd) > 0:
prev_stdout = sys.stdout
sys.stdout = self.stdout
try:
# execute command
self.popen_obj = popen = subprocess.Popen(
cmd,
bufsize=0,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
preexec_fn=None,
close_fds=False,
shell=self.shell,
cwd=self.cur_dir.encode(get_fs_encoding()),
universal_newlines=False,
startupinfo=None,
creationflags=0)
popen_stdout_r = popen.stdout.readline
popen_stdout_flush = popen.stdout.flush
txt = popen_stdout_r()
plat = platform()
while txt:
# skip flush on android
if plat[0] != 'a':
popen_stdout_flush()
if isinstance(txt, bytes):
txt = txt.decode(get_fs_encoding())
add_to_cache(txt)
txt = popen_stdout_r()
except (OSError, ValueError) as err:
add_to_cache(''.join(
(str(err),
' < ', command, ' >\n')))
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
sys.stdout = prev_stdout
self.popen_obj = None
Clock.schedule_once(remove_command_interaction_widgets, 0)
# append text to textcache
add_to_cache(self.txtinput_command_line.text + '\n')
command = txtinput_command_line.text[len(self.prompt()):]
if command == '':
self.txtinput_command_line_refocus = True
return
# store command in command_history
if self.command_history_pos > 0:
self.command_history_pos = len(command_history)
if command_history[self.command_history_pos - 1] != command:
command_history.append(command)
else:
command_history.append(command)
len_command_history = len(command_history)
self.command_history_pos = len(command_history)
# on reaching limit(cached_lines) pop first command
if len_command_history >= self.cached_commands:
self.command_history = command_history[1:]
# replace $PATH with
command = os.path.expandvars(command)
if command == 'clear' or command == 'cls':
self.clear()
txtinput_command_line.text = self.prompt()
self.txtinput_command_line_refocus = True
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
return
# if command = cd change directory
if command.startswith('cd ') or command.startswith('export '):
if command[0] == 'e':
e_q = command[7:].find('=')
_exprt = command[7:]
if e_q:
os.environ[_exprt[:e_q]] = _exprt[e_q + 1:]
self.environment = os.environ.copy()
else:
try:
command = re.sub('[ ]+', ' ', command)
if command[3] == os.sep:
os.chdir(command[3:])
else:
os.chdir(os.path.join(self.cur_dir, command[3:]))
if sys.version_info >= (3, 0):
self.cur_dir = os.getcwd()
else:
self.cur_dir = os.getcwdu()
except OSError as err:
Logger.debug('Shell Console: err:' + err.strerror +
' directory:' + str(command[3:]))
add_to_cache(''.join((err.strerror, '\n')))
txtinput_command_line.text = self.prompt()
self.txtinput_command_line_refocus = True
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
return
txtinput_command_line.text = self.prompt()
# store output in textcache
parent = txtinput_command_line.parent
# disable running a new command while and old one is running
parent.remove_widget(txtinput_command_line)
# add widget for interaction with the running command
txtinput_run_command = TextInput(multiline=False,
font_size=self.font_size)
def interact_with_command(*l):
'''Text input to interact with the running command
'''
popen_obj = self.popen_obj
if not popen_obj:
return
txt = l[0].text + '\n'
popen_obj_stdin = popen_obj.stdin
popen_obj_stdin.write(txt)
popen_obj_stdin.flush()
self.txtinput_run_command_refocus = True
self.txtinput_run_command_refocus = False
txtinput_run_command.bind(on_text_validate=interact_with_command)
txtinput_run_command.bind(focus=self.on_focus)
btn_kill = Button(text="Stop",
width=60,
size_hint=(None, 1))
self.interact_layout = il = GridLayout(rows=1, cols=2, height=27,
size_hint=(1, None))
btn_kill.bind(on_press=self.kill_process)
il.add_widget(txtinput_run_command)
il.add_widget(btn_kill)
parent.add_widget(il)
txtinput_run_command.focus = True
self.command_status = 'started'
thread.start_new_thread(run_cmd, ())
0
Example 163
View license def on_enter(self, *args):
'''When the user press enter and wants to run a command
'''
self.unbind(on_subprocess_done=self.on_enter)
# if there is a command running, it's necessary to stop it first
if self.command_status == 'started':
self.kill_process()
# restart this call after killing the running process
self.bind(on_subprocess_done=self.on_enter)
return
txtinput_command_line = self.txtinput_command_line
add_to_cache = self.add_to_cache
command_history = self.command_history
def remove_command_interaction_widgets(*args):
'''command finished: remove widget responsible for interaction
'''
parent.remove_widget(self.interact_layout)
self.interact_layout = None
# enable running a new command
try:
parent.add_widget(self.txtinput_command_line)
except:
self._initialize(0)
self._focus(txtinput_command_line, True)
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
def run_cmd(*args):
'''Run the command
'''
# this is run inside a thread so take care, avoid gui ops
try:
_posix = True
if sys.platform[0] == 'w':
_posix = False
cmd = command
if PY2 and isinstance(cmd, unicode):
cmd = command.encode(get_fs_encoding())
if not self.shell:
cmd = shlex.split(cmd, posix=_posix)
for i in range(len(cmd)):
cmd[i] = cmd[i].replace('\x01', ' ')
map(lambda s: s.decode(get_fs_encoding()), cmd)
except Exception as err:
cmd = ''
self.add_to_cache(''.join((str(err), ' <', command, ' >\n')))
if len(cmd) > 0:
prev_stdout = sys.stdout
sys.stdout = self.stdout
try:
# execute command
self.popen_obj = popen = subprocess.Popen(
cmd,
bufsize=0,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
preexec_fn=None,
close_fds=False,
shell=self.shell,
cwd=self.cur_dir.encode(get_fs_encoding()),
universal_newlines=False,
startupinfo=None,
creationflags=0)
popen_stdout_r = popen.stdout.readline
popen_stdout_flush = popen.stdout.flush
txt = popen_stdout_r()
plat = platform()
while txt:
# skip flush on android
if plat[0] != 'a':
popen_stdout_flush()
if isinstance(txt, bytes):
txt = txt.decode(get_fs_encoding())
add_to_cache(txt)
txt = popen_stdout_r()
except (OSError, ValueError) as err:
add_to_cache(''.join(
(str(err),
' < ', command, ' >\n')))
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
sys.stdout = prev_stdout
self.popen_obj = None
Clock.schedule_once(remove_command_interaction_widgets, 0)
# append text to textcache
add_to_cache(self.txtinput_command_line.text + '\n')
command = txtinput_command_line.text[len(self.prompt()):]
if command == '':
self.txtinput_command_line_refocus = True
return
# store command in command_history
if self.command_history_pos > 0:
self.command_history_pos = len(command_history)
if command_history[self.command_history_pos - 1] != command:
command_history.append(command)
else:
command_history.append(command)
len_command_history = len(command_history)
self.command_history_pos = len(command_history)
# on reaching limit(cached_lines) pop first command
if len_command_history >= self.cached_commands:
self.command_history = command_history[1:]
# replace $PATH with
command = os.path.expandvars(command)
if command == 'clear' or command == 'cls':
self.clear()
txtinput_command_line.text = self.prompt()
self.txtinput_command_line_refocus = True
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
return
# if command = cd change directory
if command.startswith('cd ') or command.startswith('export '):
if command[0] == 'e':
e_q = command[7:].find('=')
_exprt = command[7:]
if e_q:
os.environ[_exprt[:e_q]] = _exprt[e_q + 1:]
self.environment = os.environ.copy()
else:
try:
command = re.sub('[ ]+', ' ', command)
if command[3] == os.sep:
os.chdir(command[3:])
else:
os.chdir(os.path.join(self.cur_dir, command[3:]))
if sys.version_info >= (3, 0):
self.cur_dir = os.getcwd()
else:
self.cur_dir = os.getcwdu()
except OSError as err:
Logger.debug('Shell Console: err:' + err.strerror +
' directory:' + str(command[3:]))
add_to_cache(''.join((err.strerror, '\n')))
txtinput_command_line.text = self.prompt()
self.txtinput_command_line_refocus = True
self.command_status = 'closed'
self.dispatch('on_subprocess_done')
return
txtinput_command_line.text = self.prompt()
# store output in textcache
parent = txtinput_command_line.parent
# disable running a new command while and old one is running
parent.remove_widget(txtinput_command_line)
# add widget for interaction with the running command
txtinput_run_command = TextInput(multiline=False,
font_size=self.font_size)
def interact_with_command(*l):
'''Text input to interact with the running command
'''
popen_obj = self.popen_obj
if not popen_obj:
return
txt = l[0].text + '\n'
popen_obj_stdin = popen_obj.stdin
popen_obj_stdin.write(txt)
popen_obj_stdin.flush()
self.txtinput_run_command_refocus = True
self.txtinput_run_command_refocus = False
txtinput_run_command.bind(on_text_validate=interact_with_command)
txtinput_run_command.bind(focus=self.on_focus)
btn_kill = Button(text="Stop",
width=60,
size_hint=(None, 1))
self.interact_layout = il = GridLayout(rows=1, cols=2, height=27,
size_hint=(1, None))
btn_kill.bind(on_press=self.kill_process)
il.add_widget(txtinput_run_command)
il.add_widget(btn_kill)
parent.add_widget(il)
txtinput_run_command.focus = True
self.command_status = 'started'
thread.start_new_thread(run_cmd, ())
0
Example 164
View license def dump_text(self, elem, stylizer):
'''
@elem: The element in the etree that we are working on.
@stylizer: The style information attached to the element.
'''
# We can only processes tags. If there isn't a tag return any text.
if not isinstance(elem.tag, basestring) \
or namespace(elem.tag) != XHTML_NS:
p = elem.getparent()
if p is not None and isinstance(p.tag, basestring) and namespace(p.tag) == XHTML_NS \
and elem.tail:
return [elem.tail]
return ['']
# Setup our variables.
text = ['']
style = stylizer.style(elem)
tags = []
tag = barename(elem.tag)
attribs = elem.attrib
# Ignore anything that is set to not be displayed.
if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
or style['visibility'] == 'hidden':
if hasattr(elem, 'tail') and elem.tail:
return [elem.tail]
return ['']
# Soft scene breaks.
if 'margin-top' in style.cssdict() and style['margin-top'] != 'auto':
ems = min(int(round(float(style.marginTop) / style.fontSize) - 1), self.MAX_EM)
if ems >= 1:
text.append(u'\n\n\xa0' * ems)
if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div'):
if tag == 'div':
tag = 'p'
text.append(self.build_block(tag, style, attribs, stylizer))
text.append('. ')
tags.append('\n')
if style['font-style'] == 'italic' or tag in ('i', 'em'):
if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'cite'):
if self.style_italic == False: # noqa
if self.in_a_link:
text.append('_')
tags.append('_')
else:
text.append('[_')
tags.append('_]')
self.style_embed.append('_')
self.style_italic = True
if style['font-weight'] in ('bold', 'bolder') or tag in ('b', 'strong'):
if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
if self.style_bold == False: # noqa
if self.in_a_link:
text.append('*')
tags.append('*')
else:
text.append('[*')
tags.append('*]')
self.style_embed.append('*')
self.style_bold = True
if style['text-decoration'] == 'underline' or tag in ('u', 'ins'):
if tag != 'a':
if self.style_under == False: # noqa
text.append('[+')
tags.append('+]')
self.style_embed.append('+')
self.style_under = True
if style['text-decoration'] == 'line-through' or tag in ('strike', 'del', 's'):
if self.style_strike == False: # noqa
text.append('[-')
tags.append('-]')
self.style_embed.append('-')
self.style_strike = True
if tag == 'br':
for i in reversed(self.style_embed):
text.append(i)
text.append('\n')
for i in self.style_embed:
text.append(i)
tags.append('')
self.remove_space_after_newline = True
if tag == 'blockquote':
text.append('\nbq. ')
tags.append('\n')
elif tag in ('abbr', 'acronym'):
text.append('')
txt = attribs['title']
tags.append('(' + txt + ')')
elif tag == 'sup':
text.append('^')
tags.append('^')
elif tag == 'sub':
text.append('~')
tags.append('~')
elif tag == 'code':
if self.in_pre:
text.append('\nbc. ')
tags.append('')
else:
text.append('@')
tags.append('@')
elif tag == 'cite':
text.append('??')
tags.append('??')
elif tag == 'hr':
text.append('\n***')
tags.append('\n')
elif tag == 'pre':
self.in_pre = True
text.append('\npre. ')
tags.append('pre\n')
elif tag == 'a':
if self.opts.keep_links:
if attribs.has_key('href'): # noqa
text.append('"')
tags.append('a')
tags.append('":' + attribs['href'])
self.our_links.append(attribs['href'])
if attribs.has_key('title'): # noqa
tags.append('(' + attribs['title'] + ')')
self.in_a_link = True
else:
text.append('%')
tags.append('%')
elif tag == 'img':
if self.opts.keep_image_references:
txt = '!' + self.check_halign(style)
txt += self.check_valign(style)
txt += attribs['src']
text.append(txt)
if attribs.has_key('alt'): # noqa
txt = attribs['alt']
if txt != '':
text.append('(' + txt + ')')
tags.append('!')
elif tag in ('ol', 'ul'):
self.list.append({'name': tag, 'num': 0})
text.append('')
tags.append(tag)
elif tag == 'li':
if self.list:
li = self.list[-1]
else:
li = {'name': 'ul', 'num': 0}
text.append('\n')
if li['name'] == 'ul':
text.append('*' * len(self.list) + ' ')
elif li['name'] == 'ol':
text.append('#' * len(self.list) + ' ')
tags.append('')
elif tag == 'dl':
text.append('\n')
tags.append('')
elif tag == 'dt':
text.append('')
tags.append('\n')
elif tag == 'dd':
text.append(' ')
tags.append('')
elif tag == 'dd':
text.append('')
tags.append('\n')
elif tag == 'table':
txt = self.build_block(tag, style, attribs, stylizer)
txt += '. \n'
if txt != '\ntable. \n':
text.append(txt)
else:
text.append('\n')
tags.append('')
elif tag == 'tr':
txt = self.build_block('', style, attribs, stylizer)
txt += '. '
if txt != '\n. ':
txt = re.sub('\n', '', txt)
text.append(txt)
tags.append('|\n')
elif tag == 'td':
text.append('|')
txt = ''
txt += self.check_halign(style)
txt += self.check_valign(style)
if attribs.has_key('colspan'): # noqa
txt += '\\' + attribs['colspan']
if attribs.has_key('rowspan'): # noqa
txt += '/' + attribs['rowspan']
txt += self.check_styles(style)
if txt != '':
text.append(txt + '. ')
tags.append('')
elif tag == 'th':
text.append('|_. ')
tags.append('')
elif tag == 'span':
if style['font-variant'] == 'small-caps':
if self.style_smallcap == False: # noqa
text.append('&')
tags.append('&')
self.style_smallcap = True
else:
if self.in_a_link == False: # noqa
txt = '%'
if self.opts.keep_links:
txt += self.check_id_tag(attribs)
txt += self.check_styles(style)
if txt != '%':
text.append(txt)
tags.append('%')
if self.opts.keep_links and attribs.has_key('id'): # noqa
if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'span', 'table'):
text.append(self.check_id_tag(attribs))
# Process the styles for any that we want to keep
if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'hr', 'a', 'img',
'span', 'table', 'tr', 'td'):
if not self.in_a_link:
text.append(self.check_styles(style))
# Process tags that contain text.
if hasattr(elem, 'text') and elem.text:
txt = elem.text
if not self.in_pre:
txt = self.prepare_string_for_textile(self.remove_newlines(txt))
text.append(txt)
self.id_no_text = u''
# Recurse down into tags within the tag we are in.
for item in elem:
text += self.dump_text(item, stylizer)
# Close all open tags.
tags.reverse()
for t in tags:
if t in ('pre', 'ul', 'ol', 'li', 'table'):
if t == 'pre':
self.in_pre = False
elif t in ('ul', 'ol'):
if self.list:
self.list.pop()
if not self.list:
text.append('\n')
else:
if t == 'a':
self.in_a_link = False
t = ''
text.append(self.id_no_text)
self.id_no_text = u''
if t in ('*]', '*'):
self.style_bold = False
elif t in ('_]', '_'):
self.style_italic = False
elif t == '+]':
self.style_under = False
elif t == '-]':
self.style_strike = False
elif t == '&':
self.style_smallcap = False
if t in ('*]', '_]', '+]', '-]', '*', '_'):
txt = self.style_embed.pop()
text.append('%s' % t)
# Soft scene breaks.
if 'margin-bottom' in style.cssdict() and style['margin-bottom'] != 'auto':
ems = min(int(round((float(style.marginBottom) / style.fontSize) - 1)), self.MAX_EM)
if ems >= 1:
text.append(u'\n\n\xa0' * ems)
# Add the text that is outside of the tag.
if hasattr(elem, 'tail') and elem.tail:
tail = elem.tail
if not self.in_pre:
tail = self.prepare_string_for_textile(self.remove_newlines(tail))
text.append(tail)
return text
0
Example 165
View license def dump_text(self, elem, stylizer):
'''
@elem: The element in the etree that we are working on.
@stylizer: The style information attached to the element.
'''
# We can only processes tags. If there isn't a tag return any text.
if not isinstance(elem.tag, basestring) \
or namespace(elem.tag) != XHTML_NS:
p = elem.getparent()
if p is not None and isinstance(p.tag, basestring) and namespace(p.tag) == XHTML_NS \
and elem.tail:
return [elem.tail]
return ['']
# Setup our variables.
text = ['']
style = stylizer.style(elem)
tags = []
tag = barename(elem.tag)
attribs = elem.attrib
# Ignore anything that is set to not be displayed.
if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
or style['visibility'] == 'hidden':
if hasattr(elem, 'tail') and elem.tail:
return [elem.tail]
return ['']
# Soft scene breaks.
if 'margin-top' in style.cssdict() and style['margin-top'] != 'auto':
ems = min(int(round(float(style.marginTop) / style.fontSize) - 1), self.MAX_EM)
if ems >= 1:
text.append(u'\n\n\xa0' * ems)
if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div'):
if tag == 'div':
tag = 'p'
text.append(self.build_block(tag, style, attribs, stylizer))
text.append('. ')
tags.append('\n')
if style['font-style'] == 'italic' or tag in ('i', 'em'):
if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'cite'):
if self.style_italic == False: # noqa
if self.in_a_link:
text.append('_')
tags.append('_')
else:
text.append('[_')
tags.append('_]')
self.style_embed.append('_')
self.style_italic = True
if style['font-weight'] in ('bold', 'bolder') or tag in ('b', 'strong'):
if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
if self.style_bold == False: # noqa
if self.in_a_link:
text.append('*')
tags.append('*')
else:
text.append('[*')
tags.append('*]')
self.style_embed.append('*')
self.style_bold = True
if style['text-decoration'] == 'underline' or tag in ('u', 'ins'):
if tag != 'a':
if self.style_under == False: # noqa
text.append('[+')
tags.append('+]')
self.style_embed.append('+')
self.style_under = True
if style['text-decoration'] == 'line-through' or tag in ('strike', 'del', 's'):
if self.style_strike == False: # noqa
text.append('[-')
tags.append('-]')
self.style_embed.append('-')
self.style_strike = True
if tag == 'br':
for i in reversed(self.style_embed):
text.append(i)
text.append('\n')
for i in self.style_embed:
text.append(i)
tags.append('')
self.remove_space_after_newline = True
if tag == 'blockquote':
text.append('\nbq. ')
tags.append('\n')
elif tag in ('abbr', 'acronym'):
text.append('')
txt = attribs['title']
tags.append('(' + txt + ')')
elif tag == 'sup':
text.append('^')
tags.append('^')
elif tag == 'sub':
text.append('~')
tags.append('~')
elif tag == 'code':
if self.in_pre:
text.append('\nbc. ')
tags.append('')
else:
text.append('@')
tags.append('@')
elif tag == 'cite':
text.append('??')
tags.append('??')
elif tag == 'hr':
text.append('\n***')
tags.append('\n')
elif tag == 'pre':
self.in_pre = True
text.append('\npre. ')
tags.append('pre\n')
elif tag == 'a':
if self.opts.keep_links:
if attribs.has_key('href'): # noqa
text.append('"')
tags.append('a')
tags.append('":' + attribs['href'])
self.our_links.append(attribs['href'])
if attribs.has_key('title'): # noqa
tags.append('(' + attribs['title'] + ')')
self.in_a_link = True
else:
text.append('%')
tags.append('%')
elif tag == 'img':
if self.opts.keep_image_references:
txt = '!' + self.check_halign(style)
txt += self.check_valign(style)
txt += attribs['src']
text.append(txt)
if attribs.has_key('alt'): # noqa
txt = attribs['alt']
if txt != '':
text.append('(' + txt + ')')
tags.append('!')
elif tag in ('ol', 'ul'):
self.list.append({'name': tag, 'num': 0})
text.append('')
tags.append(tag)
elif tag == 'li':
if self.list:
li = self.list[-1]
else:
li = {'name': 'ul', 'num': 0}
text.append('\n')
if li['name'] == 'ul':
text.append('*' * len(self.list) + ' ')
elif li['name'] == 'ol':
text.append('#' * len(self.list) + ' ')
tags.append('')
elif tag == 'dl':
text.append('\n')
tags.append('')
elif tag == 'dt':
text.append('')
tags.append('\n')
elif tag == 'dd':
text.append(' ')
tags.append('')
elif tag == 'dd':
text.append('')
tags.append('\n')
elif tag == 'table':
txt = self.build_block(tag, style, attribs, stylizer)
txt += '. \n'
if txt != '\ntable. \n':
text.append(txt)
else:
text.append('\n')
tags.append('')
elif tag == 'tr':
txt = self.build_block('', style, attribs, stylizer)
txt += '. '
if txt != '\n. ':
txt = re.sub('\n', '', txt)
text.append(txt)
tags.append('|\n')
elif tag == 'td':
text.append('|')
txt = ''
txt += self.check_halign(style)
txt += self.check_valign(style)
if attribs.has_key('colspan'): # noqa
txt += '\\' + attribs['colspan']
if attribs.has_key('rowspan'): # noqa
txt += '/' + attribs['rowspan']
txt += self.check_styles(style)
if txt != '':
text.append(txt + '. ')
tags.append('')
elif tag == 'th':
text.append('|_. ')
tags.append('')
elif tag == 'span':
if style['font-variant'] == 'small-caps':
if self.style_smallcap == False: # noqa
text.append('&')
tags.append('&')
self.style_smallcap = True
else:
if self.in_a_link == False: # noqa
txt = '%'
if self.opts.keep_links:
txt += self.check_id_tag(attribs)
txt += self.check_styles(style)
if txt != '%':
text.append(txt)
tags.append('%')
if self.opts.keep_links and attribs.has_key('id'): # noqa
if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'span', 'table'):
text.append(self.check_id_tag(attribs))
# Process the styles for any that we want to keep
if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'hr', 'a', 'img',
'span', 'table', 'tr', 'td'):
if not self.in_a_link:
text.append(self.check_styles(style))
# Process tags that contain text.
if hasattr(elem, 'text') and elem.text:
txt = elem.text
if not self.in_pre:
txt = self.prepare_string_for_textile(self.remove_newlines(txt))
text.append(txt)
self.id_no_text = u''
# Recurse down into tags within the tag we are in.
for item in elem:
text += self.dump_text(item, stylizer)
# Close all open tags.
tags.reverse()
for t in tags:
if t in ('pre', 'ul', 'ol', 'li', 'table'):
if t == 'pre':
self.in_pre = False
elif t in ('ul', 'ol'):
if self.list:
self.list.pop()
if not self.list:
text.append('\n')
else:
if t == 'a':
self.in_a_link = False
t = ''
text.append(self.id_no_text)
self.id_no_text = u''
if t in ('*]', '*'):
self.style_bold = False
elif t in ('_]', '_'):
self.style_italic = False
elif t == '+]':
self.style_under = False
elif t == '-]':
self.style_strike = False
elif t == '&':
self.style_smallcap = False
if t in ('*]', '_]', '+]', '-]', '*', '_'):
txt = self.style_embed.pop()
text.append('%s' % t)
# Soft scene breaks.
if 'margin-bottom' in style.cssdict() and style['margin-bottom'] != 'auto':
ems = min(int(round((float(style.marginBottom) / style.fontSize) - 1)), self.MAX_EM)
if ems >= 1:
text.append(u'\n\n\xa0' * ems)
# Add the text that is outside of the tag.
if hasattr(elem, 'tail') and elem.tail:
tail = elem.tail
if not self.in_pre:
tail = self.prepare_string_for_textile(self.remove_newlines(tail))
text.append(tail)
return text
0
Example 166
View license@flask_login.login_required
def update_micropub_syndicate_to():
def adapt_expanded(targets):
"""Backcompat support for old-style "syndicate-to-expanded" properties,
e.g.,
{
"id": "twitter::kylewmahan",
"name": "@kylewmahan",
"service": "Twitter"
}
"""
if targets:
return [{
'uid': t.get('id'),
'name': '{} on {}'.format(t.get('name'), t.get('service')),
} for t in targets]
return targets
endpt = flask_login.current_user.micropub_endpoint
token = flask_login.current_user.access_token
if not endpt or not token:
return
resp = util.requests_get(endpt, params={
'q': 'syndicate-to',
}, headers={
'Authorization': 'Bearer ' + token,
'Accept': 'application/json',
})
if resp.status_code // 100 != 2:
flask.current_app.logger.warn(
'Unexpected response querying micropub endpoint %s: %s',
resp, resp.text)
return
flask.current_app.logger.debug('syndicate-to response: %s %s',
resp, resp.text)
content_type = resp.headers['content-type']
if content_type:
content_type = content_type.split(';', 1)[0]
try:
if content_type == 'application/json':
blob = resp.json()
syndicate_tos = adapt_expanded(blob.get('syndicate-to-expanded'))
if not syndicate_tos:
syndicate_tos = blob.get('syndicate-to')
else: # try to parse query string
syndicate_tos = pyquerystring.parse(resp.text).get('syndicate-to', [])
if isinstance(syndicate_tos, list):
syndicate_tos = list(syndicate_tos)
flask_login.current_user.set_setting('syndicate-to', syndicate_tos)
db.session.commit()
except ValueError as e:
flask.flash('Could not parse syndicate-to response: {}'.format(e)
@views.route('/deauthorize')
@flask_login.login_required
def deauthorize():
flask_login.current_user.micropub_endpoint = None
flask_login.current_user.access_token = None
db.session.commit()
return flask.redirect(flask.request.args.get('next')
or flask.url_for('.index'))
@login_mgr.user_loader
def load_user(url):
alt = url.rstrip('/') if url.endswith('/') else url + '/'
return User.query.filter(
(User.url == url) | (User.url == alt)).first()
@views.route('/subscribe', methods=['GET', 'POST'])
@flask_login.login_required
def subscribe():
origin = (flask.request.form.get('origin')
or flask.request.args.get('origin'))
if origin:
type = None
feed = None
typed_feed = flask.request.form.get('feed')
if typed_feed:
type, feed = typed_feed.split('|', 1)
else:
feeds = find_possible_feeds(origin)
if not feeds:
flask.flash('No feeds found for: ' + origin)
return flask.redirect(flask.url_for('.index'))
if len(feeds) > 1:
return flask.render_template(
'select-feed.jinja2', origin=origin, feeds=feeds)
feed = feeds[0]['feed']
type = feeds[0]['type']
new_feed = add_subscription(origin, feed, type)
flask.flash('Successfully subscribed to: {}'.format(new_feed.name))
return flask.redirect(flask.url_for('.index'))
if flask.request.method == 'POST':
flask.abort(400)
return flask.render_template('subscribe.jinja2')
def add_subscription(origin, feed_url, type, tags=None):
feed = Feed.query.filter_by(feed=feed_url, type=type).first()
if not feed:
name = None
if type == 'html':
flask.current_app.logger.debug('mf2py parsing %s', feed_url)
resp = util.requests_get(feed_url)
feed_text = resp.text if 'charset' in resp.headers.get('content-type', '') else resp.content
parsed = mf2util.interpret_feed(
mf2py.parse(doc=feed_text, url=feed_url), feed_url)
name = parsed.get('name')
elif type == 'xml':
flask.current_app.logger.debug('feedparser parsing %s', feed_url)
parsed = feedparser.parse(feed_url, agent=util.USER_AGENT)
if parsed.feed:
name = parsed.feed.get('title')
else:
flask.current_app.logger.error('unknown feed type %s', type)
flask.abort(400)
if not name:
p = urllib.parse.urlparse(origin)
name = p.netloc + p.path
feed = Feed(name=name[:140], origin=origin, feed=feed_url, type=type)
if feed:
db.session.add(feed)
flask_login.current_user.subscriptions.append(
Subscription(feed=feed, name=feed.name, tags=tags))
db.session.commit()
# go ahead and update the fed
tasks.q.enqueue(tasks.update_feed, feed.id)
return feed
def find_possible_feeds(origin):
# scrape an origin source to find possible alternative feeds
try:
resp = util.requests_get(origin)
except requests.exceptions.RequestException as e:
flask.flash('Error fetching source {}'.format(repr(e)))
flask.current_app.logger.warn(
'Subscribe failed for %s with error %s', origin, repr(e))
return None
feeds = []
xml_feed_types = [
'application/rss+xml',
'application/atom+xml',
'application/rdf+xml',
'application/xml',
'text/xml',
]
xml_mime_types = xml_feed_types + [
'text/xml',
'text/rss+xml',
'text/atom+xml',
]
html_feed_types = [
'text/html',
'application/xhtml+xml',
]
content_type = resp.headers['content-type']
content_type = content_type.split(';', 1)[0].strip()
if content_type in xml_mime_types:
feeds.append({
'origin': origin,
'feed': origin,
'type': 'xml',
'title': 'untitled xml feed',
})
elif content_type in html_feed_types:
parsed = mf2py.parse(doc=resp.text, url=origin)
# if text/html, then parse and look for h-entries
hfeed = mf2util.interpret_feed(parsed, origin)
if hfeed.get('entries'):
ftitle = hfeed.get('name') or 'untitled h-feed'
feeds.append({
'origin': origin,
'feed': resp.url,
'type': 'html',
'title': ftitle[:140]
})
# look for link="feed"
for furl in parsed.get('rels', {}).get('feed', []):
fprops = parsed.get('rel-urls', {}).get(furl, {})
if not fprops.get('type') or fprops.get('type') in html_feed_types:
feeds.append({
'origin': origin,
'feed': furl,
'type': 'html',
'title': fprops.get('title'),
})
# then look for link rel="alternate"
for link in parsed.get('alternates', []):
if link.get('type') in xml_feed_types:
feeds.append({
'origin': origin,
'feed': link.get('url'),
'type': 'xml',
'title': link.get('title'),
})
return feeds
@views.app_template_filter()
def prettify_url(url):
parsed = urllib.parse.urlparse(url)
if parsed.path:
return parsed.netloc + parsed.path
return parsed.netloc
@views.app_template_filter()
def domain_for_url(url):
parsed = urllib.parse.urlparse(url)
return parsed.netloc
@views.app_template_filter()
def favicon_for_url(url):
return '//www.google.com/s2/favicons?' + urllib.parse.urlencode({
'domain': url,
})
@views.app_template_filter()
def relative_time(dt):
if dt:
now = datetime.datetime.utcnow()
diff = now - dt
zero = datetime.timedelta(0)
if diff == zero:
pretty = 'Right now'
elif diff > zero:
years = diff.days // 365
hours = diff.seconds // 60 // 60
minutes = diff.seconds // 60
if years > 1:
pretty = str(years) + ' years ago'
elif diff.days == 1:
pretty = 'A day ago'
elif diff.days > 1:
pretty = str(diff.days) + ' days ago'
elif hours == 1:
pretty = 'An hour ago'
elif hours > 1:
pretty = str(hours) + ' hours ago'
elif minutes == 1:
pretty = 'A minute ago'
elif minutes > 1:
pretty = str(minutes) + ' minutes ago'
else:
pretty = str(diff.seconds) + ' seconds ago'
else:
diff = abs(diff)
years = diff.days // 365
hours = diff.seconds // 60 // 60
minutes = diff.seconds // 60
if years > 1:
pretty = str(years) + ' years from now'
elif diff.days == 1:
pretty = 'A day from now'
elif diff.days > 1:
pretty = str(diff.days) + ' days from now'
elif hours == 1:
pretty = 'An hour from now'
elif hours > 1:
pretty = str(hours) + ' hours from now'
elif minutes == 1:
pretty = 'A minute from now'
elif minutes > 1:
pretty = str(minutes) + ' minutes from now'
else:
pretty = str(diff.seconds) + ' seconds from now'
return '<time datetime="{}">{}</time>'.format(dt.isoformat(), pretty)
@views.app_template_filter()
def isoformat(dt):
return dt and dt.isoformat()
@views.app_template_filter()
def add_preview(content):
"""If a post ends with the URL of a known media source (youtube,
instagram, etc.), add the content inline.
"""
if not content or any('<' + tag in content for tag in (
'img', 'iframe', 'embed', 'audio', 'video')):
# don't add a preview to a post that already has one
return content
# flatten links and strip tags
flat = content
flat = re.sub(r'<a [^>]*href="([^"]+)"[^>]*>[^<]*</a>', r'\1', flat)
flat = re.sub(r'</?\w+[^>]*>', '', flat)
flat = flat.strip()
instagram_regex = r'https?://(?:www\.)?instagram.com/p/[\w\-]+/?'
vimeo_regex = r'https?://(?:www\.)?vimeo.com/(\d+)/?'
youtube_regex = r'https?://(?:www\.)?youtube.com/watch\?v=([\w\-]+)'
youtube_short_regex = r'https://youtu.be/([\w\-]+)'
twitter_regex = r'https?://(?:www\.)?twitter.com/(\w+)/status/(\d+)'
m = re.search(instagram_regex, flat)
if m:
ig_url = m.group(0)
media_url = urllib.parse.urljoin(ig_url, 'media/?size=l')
return '{}<a href="{}"><img src="{}" /></a>'.format(
content, ig_url, media_url)
m = re.search(vimeo_regex, flat)
if m:
# vimeo_url = m.group(0)
vimeo_id = m.group(1)
return (
'{}<iframe src="//player.vimeo.com/video/{}" width="560" '
'height="315" frameborder="0" webkitallowfullscreen '
'mozallowfullscreen allowfullscreen></iframe>'
).format(content, vimeo_id)
m = re.search(youtube_regex, flat)
if not m:
m = re.search(youtube_short_regex, content)
if m:
youtube_id = m.group(1)
return (
'{}<iframe width="560" height="315" '
'src="https://www.youtube.com/embed/{}" frameborder="0" '
'allowfullscreen></iframe>'
).format(content, youtube_id)
m = re.search(twitter_regex + '$', flat)
if m:
tweet_url = m.group()
return content + (
'<blockquote class="twitter-tweet" lang="en" data-cards="hidden">'
'<a href="{}"></a></blockquote>'
).format(tweet_url)
return content
@views.app_template_filter()
def proxy_image(url):
proxy_url = flask.current_app.config.get('IMAGEPROXY_URL')
proxy_key = flask.current_app.config.get('IMAGEPROXY_KEY')
if proxy_url and proxy_key:
sig = base64.urlsafe_b64encode(
hmac.new(proxy_key.encode(), url.encode(), hashlib.sha256).digest()
).decode()
return '/'.join((proxy_url.rstrip('/'), 's' + sig, url))
pilbox_url = flask.current_app.config.get('PILBOX_URL')
pilbox_key = flask.current_app.config.get('PILBOX_KEY')
if pilbox_url and pilbox_key:
query = urllib.parse.urlencode({'url': url, 'op': 'noop'})
sig = hmac.new(pilbox_key.encode(), query.encode(), hashlib.sha1).hexdigest()
query += '&sig=' + sig
return pilbox_url + '?' + query
camo_url = flask.current_app.config.get('CAMO_URL')
camo_key = flask.current_app.config.get('CAMO_KEY')
if camo_url and camo_key:
digest = hmac.new(camo_key.encode(), url.encode(), hashlib.sha1).hexdigest()
return (urllib.parse.urljoin(camo_url, digest)
+ '?url=' + urllib.parse.quote_plus(url))
return url
@views.app_template_filter()
def proxy_all(content):
def repl(m):
attrs = m.group(1)
url = m.group(2)
url = url.replace('&', '&')
return '<img{} src="{}"'.format(attrs, proxy_image(url))
if content:
return IMAGE_TAG_RE.sub(repl, content)
@views.app_template_global()
def url_for_other_page(page):
"""http://flask.pocoo.org/snippets/44/#URL+Generation+Helper
"""
args = flask.request.view_args.copy()
args.update(flask.request.args)
args['page'] = page
return flask.url_for(flask.request.endpoint, **args)
def dedupe_copies(entries):
all_copies = set()
for entry in entries:
syndurls = entry.get_property('syndication')
if syndurls:
copies = [e for e in entries if e.permalink in syndurls]
entry._syndicated_copies = copies
all_copies.update(copies)
return [e for e in entries if e not in all_copies]
def font_awesome_class_for_service(service):
service = service.lower()
if service == 'facebook':
return 'fa fa-facebook'
if service == 'twitter':
return 'fa fa-twitter'
if service == 'instagram':
return 'fa fa-instagram'
if service == 'flickr':
return 'fa fa-flickr'
if service == 'googleplus' or service == 'g+' or service == 'google plus' or service == 'google+':
return 'fa fa-google-plus'
if service == 'hacker news' or service == 'hackernews':
return 'fa fa-hacker-news'
if service == 'indienews':
return 'fa fa-newspaper-o'
if service == 'linkedin':
return 'fa fa-linkedin'
if service == 'foursquare' or service == 'swarm':
return 'fa fa-foursquare'
return 'fa fa-send'
@views.app_template_filter('render_syndication_target_id')
def render_syndication_target_id(target):
if isinstance(target, dict):
id = target.get('uid') or target.get('id')
else:
id = target
return util.html_escape(id)
@views.app_template_filter('render_syndication_target')
def render_syndication_target(target):
if isinstance(target, dict):
full_name = target.get('name')
return util.html_escape(full_name)
return util.html_escape(prettify_url(target))
@views.app_template_test('syndicated_to')
def is_syndicated_to(entry, target):
def same_domain(u1, u2):
return domain_for_url(u1) == domain_for_url(u2)
if isinstance(target, dict):
return False # TODO
return same_domain(entry.permalink, target) or any(
same_domain(syndurl, target)
for syndurl in entry.get_property('syndication', []))
0
Example 167
View license@flask_login.login_required
def update_micropub_syndicate_to():
def adapt_expanded(targets):
"""Backcompat support for old-style "syndicate-to-expanded" properties,
e.g.,
{
"id": "twitter::kylewmahan",
"name": "@kylewmahan",
"service": "Twitter"
}
"""
if targets:
return [{
'uid': t.get('id'),
'name': '{} on {}'.format(t.get('name'), t.get('service')),
} for t in targets]
return targets
endpt = flask_login.current_user.micropub_endpoint
token = flask_login.current_user.access_token
if not endpt or not token:
return
resp = util.requests_get(endpt, params={
'q': 'syndicate-to',
}, headers={
'Authorization': 'Bearer ' + token,
'Accept': 'application/json',
})
if resp.status_code // 100 != 2:
flask.current_app.logger.warn(
'Unexpected response querying micropub endpoint %s: %s',
resp, resp.text)
return
flask.current_app.logger.debug('syndicate-to response: %s %s',
resp, resp.text)
content_type = resp.headers['content-type']
if content_type:
content_type = content_type.split(';', 1)[0]
try:
if content_type == 'application/json':
blob = resp.json()
syndicate_tos = adapt_expanded(blob.get('syndicate-to-expanded'))
if not syndicate_tos:
syndicate_tos = blob.get('syndicate-to')
else: # try to parse query string
syndicate_tos = pyquerystring.parse(resp.text).get('syndicate-to', [])
if isinstance(syndicate_tos, list):
syndicate_tos = list(syndicate_tos)
flask_login.current_user.set_setting('syndicate-to', syndicate_tos)
db.session.commit()
except ValueError as e:
flask.flash('Could not parse syndicate-to response: {}'.format(e)
@views.route('/deauthorize')
@flask_login.login_required
def deauthorize():
flask_login.current_user.micropub_endpoint = None
flask_login.current_user.access_token = None
db.session.commit()
return flask.redirect(flask.request.args.get('next')
or flask.url_for('.index'))
@login_mgr.user_loader
def load_user(url):
alt = url.rstrip('/') if url.endswith('/') else url + '/'
return User.query.filter(
(User.url == url) | (User.url == alt)).first()
@views.route('/subscribe', methods=['GET', 'POST'])
@flask_login.login_required
def subscribe():
origin = (flask.request.form.get('origin')
or flask.request.args.get('origin'))
if origin:
type = None
feed = None
typed_feed = flask.request.form.get('feed')
if typed_feed:
type, feed = typed_feed.split('|', 1)
else:
feeds = find_possible_feeds(origin)
if not feeds:
flask.flash('No feeds found for: ' + origin)
return flask.redirect(flask.url_for('.index'))
if len(feeds) > 1:
return flask.render_template(
'select-feed.jinja2', origin=origin, feeds=feeds)
feed = feeds[0]['feed']
type = feeds[0]['type']
new_feed = add_subscription(origin, feed, type)
flask.flash('Successfully subscribed to: {}'.format(new_feed.name))
return flask.redirect(flask.url_for('.index'))
if flask.request.method == 'POST':
flask.abort(400)
return flask.render_template('subscribe.jinja2')
def add_subscription(origin, feed_url, type, tags=None):
feed = Feed.query.filter_by(feed=feed_url, type=type).first()
if not feed:
name = None
if type == 'html':
flask.current_app.logger.debug('mf2py parsing %s', feed_url)
resp = util.requests_get(feed_url)
feed_text = resp.text if 'charset' in resp.headers.get('content-type', '') else resp.content
parsed = mf2util.interpret_feed(
mf2py.parse(doc=feed_text, url=feed_url), feed_url)
name = parsed.get('name')
elif type == 'xml':
flask.current_app.logger.debug('feedparser parsing %s', feed_url)
parsed = feedparser.parse(feed_url, agent=util.USER_AGENT)
if parsed.feed:
name = parsed.feed.get('title')
else:
flask.current_app.logger.error('unknown feed type %s', type)
flask.abort(400)
if not name:
p = urllib.parse.urlparse(origin)
name = p.netloc + p.path
feed = Feed(name=name[:140], origin=origin, feed=feed_url, type=type)
if feed:
db.session.add(feed)
flask_login.current_user.subscriptions.append(
Subscription(feed=feed, name=feed.name, tags=tags))
db.session.commit()
# go ahead and update the fed
tasks.q.enqueue(tasks.update_feed, feed.id)
return feed
def find_possible_feeds(origin):
# scrape an origin source to find possible alternative feeds
try:
resp = util.requests_get(origin)
except requests.exceptions.RequestException as e:
flask.flash('Error fetching source {}'.format(repr(e)))
flask.current_app.logger.warn(
'Subscribe failed for %s with error %s', origin, repr(e))
return None
feeds = []
xml_feed_types = [
'application/rss+xml',
'application/atom+xml',
'application/rdf+xml',
'application/xml',
'text/xml',
]
xml_mime_types = xml_feed_types + [
'text/xml',
'text/rss+xml',
'text/atom+xml',
]
html_feed_types = [
'text/html',
'application/xhtml+xml',
]
content_type = resp.headers['content-type']
content_type = content_type.split(';', 1)[0].strip()
if content_type in xml_mime_types:
feeds.append({
'origin': origin,
'feed': origin,
'type': 'xml',
'title': 'untitled xml feed',
})
elif content_type in html_feed_types:
parsed = mf2py.parse(doc=resp.text, url=origin)
# if text/html, then parse and look for h-entries
hfeed = mf2util.interpret_feed(parsed, origin)
if hfeed.get('entries'):
ftitle = hfeed.get('name') or 'untitled h-feed'
feeds.append({
'origin': origin,
'feed': resp.url,
'type': 'html',
'title': ftitle[:140]
})
# look for link="feed"
for furl in parsed.get('rels', {}).get('feed', []):
fprops = parsed.get('rel-urls', {}).get(furl, {})
if not fprops.get('type') or fprops.get('type') in html_feed_types:
feeds.append({
'origin': origin,
'feed': furl,
'type': 'html',
'title': fprops.get('title'),
})
# then look for link rel="alternate"
for link in parsed.get('alternates', []):
if link.get('type') in xml_feed_types:
feeds.append({
'origin': origin,
'feed': link.get('url'),
'type': 'xml',
'title': link.get('title'),
})
return feeds
@views.app_template_filter()
def prettify_url(url):
parsed = urllib.parse.urlparse(url)
if parsed.path:
return parsed.netloc + parsed.path
return parsed.netloc
@views.app_template_filter()
def domain_for_url(url):
parsed = urllib.parse.urlparse(url)
return parsed.netloc
@views.app_template_filter()
def favicon_for_url(url):
return '//www.google.com/s2/favicons?' + urllib.parse.urlencode({
'domain': url,
})
@views.app_template_filter()
def relative_time(dt):
if dt:
now = datetime.datetime.utcnow()
diff = now - dt
zero = datetime.timedelta(0)
if diff == zero:
pretty = 'Right now'
elif diff > zero:
years = diff.days // 365
hours = diff.seconds // 60 // 60
minutes = diff.seconds // 60
if years > 1:
pretty = str(years) + ' years ago'
elif diff.days == 1:
pretty = 'A day ago'
elif diff.days > 1:
pretty = str(diff.days) + ' days ago'
elif hours == 1:
pretty = 'An hour ago'
elif hours > 1:
pretty = str(hours) + ' hours ago'
elif minutes == 1:
pretty = 'A minute ago'
elif minutes > 1:
pretty = str(minutes) + ' minutes ago'
else:
pretty = str(diff.seconds) + ' seconds ago'
else:
diff = abs(diff)
years = diff.days // 365
hours = diff.seconds // 60 // 60
minutes = diff.seconds // 60
if years > 1:
pretty = str(years) + ' years from now'
elif diff.days == 1:
pretty = 'A day from now'
elif diff.days > 1:
pretty = str(diff.days) + ' days from now'
elif hours == 1:
pretty = 'An hour from now'
elif hours > 1:
pretty = str(hours) + ' hours from now'
elif minutes == 1:
pretty = 'A minute from now'
elif minutes > 1:
pretty = str(minutes) + ' minutes from now'
else:
pretty = str(diff.seconds) + ' seconds from now'
return '<time datetime="{}">{}</time>'.format(dt.isoformat(), pretty)
@views.app_template_filter()
def isoformat(dt):
return dt and dt.isoformat()
@views.app_template_filter()
def add_preview(content):
"""If a post ends with the URL of a known media source (youtube,
instagram, etc.), add the content inline.
"""
if not content or any('<' + tag in content for tag in (
'img', 'iframe', 'embed', 'audio', 'video')):
# don't add a preview to a post that already has one
return content
# flatten links and strip tags
flat = content
flat = re.sub(r'<a [^>]*href="([^"]+)"[^>]*>[^<]*</a>', r'\1', flat)
flat = re.sub(r'</?\w+[^>]*>', '', flat)
flat = flat.strip()
instagram_regex = r'https?://(?:www\.)?instagram.com/p/[\w\-]+/?'
vimeo_regex = r'https?://(?:www\.)?vimeo.com/(\d+)/?'
youtube_regex = r'https?://(?:www\.)?youtube.com/watch\?v=([\w\-]+)'
youtube_short_regex = r'https://youtu.be/([\w\-]+)'
twitter_regex = r'https?://(?:www\.)?twitter.com/(\w+)/status/(\d+)'
m = re.search(instagram_regex, flat)
if m:
ig_url = m.group(0)
media_url = urllib.parse.urljoin(ig_url, 'media/?size=l')
return '{}<a href="{}"><img src="{}" /></a>'.format(
content, ig_url, media_url)
m = re.search(vimeo_regex, flat)
if m:
# vimeo_url = m.group(0)
vimeo_id = m.group(1)
return (
'{}<iframe src="//player.vimeo.com/video/{}" width="560" '
'height="315" frameborder="0" webkitallowfullscreen '
'mozallowfullscreen allowfullscreen></iframe>'
).format(content, vimeo_id)
m = re.search(youtube_regex, flat)
if not m:
m = re.search(youtube_short_regex, content)
if m:
youtube_id = m.group(1)
return (
'{}<iframe width="560" height="315" '
'src="https://www.youtube.com/embed/{}" frameborder="0" '
'allowfullscreen></iframe>'
).format(content, youtube_id)
m = re.search(twitter_regex + '$', flat)
if m:
tweet_url = m.group()
return content + (
'<blockquote class="twitter-tweet" lang="en" data-cards="hidden">'
'<a href="{}"></a></blockquote>'
).format(tweet_url)
return content
@views.app_template_filter()
def proxy_image(url):
proxy_url = flask.current_app.config.get('IMAGEPROXY_URL')
proxy_key = flask.current_app.config.get('IMAGEPROXY_KEY')
if proxy_url and proxy_key:
sig = base64.urlsafe_b64encode(
hmac.new(proxy_key.encode(), url.encode(), hashlib.sha256).digest()
).decode()
return '/'.join((proxy_url.rstrip('/'), 's' + sig, url))
pilbox_url = flask.current_app.config.get('PILBOX_URL')
pilbox_key = flask.current_app.config.get('PILBOX_KEY')
if pilbox_url and pilbox_key:
query = urllib.parse.urlencode({'url': url, 'op': 'noop'})
sig = hmac.new(pilbox_key.encode(), query.encode(), hashlib.sha1).hexdigest()
query += '&sig=' + sig
return pilbox_url + '?' + query
camo_url = flask.current_app.config.get('CAMO_URL')
camo_key = flask.current_app.config.get('CAMO_KEY')
if camo_url and camo_key:
digest = hmac.new(camo_key.encode(), url.encode(), hashlib.sha1).hexdigest()
return (urllib.parse.urljoin(camo_url, digest)
+ '?url=' + urllib.parse.quote_plus(url))
return url
@views.app_template_filter()
def proxy_all(content):
def repl(m):
attrs = m.group(1)
url = m.group(2)
url = url.replace('&', '&')
return '<img{} src="{}"'.format(attrs, proxy_image(url))
if content:
return IMAGE_TAG_RE.sub(repl, content)
@views.app_template_global()
def url_for_other_page(page):
"""http://flask.pocoo.org/snippets/44/#URL+Generation+Helper
"""
args = flask.request.view_args.copy()
args.update(flask.request.args)
args['page'] = page
return flask.url_for(flask.request.endpoint, **args)
def dedupe_copies(entries):
all_copies = set()
for entry in entries:
syndurls = entry.get_property('syndication')
if syndurls:
copies = [e for e in entries if e.permalink in syndurls]
entry._syndicated_copies = copies
all_copies.update(copies)
return [e for e in entries if e not in all_copies]
def font_awesome_class_for_service(service):
service = service.lower()
if service == 'facebook':
return 'fa fa-facebook'
if service == 'twitter':
return 'fa fa-twitter'
if service == 'instagram':
return 'fa fa-instagram'
if service == 'flickr':
return 'fa fa-flickr'
if service == 'googleplus' or service == 'g+' or service == 'google plus' or service == 'google+':
return 'fa fa-google-plus'
if service == 'hacker news' or service == 'hackernews':
return 'fa fa-hacker-news'
if service == 'indienews':
return 'fa fa-newspaper-o'
if service == 'linkedin':
return 'fa fa-linkedin'
if service == 'foursquare' or service == 'swarm':
return 'fa fa-foursquare'
return 'fa fa-send'
@views.app_template_filter('render_syndication_target_id')
def render_syndication_target_id(target):
if isinstance(target, dict):
id = target.get('uid') or target.get('id')
else:
id = target
return util.html_escape(id)
@views.app_template_filter('render_syndication_target')
def render_syndication_target(target):
if isinstance(target, dict):
full_name = target.get('name')
return util.html_escape(full_name)
return util.html_escape(prettify_url(target))
@views.app_template_test('syndicated_to')
def is_syndicated_to(entry, target):
def same_domain(u1, u2):
return domain_for_url(u1) == domain_for_url(u2)
if isinstance(target, dict):
return False # TODO
return same_domain(entry.permalink, target) or any(
same_domain(syndurl, target)
for syndurl in entry.get_property('syndication', []))
0
Example 168
View licensedef parse(input_filename, output_filename):
"Feed it a file, and it'll output a fixed one"
# State storage
if input_filename == "-":
num_lines = -1
else:
num_lines = int(subprocess.check_output(["wc", "-l", input_filename]).strip().split()[0])
tables = {}
current_table = None
creation_lines = []
enum_types = []
foreign_key_lines = []
fulltext_key_lines = []
sequence_lines = []
cast_lines = []
num_inserts = 0
started = time.time()
# Open output file and write header. Logging file handle will be stdout
# unless we're writing output to stdout, in which case NO PROGRESS FOR YOU.
if output_filename == "-":
output = sys.stdout
logging = open(os.devnull, "w")
else:
output = open(output_filename, "w")
logging = sys.stdout
if input_filename == "-":
input_fh = sys.stdin
else:
input_fh = open(input_filename)
output.write("-- Converted by db_converter\n")
output.write("START TRANSACTION;\n")
output.write("SET standard_conforming_strings=off;\n")
output.write("SET escape_string_warning=off;\n")
output.write("SET CONSTRAINTS ALL DEFERRED;\n\n")
for i, line in enumerate(input_fh):
time_taken = time.time() - started
percentage_done = (i+1) / float(num_lines)
secs_left = (time_taken / percentage_done) - time_taken
logging.write("\rLine %i (of %s: %.2f%%) [%s tables] [%s inserts] [ETA: %i min %i sec]" % (
i + 1,
num_lines,
((i+1)/float(num_lines))*100,
len(tables),
num_inserts,
secs_left // 60,
secs_left % 60,
))
logging.flush()
line = line.decode("utf8").strip().replace(r"\\", "WUBWUBREALSLASHWUB").replace(r"\'", "''").replace("WUBWUBREALSLASHWUB", r"\\")
# Ignore comment lines
if line.startswith("--") or line.startswith("/*") or line.startswith("LOCK TABLES") or line.startswith("DROP TABLE") or line.startswith("UNLOCK TABLES") or not line:
continue
# Outside of anything handling
if current_table is None:
# Start of a table creation statement?
if line.startswith("CREATE TABLE"):
current_table = line.split('"')[1]
tables[current_table] = {"columns": []}
creation_lines = []
# Inserting data into a table?
elif line.startswith("INSERT INTO"):
output.write(line.encode("utf8").replace("'0000-00-00 00:00:00'", "NULL") + "\n")
num_inserts += 1
# ???
else:
print "\n ! Unknown line in main body: %s" % line
# Inside-create-statement handling
else:
# Is it a column?
if line.startswith('"'):
useless, name, definition = line.strip(",").split('"',2)
try:
type, extra = definition.strip().split(" ", 1)
# This must be a tricky enum
if ')' in extra:
type, extra = definition.strip().split(")")
except ValueError:
type = definition.strip()
extra = ""
extra = re.sub("CHARACTER SET [\w\d]+\s*", "", extra.replace("unsigned", ""))
extra = re.sub("COLLATE [\w\d]+\s*", "", extra.replace("unsigned", ""))
# See if it needs type conversion
final_type = None
set_sequence = None
if type.startswith("tinyint("):
type = "int4"
set_sequence = True
final_type = "boolean"
elif type.startswith("int("):
type = "integer"
set_sequence = True
elif type.startswith("bigint("):
type = "bigint"
set_sequence = True
elif type == "longtext":
type = "text"
elif type == "mediumtext":
type = "text"
elif type == "tinytext":
type = "text"
elif type.startswith("varchar("):
size = int(type.split("(")[1].rstrip(")"))
type = "varchar(%s)" % (size * 2)
elif type.startswith("smallint("):
type = "int2"
set_sequence = True
elif type == "datetime":
type = "timestamp with time zone"
elif type == "double":
type = "double precision"
elif type.endswith("blob"):
type = "bytea"
elif type.startswith("enum(") or type.startswith("set("):
types_str = type.split("(")[1].rstrip(")").rstrip('"')
types_arr = [type_str.strip('\'') for type_str in types_str.split(",")]
# Considered using values to make a name, but its dodgy
# enum_name = '_'.join(types_arr)
enum_name = "{0}_{1}".format(current_table, name)
if enum_name not in enum_types:
output.write("CREATE TYPE {0} AS ENUM ({1}); \n".format(enum_name, types_str));
enum_types.append(enum_name)
type = enum_name
if final_type:
cast_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"%s\" DROP DEFAULT, ALTER COLUMN \"%s\" TYPE %s USING CAST(\"%s\" as %s)" % (current_table, name, name, final_type, name, final_type))
# ID fields need sequences [if they are integers?]
if name == "id" and set_sequence is True:
sequence_lines.append("CREATE SEQUENCE %s_id_seq" % (current_table))
sequence_lines.append("SELECT setval('%s_id_seq', max(id)) FROM %s" % (current_table, current_table))
sequence_lines.append("ALTER TABLE \"%s\" ALTER COLUMN \"id\" SET DEFAULT nextval('%s_id_seq')" % (current_table, current_table))
# Record it
creation_lines.append('"%s" %s %s' % (name, type, extra))
tables[current_table]['columns'].append((name, type, extra))
# Is it a constraint or something?
elif line.startswith("PRIMARY KEY"):
creation_lines.append(line.rstrip(","))
elif line.startswith("CONSTRAINT"):
foreign_key_lines.append("ALTER TABLE \"%s\" ADD CONSTRAINT %s DEFERRABLE INITIALLY DEFERRED" % (current_table, line.split("CONSTRAINT")[1].strip().rstrip(",")))
foreign_key_lines.append("CREATE INDEX ON \"%s\" %s" % (current_table, line.split("FOREIGN KEY")[1].split("REFERENCES")[0].strip().rstrip(",")))
elif line.startswith("UNIQUE KEY"):
creation_lines.append("UNIQUE (%s)" % line.split("(")[1].split(")")[0])
elif line.startswith("FULLTEXT KEY"):
fulltext_keys = " || ' ' || ".join( line.split('(')[-1].split(')')[0].replace('"', '').split(',') )
fulltext_key_lines.append("CREATE INDEX ON %s USING gin(to_tsvector('english', %s))" % (current_table, fulltext_keys))
elif line.startswith("KEY"):
pass
# Is it the end of the table?
elif line == ");":
output.write("CREATE TABLE \"%s\" (\n" % current_table)
for i, line in enumerate(creation_lines):
output.write(" %s%s\n" % (line, "," if i != (len(creation_lines) - 1) else ""))
output.write(');\n\n')
current_table = None
# ???
else:
print "\n ! Unknown line inside table creation: %s" % line
# Finish file
output.write("\n-- Post-data save --\n")
output.write("COMMIT;\n")
output.write("START TRANSACTION;\n")
# Write typecasts out
output.write("\n-- Typecasts --\n")
for line in cast_lines:
output.write("%s;\n" % line)
# Write FK constraints out
output.write("\n-- Foreign keys --\n")
for line in foreign_key_lines:
output.write("%s;\n" % line)
# Write sequences out
output.write("\n-- Sequences --\n")
for line in sequence_lines:
output.write("%s;\n" % line)
# Write full-text indexkeyses out
output.write("\n-- Full Text keys --\n")
for line in fulltext_key_lines:
output.write("%s;\n" % line)
# Finish file
output.write("\n")
output.write("COMMIT;\n")
print ""
0
Example 169
View license def perform_login(self, credentials, cookie_handler):
"""
Parses CSRF token if available and performs login request.
"""
if self.cookies:
self.add_cookies(cookie_handler, self.cookies)
if self.csrf_token_name:
self.url = self.csrf_url
csrf_response = self.get_page()
if not csrf_response:
if(self.invalid_http_status and self.response_status and
int(self.invalid_http_status["value"]) == int(
self.response_status)):
if self.verbose:
print("%s %s\n" %
(INFO, self.invalid_http_status["msg"] if "msg"
in self.invalid_http_status else
"Your IP may have been blocked..."))
elif self.verbose:
print("%s problem receiving HTTP response "
"while fetching token!\n" % ERROR)
return
if self.csrf_regex or self.csrf_setcookie:
match = search(self.csrf_regex if self.csrf_regex else
self.csrf_setcookie, csrf_response if
self.csrf_regex else str(cookie_handler), I)
if match:
self.csrf_token = (match.group("token") if "token" in
match.groupdict() else match.group(1))
else:
self.status = {"status": 0, "msg": "No token"}
else:
start = csrf_response.find(self.csrf_start)
if start == -1 and self.verbose: # lvl 1 verbose
self.status = {"status": 0, "msg": "No token"}
end = csrf_response.find(self.csrf_end,
start+len(self.csrf_start))
self.csrf_token = csrf_response[start+len(self.csrf_start):end]
if self.status["msg"] == "No token" or not self.csrf_token:
if self.verbose:
print "%s CSRF token not found. Skipping site...\n" % WARN
return
if self.verbose:
print("%s Authentication Token: \"%s\"" %
(INFO, color(self.csrf_token)))
def replace_param(string, param, value, param_format=None):
"""
Replace data in parameters with given string.
Parameter format can be json or normal POST data.
"""
if param_format == "json":
return sub(r"(?P<json_replacement>\"%s\"\s*:\s*)\"\s*\"" %
escape(str(param)), "\\1\"%s\"" % value, string)
elif param_format == "header":
return sub(r"%s=[^\\n]*" % escape(str(param)), r"%s=%s" %
(str(param).encode('string-escape'),
str(value).encode('string-escape')), string)
else:
return sub(r"%s=[^&]*" % escape(str(param)), r"%s=%s" %
(str(param).encode('string-escape'),
str(value).encode('string-escape')), string)
if self.multiple_params:
multiple_params_response = ""
if(self.csrf_token_name and
self.csrf_url == self.multiple_params_url):
multiple_params_response = csrf_response
else:
self.url = self.multiple_params_url
multiple_params_response = self.get_page()
if(self.invalid_http_status and self.response_status and
int(self.invalid_http_status["value"]) == int(
self.response_status) and self.verbose):
print("%s %s\n" % (INFO, self.invalid_http_status["msg"]
if "msg" in self.invalid_http_status else
"Your IP may have been blocked..."))
return
if not multiple_params_response:
print("%s problem receiving HTTP response while fetching "
"params! Skipping site...\n" % ERROR)
return
for _ in self.multiple_params:
regex = (_["regex"] if "regex" in _ else
r"<\w+[^>]*(value\s*=\s*\"[^\"]*\"|name\s*=\s*"
r"\"?{0}(?:\"|\s))[^>]*(value\s*=\s*\"[^\"]*\""
r"|name\s*=\s*\"?{0}(?:\"|\s))[^>]*>"
.format(escape(_["value"])))
match = search(regex, multiple_params_response)
if not match or "value" not in _:
if self.verbose:
print("%s no match for parameter \"%s\"! "
"Skipping site...\n" %
(WARN, color(_["value"])))
self.status = {"status": 0, "msg": "No token"}
return
if "regex" in _:
value = (match.group("value")
if "value" in match.groupdict() else
match.group(1))
elif "value" in _:
for attrib in match.groups():
attrib = str(attrib).strip().split("=", 1)
if attrib[0] == "value":
value = attrib[1].strip("\"")
if "type" not in _:
_["type"] = "data"
if _["type"] == "data" and self.data:
self.data = replace_param(self.data, _["value"], value)
elif _["type"] == "header":
self.headers = replace_param(self.headers, _["value"],
value, "header")
elif _["type"] == "cookie":
self.add_cookies(cookie_handler, "%s=%s;" % (_["value"],
value))
else:
pass # NEED TO REPLACE GET PARAMS
if(credentials["email"] and
self.username_or_email in("email", "both")):
login = credentials["email"]
elif(credentials["email"] and self.email_exception and
self.username_or_email == "username" and
search(self.email_exception, credentials["email"])):
login = credentials["email"]
else:
login = credentials["username"]
# need to implement support for GET logins lulz
if self.time_parameter:
if "type" not in self.time_parameter:
self.time_parameter["type"] = "epoch"
if self.time_parameter["type"] == "epoch":
if self.data:
self.data = replace_param(self.data,
self.time_parameter["value"],
time())
if self.data:
self.data = replace_param(self.data, self.login_parameter,
login, self.login_parameter_type)
self.data = replace_param(self.data, self.password_parameter,
credentials["password"],
self.login_parameter_type)
# need to be able to specify where tokens can be replaced
if self.csrf_token:
self.csrf_token = quote(self.csrf_token)
if self.data:
self.data = replace_param(self.data,
self.csrf_token_name,
self.csrf_token,
self.login_parameter_type)
if self.headers:
self.headers = replace_param(self.headers,
self.csrf_token_name,
self.csrf_token, "header")
if self.cookies:
self.cookies = replace_param(self.cookies,
self.csrf_token_name,
self.csrf_token)
self.add_cookies(cookie_handler, self.cookies)
self.url = self.login_url
login_response = self.get_page(data=True if self.data else False,
header=True if self.headers else False)
if not login_response:
if self.verbose:
print("%s no response received! Skipping site...\n" % WARN)
return False
if self.login_redirect:
if self.login_redirect_type == "regex":
self.url = search(self.login_redirect, login_response)
self.url = (self.url.group("URL")
if "URL" in self.url.groupdict()
else self.url.group(1))
else:
self.url = self.login_redirect
self.url = self.url.replace("\\", "")
login_response = self.get_page(data=True if self.data else False,
header=True if self.headers
else False)
if not login_response:
if self.verbose:
print("%s no response received during login redirect! "
"Skipping site...\n" % WARN)
return False
# The code for these IF checks need to be cleaned up
# If invalid credentials http status code is returned
elif (self.invalid_http_status and self.response_status and
str(self.invalid_http_status["value"]) ==
str(self.response_status)):
if("msg" in self.invalid_http_status or not
login_response.strip("[]")):
if self.verbose:
print("%s %s\n" %
(INFO, self.invalid_http_status["msg"] if "msg"
in self.invalid_http_status else
"The provided credentials are incorrect "
"or the account doesn't exist.\n"))
return False
# If captcha flag is set and found in login response
if self.captcha_flag and self.captcha_flag in login_response:
if self.verbose:
print "%s captcha detected! Skipping site...\n" % WARN
return False
# If custom search is set and found in response
elif self.custom_search and search(self.custom_search['regex'],
login_response):
if self.verbose:
print "%s %s\n" % (INFO, self.custom_search["value"])
return False
# Valid password string in response
elif self.valid_password and self.valid_password in login_response:
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header type REGEX
elif (self.valid_response_header and
self.valid_response_header_type == "regex" and
search(self.valid_response_header,
str(self.response_headers))):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header for cookies type REGEX
elif (self.valid_response_header and
self.valid_response_header_type == "regex" and
search(self.valid_response_header, str(cookie_handler))):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header type normal
elif (self.valid_response_header and self.valid_response_header
in str(self.response_headers)):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header for cookies type normal
elif (self.valid_response_header and self.valid_response_header
in str(cookie_handler)):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Custom message when specified invalid header is detected
elif (self.custom_response_header and
self.custom_response_header["value"] in
str(self.response_headers)):
if self.verbose:
print "%s %s" % (INFO, self.custom_response_header["msg"])
return False
# Invalid account string found in login response
elif self.invalid_account and self.invalid_account in login_response:
if self.verbose:
print("%s The provided account doesn't exist on this site.\n"
% INFO)
return False
# User exists, but account isn't activate.
elif self.inactive_account and self.inactive_account in login_response:
if self.verbose:
print("%s The user exists, but the account isn't activate.\n"
% INFO)
return False
# User exists, but invalid password string in login response
elif (self.invalid_password and self.invalid_account and
self.invalid_password in login_response):
if self.verbose:
print("%s The user exists, but the password is incorrect.\n"
% INFO)
return False
# Invalid password string in login response
elif (self.invalid_password and not self.invalid_account and
self.invalid_password in login_response):
if self.verbose:
print("%s The provided credentials are incorrect "
"or the account doesn't exist.\n" % INFO)
return False
# Unhandled case
else:
if self.verbose:
print "%s Unable to login! Skipping site...\n" % WARN
return False
0
Example 170
View license def perform_login(self, credentials, cookie_handler):
"""
Parses CSRF token if available and performs login request.
"""
if self.cookies:
self.add_cookies(cookie_handler, self.cookies)
if self.csrf_token_name:
self.url = self.csrf_url
csrf_response = self.get_page()
if not csrf_response:
if(self.invalid_http_status and self.response_status and
int(self.invalid_http_status["value"]) == int(
self.response_status)):
if self.verbose:
print("%s %s\n" %
(INFO, self.invalid_http_status["msg"] if "msg"
in self.invalid_http_status else
"Your IP may have been blocked..."))
elif self.verbose:
print("%s problem receiving HTTP response "
"while fetching token!\n" % ERROR)
return
if self.csrf_regex or self.csrf_setcookie:
match = search(self.csrf_regex if self.csrf_regex else
self.csrf_setcookie, csrf_response if
self.csrf_regex else str(cookie_handler), I)
if match:
self.csrf_token = (match.group("token") if "token" in
match.groupdict() else match.group(1))
else:
self.status = {"status": 0, "msg": "No token"}
else:
start = csrf_response.find(self.csrf_start)
if start == -1 and self.verbose: # lvl 1 verbose
self.status = {"status": 0, "msg": "No token"}
end = csrf_response.find(self.csrf_end,
start+len(self.csrf_start))
self.csrf_token = csrf_response[start+len(self.csrf_start):end]
if self.status["msg"] == "No token" or not self.csrf_token:
if self.verbose:
print "%s CSRF token not found. Skipping site...\n" % WARN
return
if self.verbose:
print("%s Authentication Token: \"%s\"" %
(INFO, color(self.csrf_token)))
def replace_param(string, param, value, param_format=None):
"""
Replace data in parameters with given string.
Parameter format can be json or normal POST data.
"""
if param_format == "json":
return sub(r"(?P<json_replacement>\"%s\"\s*:\s*)\"\s*\"" %
escape(str(param)), "\\1\"%s\"" % value, string)
elif param_format == "header":
return sub(r"%s=[^\\n]*" % escape(str(param)), r"%s=%s" %
(str(param).encode('string-escape'),
str(value).encode('string-escape')), string)
else:
return sub(r"%s=[^&]*" % escape(str(param)), r"%s=%s" %
(str(param).encode('string-escape'),
str(value).encode('string-escape')), string)
if self.multiple_params:
multiple_params_response = ""
if(self.csrf_token_name and
self.csrf_url == self.multiple_params_url):
multiple_params_response = csrf_response
else:
self.url = self.multiple_params_url
multiple_params_response = self.get_page()
if(self.invalid_http_status and self.response_status and
int(self.invalid_http_status["value"]) == int(
self.response_status) and self.verbose):
print("%s %s\n" % (INFO, self.invalid_http_status["msg"]
if "msg" in self.invalid_http_status else
"Your IP may have been blocked..."))
return
if not multiple_params_response:
print("%s problem receiving HTTP response while fetching "
"params! Skipping site...\n" % ERROR)
return
for _ in self.multiple_params:
regex = (_["regex"] if "regex" in _ else
r"<\w+[^>]*(value\s*=\s*\"[^\"]*\"|name\s*=\s*"
r"\"?{0}(?:\"|\s))[^>]*(value\s*=\s*\"[^\"]*\""
r"|name\s*=\s*\"?{0}(?:\"|\s))[^>]*>"
.format(escape(_["value"])))
match = search(regex, multiple_params_response)
if not match or "value" not in _:
if self.verbose:
print("%s no match for parameter \"%s\"! "
"Skipping site...\n" %
(WARN, color(_["value"])))
self.status = {"status": 0, "msg": "No token"}
return
if "regex" in _:
value = (match.group("value")
if "value" in match.groupdict() else
match.group(1))
elif "value" in _:
for attrib in match.groups():
attrib = str(attrib).strip().split("=", 1)
if attrib[0] == "value":
value = attrib[1].strip("\"")
if "type" not in _:
_["type"] = "data"
if _["type"] == "data" and self.data:
self.data = replace_param(self.data, _["value"], value)
elif _["type"] == "header":
self.headers = replace_param(self.headers, _["value"],
value, "header")
elif _["type"] == "cookie":
self.add_cookies(cookie_handler, "%s=%s;" % (_["value"],
value))
else:
pass # NEED TO REPLACE GET PARAMS
if(credentials["email"] and
self.username_or_email in("email", "both")):
login = credentials["email"]
elif(credentials["email"] and self.email_exception and
self.username_or_email == "username" and
search(self.email_exception, credentials["email"])):
login = credentials["email"]
else:
login = credentials["username"]
# need to implement support for GET logins lulz
if self.time_parameter:
if "type" not in self.time_parameter:
self.time_parameter["type"] = "epoch"
if self.time_parameter["type"] == "epoch":
if self.data:
self.data = replace_param(self.data,
self.time_parameter["value"],
time())
if self.data:
self.data = replace_param(self.data, self.login_parameter,
login, self.login_parameter_type)
self.data = replace_param(self.data, self.password_parameter,
credentials["password"],
self.login_parameter_type)
# need to be able to specify where tokens can be replaced
if self.csrf_token:
self.csrf_token = quote(self.csrf_token)
if self.data:
self.data = replace_param(self.data,
self.csrf_token_name,
self.csrf_token,
self.login_parameter_type)
if self.headers:
self.headers = replace_param(self.headers,
self.csrf_token_name,
self.csrf_token, "header")
if self.cookies:
self.cookies = replace_param(self.cookies,
self.csrf_token_name,
self.csrf_token)
self.add_cookies(cookie_handler, self.cookies)
self.url = self.login_url
login_response = self.get_page(data=True if self.data else False,
header=True if self.headers else False)
if not login_response:
if self.verbose:
print("%s no response received! Skipping site...\n" % WARN)
return False
if self.login_redirect:
if self.login_redirect_type == "regex":
self.url = search(self.login_redirect, login_response)
self.url = (self.url.group("URL")
if "URL" in self.url.groupdict()
else self.url.group(1))
else:
self.url = self.login_redirect
self.url = self.url.replace("\\", "")
login_response = self.get_page(data=True if self.data else False,
header=True if self.headers
else False)
if not login_response:
if self.verbose:
print("%s no response received during login redirect! "
"Skipping site...\n" % WARN)
return False
# The code for these IF checks need to be cleaned up
# If invalid credentials http status code is returned
elif (self.invalid_http_status and self.response_status and
str(self.invalid_http_status["value"]) ==
str(self.response_status)):
if("msg" in self.invalid_http_status or not
login_response.strip("[]")):
if self.verbose:
print("%s %s\n" %
(INFO, self.invalid_http_status["msg"] if "msg"
in self.invalid_http_status else
"The provided credentials are incorrect "
"or the account doesn't exist.\n"))
return False
# If captcha flag is set and found in login response
if self.captcha_flag and self.captcha_flag in login_response:
if self.verbose:
print "%s captcha detected! Skipping site...\n" % WARN
return False
# If custom search is set and found in response
elif self.custom_search and search(self.custom_search['regex'],
login_response):
if self.verbose:
print "%s %s\n" % (INFO, self.custom_search["value"])
return False
# Valid password string in response
elif self.valid_password and self.valid_password in login_response:
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header type REGEX
elif (self.valid_response_header and
self.valid_response_header_type == "regex" and
search(self.valid_response_header,
str(self.response_headers))):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header for cookies type REGEX
elif (self.valid_response_header and
self.valid_response_header_type == "regex" and
search(self.valid_response_header, str(cookie_handler))):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header type normal
elif (self.valid_response_header and self.valid_response_header
in str(self.response_headers)):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Valid response header for cookies type normal
elif (self.valid_response_header and self.valid_response_header
in str(cookie_handler)):
print "%s Credentials worked! Successfully logged in.\n" % PLUS
return True
# Custom message when specified invalid header is detected
elif (self.custom_response_header and
self.custom_response_header["value"] in
str(self.response_headers)):
if self.verbose:
print "%s %s" % (INFO, self.custom_response_header["msg"])
return False
# Invalid account string found in login response
elif self.invalid_account and self.invalid_account in login_response:
if self.verbose:
print("%s The provided account doesn't exist on this site.\n"
% INFO)
return False
# User exists, but account isn't activate.
elif self.inactive_account and self.inactive_account in login_response:
if self.verbose:
print("%s The user exists, but the account isn't activate.\n"
% INFO)
return False
# User exists, but invalid password string in login response
elif (self.invalid_password and self.invalid_account and
self.invalid_password in login_response):
if self.verbose:
print("%s The user exists, but the password is incorrect.\n"
% INFO)
return False
# Invalid password string in login response
elif (self.invalid_password and not self.invalid_account and
self.invalid_password in login_response):
if self.verbose:
print("%s The provided credentials are incorrect "
"or the account doesn't exist.\n" % INFO)
return False
# Unhandled case
else:
if self.verbose:
print "%s Unable to login! Skipping site...\n" % WARN
return False
0
Example 171
View licensedef _make_jsmin(python_only=False):
"""
Generate JS minifier based on `jsmin.c by Douglas Crockford`_
.. _jsmin.c by Douglas Crockford:
http://www.crockford.com/javascript/jsmin.c
:Parameters:
`python_only` : ``bool``
Use only the python variant. If true, the c extension is not even
tried to be loaded.
:Return: Minifier
:Rtype: ``callable``
"""
# pylint: disable = unused-variable
# pylint: disable = too-many-locals
if not python_only:
try:
import _rjsmin
except ImportError:
pass
else:
return _rjsmin.jsmin
try:
xrange
except NameError:
xrange = range # pylint: disable = redefined-builtin
space_chars = r'[\000-\011\013\014\016-\040]'
line_comment = r'(?://[^\r\n]*)'
space_comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
space_comment_nobang = r'(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*\*+)*/)'
bang_comment = r'(?:/\*![^*]*\*+(?:[^/*][^*]*\*+)*/)'
string1 = \
r'(?:\047[^\047\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)'
string2 = r'(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^"\\\r\n]*)*")'
strings = r'(?:%s|%s)' % (string1, string2)
charclass = r'(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\])'
nospecial = r'[^/\\\[\r\n]'
regex = r'(?:/(?![\r\n/*])%s*(?:(?:\\[^\r\n]|%s)%s*)*/)' % (
nospecial, charclass, nospecial
)
space = r'(?:%s|%s)' % (space_chars, space_comment)
newline = r'(?:%s?[\r\n])' % line_comment
def fix_charclass(result):
""" Fixup string of chars to fit into a regex char class """
pos = result.find('-')
if pos >= 0:
result = r'%s%s-' % (result[:pos], result[pos + 1:])
def sequentize(string):
"""
Notate consecutive characters as sequence
(1-4 instead of 1234)
"""
first, last, result = None, None, []
for char in map(ord, string):
if last is None:
first = last = char
elif last + 1 == char:
last = char
else:
result.append((first, last))
first = last = char
if last is not None:
result.append((first, last))
return ''.join(['%s%s%s' % (
chr(first),
last > first + 1 and '-' or '',
last != first and chr(last) or ''
) for first, last in result]) # noqa
return _re.sub(
r'([\000-\040\047])', # \047 for better portability
lambda m: '\\%03o' % ord(m.group(1)), (
sequentize(result)
.replace('\\', '\\\\')
.replace('[', '\\[')
.replace(']', '\\]')
)
)
def id_literal_(what):
""" Make id_literal like char class """
match = _re.compile(what).match
result = ''.join([
chr(c) for c in xrange(127) if not match(chr(c))
])
return '[^%s]' % fix_charclass(result)
def not_id_literal_(keep):
""" Make negated id_literal like char class """
match = _re.compile(id_literal_(keep)).match
result = ''.join([
chr(c) for c in xrange(127) if not match(chr(c))
])
return r'[%s]' % fix_charclass(result)
not_id_literal = not_id_literal_(r'[a-zA-Z0-9_$]')
preregex1 = r'[(,=:\[!&|?{};\r\n]'
preregex2 = r'%(not_id_literal)sreturn' % locals()
id_literal = id_literal_(r'[a-zA-Z0-9_$]')
id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]')
id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047+-]')
post_regex_off = id_literal_(r'[^\000-\040}\])?:|,;.&=+-]')
dull = r'[^\047"/\000-\040]'
space_sub_simple = _re.compile((
# noqa pylint: disable = bad-continuation
r'(%(dull)s+)' # 0
r'|(%(strings)s%(dull)s*)' # 1
r'|(?<=%(preregex1)s)'
r'%(space)s*(?:%(newline)s%(space)s*)*'
r'(%(regex)s)' # 2
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 3
r'(?=%(post_regex_off)s))?'
r'|(?<=%(preregex2)s)'
r'%(space)s*(?:(%(newline)s)%(space)s*)*' # 4
r'(%(regex)s)' # 5
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 6
r'(?=%(post_regex_off)s))?'
r'|(?<=%(id_literal_close)s)'
r'%(space)s*(?:(%(newline)s)%(space)s*)+' # 7
r'(?=%(id_literal_open)s)'
r'|(?<=%(id_literal)s)(%(space)s)+(?=%(id_literal)s)' # 8
r'|(?<=\+)(%(space)s)+(?=\+)' # 9
r'|(?<=-)(%(space)s)+(?=-)' # 10
r'|%(space)s+'
r'|(?:%(newline)s%(space)s*)+'
) % locals()).sub
# print space_sub_simple.__self__.pattern
def space_subber_simple(match):
""" Substitution callback """
# pylint: disable = too-many-return-statements
groups = match.groups()
if groups[0]:
return groups[0]
elif groups[1]:
return groups[1]
elif groups[2]:
if groups[3]:
return groups[2] + '\n'
return groups[2]
elif groups[5]:
return "%s%s%s" % (
groups[4] and '\n' or '',
groups[5],
groups[6] and '\n' or '',
)
elif groups[7]:
return '\n'
elif groups[8] or groups[9] or groups[10]:
return ' '
else:
return ''
space_sub_banged = _re.compile((
# noqa pylint: disable = bad-continuation
r'(%(dull)s+)' # 0
r'|(%(strings)s%(dull)s*)' # 1
r'|(?<=%(preregex1)s)'
r'(%(space)s*(?:%(newline)s%(space)s*)*)' # 2
r'(%(regex)s)' # 3
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 4
r'(?=%(post_regex_off)s))?'
r'|(?<=%(preregex2)s)'
r'(%(space)s*(?:(%(newline)s)%(space)s*)*)' # 5, 6
r'(%(regex)s)' # 7
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 8
r'(?=%(post_regex_off)s))?'
r'|(?<=%(id_literal_close)s)'
r'(%(space)s*(?:%(newline)s%(space)s*)+)' # 9
r'(?=%(id_literal_open)s)'
r'|(?<=%(id_literal)s)(%(space)s+)(?=%(id_literal)s)' # 10
r'|(?<=\+)(%(space)s+)(?=\+)' # 11
r'|(?<=-)(%(space)s+)(?=-)' # 12
r'|(%(space)s+)' # 13
r'|((?:%(newline)s%(space)s*)+)' # 14
) % locals()).sub
# print space_sub_banged.__self__.pattern
keep = _re.compile((
r'%(space_chars)s+|%(space_comment_nobang)s+|%(newline)s+'
r'|(%(bang_comment)s+)'
) % locals()).sub
keeper = lambda m: m.groups()[0] or ''
# print keep.__self__.pattern
def space_subber_banged(match):
""" Substitution callback """
# pylint: disable = too-many-return-statements
groups = match.groups()
if groups[0]:
return groups[0]
elif groups[1]:
return groups[1]
elif groups[3]:
return "%s%s%s%s" % (
keep(keeper, groups[2]),
groups[3],
keep(keeper, groups[4] or ''),
groups[4] and '\n' or '',
)
elif groups[7]:
return "%s%s%s%s%s" % (
keep(keeper, groups[5]),
groups[6] and '\n' or '',
groups[7],
keep(keeper, groups[8] or ''),
groups[8] and '\n' or '',
)
elif groups[9]:
return keep(keeper, groups[9]) + '\n'
elif groups[10] or groups[11] or groups[12]:
return keep(keeper, groups[10] or groups[11] or groups[12]) or ' '
else:
return keep(keeper, groups[13] or groups[14])
def jsmin(script, keep_bang_comments=False):
r"""
Minify javascript based on `jsmin.c by Douglas Crockford`_\.
Instead of parsing the stream char by char, it uses a regular
expression approach which minifies the whole script with one big
substitution regex.
.. _jsmin.c by Douglas Crockford:
http://www.crockford.com/javascript/jsmin.c
:Parameters:
`script` : ``str``
Script to minify
`keep_bang_comments` : ``bool``
Keep comments starting with an exclamation mark? (``/*!...*/``)
:Return: Minified script
:Rtype: ``str``
"""
# pylint: disable = redefined-outer-name
if keep_bang_comments:
return space_sub_banged(
space_subber_banged, '\n%s\n' % script
).strip()
else:
return space_sub_simple(
space_subber_simple, '\n%s\n' % script
).strip()
return jsmin
0
Example 172
View license def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with normalize-space
re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
0
Example 173
View license def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with normalize-space
re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
0
Example 174
View licensedef main(screen):
global stdscr
global votecount
global window_width
global max_yeacount_len
global max_naycount_len
global max_percentage_len
global ballot_entries
global votewin
global masternodes
global C_YELLOW, C_GREEN, C_RED, C_CYAN
stdscr = screen
stdscr.scrollok(1)
git_describe = run_command(
'GIT_DIR=%s GIT_WORK_TREE=%s git describe' %
(git_dir + '/.git', git_dir)).rstrip("\n").split('-')
try:
GIT_VERSION = ('-').join((git_describe[i] for i in [1, 2]))
version = 'v' + VERSION + ' (' + GIT_VERSION + ')'
except IndexError:
version = 'v' + VERSION
try:
curses.curs_set(2)
except:
pass
if curses.has_colors():
curses.start_color()
curses.use_default_colors()
for i in range(0, curses.COLORS):
curses.init_pair(i + 1, i, -1)
C_CYAN = curses.color_pair(7)
C_YELLOW = curses.color_pair(4)
C_GREEN = curses.color_pair(3)
C_RED = curses.color_pair(2)
# test dash-cli in path -- TODO make robust
try:
run_command('dash-cli getinfo')
except subprocess.CalledProcessError:
quit(
"--> cannot find dash-cli in $PATH\n" +
" do: export PATH=/path/to/dash-cli-folder:$PATH\n" +
" and try again\n")
loadwin = curses.newwin(40, 40, 1, 2)
loadwin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
loadwin.addstr(2, 2, 'loading votes... please wait', C_GREEN)
loadwin.refresh()
mncount = int(run_command('dash-cli masternode count'))
block_height = int(run_command('dash-cli getblockcount'))
# get ballot
ballots = json.loads(run_command('dash-cli mnbudget show'))
ballot = {}
for entry in ballots:
# prune expired proposals
if ballots[entry][u'BlockEnd'] < block_height:
continue
# prune completely funded proposals
if ballots[entry][u'RemainingPaymentCount'] < 1:
continue
ballots[entry][u'vote'] = 'ABSTAIN'
ballots[entry][u'votes'] = json.loads(
run_command(
'dash-cli mnbudget getvotes %s' %
entry))
ballot[entry] = ballots[entry]
ballot_entries = sorted(ballot, key=lambda s: ballot[s]['BlockStart'])
votecount = len(ballot_entries)
max_proposal_len = 0
max_yeacount_len = 0
max_naycount_len = 0
max_percentage_len = 0
for entry in ballot_entries:
yeas = ballot[entry][u'Yeas']
nays = ballot[entry][u'Nays']
percentage = "{0:.1f}".format(
(float((yeas + nays)) / float(mncount)) * 100)
ballot[entry][u'vote_turnout'] = percentage
ballot[entry][u'vote_threshold'] = (
yeas + nays) > mncount/10 and True or False
ballot[entry][u'vote_passing'] = (
yeas - nays) > mncount/10 and True or False
max_proposal_len = max(
max_proposal_len,
len(entry))
max_yeacount_len = max(max_yeacount_len, len(str(yeas)))
max_naycount_len = max(max_naycount_len, len(str(nays)))
max_percentage_len = max(max_percentage_len, len(str(percentage)))
# extract mnprivkey,txid-txidx from masternode.conf
masternodes = {}
with open(os.path.join(dash_conf_dir, 'masternode.conf'), 'r') as f:
lines = list(
line
for line in
(l.strip() for l in f)
if line and not line.startswith('#'))
for line in lines:
conf = line.split()
masternodes[conf[3] + '-' + conf[4]] = {
"alias": conf[0],
"mnprivkey": conf[2],
"fundtx": conf[3] +
'-' +
conf[4],
"txid": conf[3],
"txout": conf[4]}
if not masternodes:
# fallback to dash.conf entries if no masternode.conf entries
with open(os.path.join(dash_conf_dir, 'dash.conf'), 'r') as f:
lines = list(
line
for line in
(l.strip() for l in f)
if line and not line.startswith('#'))
conf = {}
for line in lines:
n, v = line.split('=')
conf[n.strip(' ')] = v.strip(' ')
conf['masternodeaddr'] = re.sub(
'[\[\]]',
'',
conf['masternodeaddr'])
if all(k in conf for k in ('masternode', 'masternodeaddr', 'masternodeprivkey')):
# get funding tx from dashninja
import urllib2
mninfo = urllib2.urlopen(
"https://dashninja.pl/api/masternodes?ips=[\"" +
conf['masternodeaddr'] +
"\"]&portcheck=1").read()
try:
mndata = json.loads(mninfo)
d = mndata[u'data'][0]
except:
quit('cannot retrieve masternode info from dashninja')
vin = str(d[u'MasternodeOutputHash'])
vidx = str(d[u'MasternodeOutputIndex'])
masternodes[vin + '-' + vidx] = {
"alias": conf['masternodeaddr'],
"mnprivkey": conf['masternodeprivkey'],
"fundtx": vin +
'-' +
vidx,
"txid": vin,
"txout": vidx}
else:
quit('cannot find masternode information in dash.conf')
# TODO open previous votes/local storage something
for entry in ballot:
ballot[entry][u'previously_voted'] = 0
for hash in ballot[entry][u'votes']:
if hash in masternodes:
if ballot[entry][u'votes'][hash][u'Vote'] == 'YES':
ballot[entry][u'previously_voted'] = 1
else:
ballot[entry][u'previously_voted'] = 2
loadwin.erase()
window_width = 35
window_width = max(window_width, max_proposal_len +
max_percentage_len +
max_yeacount_len +
max_naycount_len +
len(str(len(masternodes))))
votewin = curses.newwin(votecount + 9, window_width + 17, 1, 2)
votewin.keypad(1)
votewin.border()
votewin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
votewin.addstr(
2,
2,
'use arrow keys to set votes for %s masternodes' %
len(masternodes),
C_YELLOW)
votewin.addstr(3, 2, 'hit enter on CONFIRM to vote - q to quit', C_YELLOW)
votewin.addstr(4, 3, '*', C_GREEN)
votewin.addstr(4, 4, '/', C_CYAN)
votewin.addstr(4, 5, '*', C_RED)
votewin.addstr(4, 7, '== previously voted proposal (yes/no)', C_YELLOW)
_y = 5
for entry in ballot_entries:
_y += 1
x = 4
yeas = ballot[entry][u'Yeas']
nays = ballot[entry][u'Nays']
percentage = ballot[entry][u'vote_turnout']
passing = ballot[entry][u'vote_passing']
threshold = ballot[entry][u'vote_threshold']
if ballot[entry][u'previously_voted'] > 0:
direction = ballot[entry][u'previously_voted']
votewin.addstr(_y, x-1, '*', direction == 1 and C_GREEN or C_RED)
fmt_entry = "%-"+str(max_proposal_len + 2)+"s"
votewin.addstr(
_y,
x,
fmt_entry % entry,
passing and C_GREEN or threshold and C_RED or C_YELLOW)
for x in range(max_yeacount_len - len(str(yeas))):
votewin.addstr(' ')
votewin.addstr(str(yeas), C_GREEN)
votewin.addstr('/', C_CYAN)
votewin.addstr(str(nays), C_RED)
votewin.addstr(' ')
for x in range(max_naycount_len - len(str(nays))):
votewin.addstr(' ')
for x in range(max_percentage_len - len(str(percentage))):
votewin.addstr(' ')
votewin.addstr(str(percentage) + "%", C_CYAN)
votewin.addstr(' ')
votewin.addstr('ABSTAIN', C_YELLOW)
votewin.addstr(
_y + 2,
window_width + 7,
'confirm',
C_YELLOW)
votewin.move(0 + 6, window_width + 7)
votewin.refresh()
keys = {
113: lambda s: quit(),
curses.KEY_UP: lambda s: prev_vote(s),
curses.KEY_DOWN: lambda s: next_vote(s),
curses.KEY_RIGHT: lambda s: set_vote(ballot, s, 1),
curses.KEY_LEFT: lambda s: set_vote(ballot, s, -1),
107: lambda s: prev_vote(s),
106: lambda s: next_vote(s),
108: lambda s: set_vote(ballot, s, 1),
104: lambda s: set_vote(ballot, s, -1),
10: lambda s: submit_votes(stdscr, ballot, s)
}
sel_vote = 0
while True:
key = votewin.getch()
f = keys.get(key)
if hasattr(f, '__call__'):
sel_vote = f(sel_vote)
try:
entry_vote = ballot[ballot_entries[sel_vote]][u'vote']
except IndexError:
# CONFIRM button
entry_vote = ''
if key != 10:
update_vote_display(votewin, sel_vote, entry_vote)
0
Example 175
View licensedef main(screen):
global stdscr
global votecount
global window_width
global max_yeacount_len
global max_naycount_len
global max_percentage_len
global ballot_entries
global votewin
global masternodes
global C_YELLOW, C_GREEN, C_RED, C_CYAN
stdscr = screen
stdscr.scrollok(1)
git_describe = run_command(
'GIT_DIR=%s GIT_WORK_TREE=%s git describe' %
(git_dir + '/.git', git_dir)).rstrip("\n").split('-')
try:
GIT_VERSION = ('-').join((git_describe[i] for i in [1, 2]))
version = 'v' + VERSION + ' (' + GIT_VERSION + ')'
except IndexError:
version = 'v' + VERSION
try:
curses.curs_set(2)
except:
pass
if curses.has_colors():
curses.start_color()
curses.use_default_colors()
for i in range(0, curses.COLORS):
curses.init_pair(i + 1, i, -1)
C_CYAN = curses.color_pair(7)
C_YELLOW = curses.color_pair(4)
C_GREEN = curses.color_pair(3)
C_RED = curses.color_pair(2)
# test dash-cli in path -- TODO make robust
try:
run_command('dash-cli getinfo')
except subprocess.CalledProcessError:
quit(
"--> cannot find dash-cli in $PATH\n" +
" do: export PATH=/path/to/dash-cli-folder:$PATH\n" +
" and try again\n")
loadwin = curses.newwin(40, 40, 1, 2)
loadwin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
loadwin.addstr(2, 2, 'loading votes... please wait', C_GREEN)
loadwin.refresh()
mncount = int(run_command('dash-cli masternode count'))
block_height = int(run_command('dash-cli getblockcount'))
# get ballot
ballots = json.loads(run_command('dash-cli mnbudget show'))
ballot = {}
for entry in ballots:
# prune expired proposals
if ballots[entry][u'BlockEnd'] < block_height:
continue
# prune completely funded proposals
if ballots[entry][u'RemainingPaymentCount'] < 1:
continue
ballots[entry][u'vote'] = 'ABSTAIN'
ballots[entry][u'votes'] = json.loads(
run_command(
'dash-cli mnbudget getvotes %s' %
entry))
ballot[entry] = ballots[entry]
ballot_entries = sorted(ballot, key=lambda s: ballot[s]['BlockStart'])
votecount = len(ballot_entries)
max_proposal_len = 0
max_yeacount_len = 0
max_naycount_len = 0
max_percentage_len = 0
for entry in ballot_entries:
yeas = ballot[entry][u'Yeas']
nays = ballot[entry][u'Nays']
percentage = "{0:.1f}".format(
(float((yeas + nays)) / float(mncount)) * 100)
ballot[entry][u'vote_turnout'] = percentage
ballot[entry][u'vote_threshold'] = (
yeas + nays) > mncount/10 and True or False
ballot[entry][u'vote_passing'] = (
yeas - nays) > mncount/10 and True or False
max_proposal_len = max(
max_proposal_len,
len(entry))
max_yeacount_len = max(max_yeacount_len, len(str(yeas)))
max_naycount_len = max(max_naycount_len, len(str(nays)))
max_percentage_len = max(max_percentage_len, len(str(percentage)))
# extract mnprivkey,txid-txidx from masternode.conf
masternodes = {}
with open(os.path.join(dash_conf_dir, 'masternode.conf'), 'r') as f:
lines = list(
line
for line in
(l.strip() for l in f)
if line and not line.startswith('#'))
for line in lines:
conf = line.split()
masternodes[conf[3] + '-' + conf[4]] = {
"alias": conf[0],
"mnprivkey": conf[2],
"fundtx": conf[3] +
'-' +
conf[4],
"txid": conf[3],
"txout": conf[4]}
if not masternodes:
# fallback to dash.conf entries if no masternode.conf entries
with open(os.path.join(dash_conf_dir, 'dash.conf'), 'r') as f:
lines = list(
line
for line in
(l.strip() for l in f)
if line and not line.startswith('#'))
conf = {}
for line in lines:
n, v = line.split('=')
conf[n.strip(' ')] = v.strip(' ')
conf['masternodeaddr'] = re.sub(
'[\[\]]',
'',
conf['masternodeaddr'])
if all(k in conf for k in ('masternode', 'masternodeaddr', 'masternodeprivkey')):
# get funding tx from dashninja
import urllib2
mninfo = urllib2.urlopen(
"https://dashninja.pl/api/masternodes?ips=[\"" +
conf['masternodeaddr'] +
"\"]&portcheck=1").read()
try:
mndata = json.loads(mninfo)
d = mndata[u'data'][0]
except:
quit('cannot retrieve masternode info from dashninja')
vin = str(d[u'MasternodeOutputHash'])
vidx = str(d[u'MasternodeOutputIndex'])
masternodes[vin + '-' + vidx] = {
"alias": conf['masternodeaddr'],
"mnprivkey": conf['masternodeprivkey'],
"fundtx": vin +
'-' +
vidx,
"txid": vin,
"txout": vidx}
else:
quit('cannot find masternode information in dash.conf')
# TODO open previous votes/local storage something
for entry in ballot:
ballot[entry][u'previously_voted'] = 0
for hash in ballot[entry][u'votes']:
if hash in masternodes:
if ballot[entry][u'votes'][hash][u'Vote'] == 'YES':
ballot[entry][u'previously_voted'] = 1
else:
ballot[entry][u'previously_voted'] = 2
loadwin.erase()
window_width = 35
window_width = max(window_width, max_proposal_len +
max_percentage_len +
max_yeacount_len +
max_naycount_len +
len(str(len(masternodes))))
votewin = curses.newwin(votecount + 9, window_width + 17, 1, 2)
votewin.keypad(1)
votewin.border()
votewin.addstr(1, 2, 'dashvote version: ' + version, C_CYAN)
votewin.addstr(
2,
2,
'use arrow keys to set votes for %s masternodes' %
len(masternodes),
C_YELLOW)
votewin.addstr(3, 2, 'hit enter on CONFIRM to vote - q to quit', C_YELLOW)
votewin.addstr(4, 3, '*', C_GREEN)
votewin.addstr(4, 4, '/', C_CYAN)
votewin.addstr(4, 5, '*', C_RED)
votewin.addstr(4, 7, '== previously voted proposal (yes/no)', C_YELLOW)
_y = 5
for entry in ballot_entries:
_y += 1
x = 4
yeas = ballot[entry][u'Yeas']
nays = ballot[entry][u'Nays']
percentage = ballot[entry][u'vote_turnout']
passing = ballot[entry][u'vote_passing']
threshold = ballot[entry][u'vote_threshold']
if ballot[entry][u'previously_voted'] > 0:
direction = ballot[entry][u'previously_voted']
votewin.addstr(_y, x-1, '*', direction == 1 and C_GREEN or C_RED)
fmt_entry = "%-"+str(max_proposal_len + 2)+"s"
votewin.addstr(
_y,
x,
fmt_entry % entry,
passing and C_GREEN or threshold and C_RED or C_YELLOW)
for x in range(max_yeacount_len - len(str(yeas))):
votewin.addstr(' ')
votewin.addstr(str(yeas), C_GREEN)
votewin.addstr('/', C_CYAN)
votewin.addstr(str(nays), C_RED)
votewin.addstr(' ')
for x in range(max_naycount_len - len(str(nays))):
votewin.addstr(' ')
for x in range(max_percentage_len - len(str(percentage))):
votewin.addstr(' ')
votewin.addstr(str(percentage) + "%", C_CYAN)
votewin.addstr(' ')
votewin.addstr('ABSTAIN', C_YELLOW)
votewin.addstr(
_y + 2,
window_width + 7,
'confirm',
C_YELLOW)
votewin.move(0 + 6, window_width + 7)
votewin.refresh()
keys = {
113: lambda s: quit(),
curses.KEY_UP: lambda s: prev_vote(s),
curses.KEY_DOWN: lambda s: next_vote(s),
curses.KEY_RIGHT: lambda s: set_vote(ballot, s, 1),
curses.KEY_LEFT: lambda s: set_vote(ballot, s, -1),
107: lambda s: prev_vote(s),
106: lambda s: next_vote(s),
108: lambda s: set_vote(ballot, s, 1),
104: lambda s: set_vote(ballot, s, -1),
10: lambda s: submit_votes(stdscr, ballot, s)
}
sel_vote = 0
while True:
key = votewin.getch()
f = keys.get(key)
if hasattr(f, '__call__'):
sel_vote = f(sel_vote)
try:
entry_vote = ballot[ballot_entries[sel_vote]][u'vote']
except IndexError:
# CONFIRM button
entry_vote = ''
if key != 10:
update_vote_display(votewin, sel_vote, entry_vote)
0
Example 176
View licensedef main(_name_,_HBSurfaces,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVgen,_powerOutput,PVinverter,sandiaMode_):
""" This function is the heart of this component it takes all the component arguments and writes one PV generator onto each Honeybee surface connected to this component
Args:
The arguements seen in the function definition are the same as the arguements on the panel.
Returns:
The properties of the PV generators connected to this component these properties are then written to an IDF file in Honeybee_ Run Energy Simulation component.
"""
HBSurfacesfromhive = hb_hive.callFromHoneybeeHive(_HBSurfaces) # Call Honeybee surfaces from hive
PVgencount = 0
# CHECK that the same surfaces are not having PV generators added to them twice. If they are stop the component.
class myDict(dict):
def __init__(self):
self = dict()
def add(self, key, value):
self[key] = value
HBsurfaces = myDict()
for HBsurface in HBSurfacesfromhive:
# In order of Honeybee surface, Honeybee surface ID and Brep which corresponds to that Honeybee surface
HBsurfaces.add(HBsurface,HBsurface.ID)
if len(HBsurfaces.values()) != len(set(HBsurfaces.values())):
# Identify duplicateIDs
duplicateIDs = collections.Counter([item for item in HBsurfaces.values()]).keys()
surfaceCount = 0
for surface,surfaceID in HBsurfaces.items():
if surfaceID in duplicateIDs:
warn = "Into the input _HBSurfaces you have input the same surface " + HBsurface.name + " twice. Use Geometry preview to make sure \n"+\
"that the same surfaces are not being input several times. This component will not run until this is fixed."
ghenv.Component.AddRuntimeMessage(gh.GH_RuntimeMessageLevel.Error, warn)
# Draw a preview of the geometry so the user can identify it.
preview = Grasshopper.Kernel.GH_PreviewUtil()
preview.AddBrep(_HBSurfaces[surfaceCount])
surfaceCount =+ 1
return
try:
for name,surface,celleff,mode,parallel,series,modelcost,powerout in itertools.izip_longest(_name_,HBSurfacesfromhive,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVPerModule,_powerOutputPerModule):
surface.containsPVgen = True # Set this it true so the surface can be identified in run and write IDF component
surface.PVgenlist = [] # Set the PVgenlist of each surface back to empty otherwise PVgen objects will accumulate on each run
namePVperform = "PVperformance" + str(PVgencount)+ " " + str(surface.name) # Create a name for the PVperformance object for each PV generator - this is always created by automatically here not by the user
# Raise some warnings for the following circumstances
centerPt, normalVector = surface.getSrfCenPtandNormalAlternate()
if normalVector[2] == -1:
w = gh.GH_RuntimeMessageLevel.Warning
warn = "Are you sure that you want to mount PV generators on " + str(surface.name) +" \n"+\
"the surface normal of this surface is facing down!"
ghenv.Component.AddRuntimeMessage(w, warn)
if surface.type == 2.5:
w = gh.GH_RuntimeMessageLevel.Warning
warn = str(surface.name) +" is a on-ground slab you probably don't want to mount PV panels on it \n"+\
"but we will go ahead and do it anyway."
ghenv.Component.AddRuntimeMessage(w, warn)
if surface.type == 2.25:
w = gh.GH_RuntimeMessageLevel.Warning
warn = str(surface.name) +" is a underground slab you probably don't want to mount PV panels on it \n"+\
"but we will go ahead and do it anyway."
ghenv.Component.AddRuntimeMessage(w, warn)
if surface.type == 0.5:
w = gh.GH_RuntimeMessageLevel.Warning
warn = str(surface.name) +" is a underground wall you probably don't want to mount PV panels on it \n"+\
"but we will go ahead and do it anyway."
ghenv.Component.AddRuntimeMessage(w, warn)
try:
name = _name_[PVgencount]
message0 = _name_[PVgencount] + " is mounted on Honeybee surface "+ str(surface.name)
except IndexError:
name = "PVgenerator" + str(PVgencount)
message0 = "For this generator no name has been given so a default name of PVgenerator" + str(PVgencount) + " has been assigned. This generator is mounted on Honeybee surface " + str(surface.name) # If no name given for a PV generator assign one.
if sandiaMode_ == []:
try:
_cellsEfficiency[PVgencount]
message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
except IndexError:
celleff = _cellsEfficiency[0]
message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
try:
_integrationMode[PVgencount]
message3 = "The integration mode is " + returnmodename(mode)
except IndexError:
mode = _integrationMode[0]
message3 = "The integration mode is "+ returnmodename(mode)
try:
_NoParallel[PVgencount]
message4 = "The number of PV modules in parallel is "+ str(parallel)
except IndexError:
parallel = _NoParallel[0]
message4 = "The number of PV modules in parallel is "+ str(parallel)
try:
_Noseries[PVgencount]
message5 = "The number of PV modules in series is " + str(series)
except IndexError:
series = _Noseries[0]
message5 = "The number of PV modules in series is " + str(series)
message6 = "So " + name + " is made up of "+ str(series*parallel) + " PV modules"
try:
_costPVPerModule[PVgencount]
message7 = "The cost per PV module is " + str(modelcost) + " US dollars" + " \n " +\
"Therefore the total cost of "+ name +" is " + str(modelcost*float(parallel*series)) + " US dollars"
except IndexError:
modelcost = _costPVPerModule[0]
message7 = "The cost per PV module is " + str(_costPVPerModule[0]) + " \n " +\
"Therefore the total cost of "+ name + " is " + str(modelcost*float(parallel*series)) + " in US dollars "
try:
_powerOutputPerModule[PVgencount]
message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
"Therefore the total power output of "+ name+" is " + str(powerout*float(parallel*series)) + " W "
except IndexError:
powerout = _powerOutputPerModule[0]
message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
"Therefore the total power output of "+ name+ " is " + str(powerout*float(parallel*series)) + " W "
# The surface area of the surface that the PV panels cover are calculated from this equation.
if sandiaMode_ == []:
SA_solarcell = (powerout*parallel*series)/(1000*surface.getArea()*celleff)
if sandiaMode_ != []:
# Get Active Area from Sandria
for lineCount,line in enumerate(sandiaMode_):
if "!- Active Area {m2}" in line:
# [^.0-9] remove all non numeric characters from a string using Regex!
areaPerPanel = float(re.sub(r'[^.0-9]', "", line))
SA_solarcell = (parallel*series*areaPerPanel)/(surface.getArea())
if (SA_solarcell > 0.85) and (SA_solarcell < 1):
warn = "According to your inputs solar panels cover more than 85% of the surface area of the Honeybee surface "+ surface.name+ "\n"+\
"Are you sure that the power rating and the number of modules on this surface are correct? \n "+\
"This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on. \n"+ \
"Don't worry though this component will still run!"
w = gh.GH_RuntimeMessageLevel.Warning
ghenv.Component.AddRuntimeMessage(w, warn)
if SA_solarcell >= 1:
warn = "Error! According to your inputs solar panels cover more than 100% of the surface area of the Honeybee surface " + surface.name+"\n"+\
"Are you sure that the power rating and the number of modules on each surface is correct? \n "+\
"This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on."
"This component will not run until the surface area is less than 100%."
w = gh.GH_RuntimeMessageLevel.Error
ghenv.Component.AddRuntimeMessage(w, warn)
return
message1 = "According to the data you entered it is calculated that the modules of this PV generator cover roughly "+str(round(SA_solarcell*100,2))+ " % of the surface area of this surface"
# A hb_EPShdSurface
if surface.type == 6: ## A bit of a hack to get context surface name the same as being produced by EPShdSurface
coordinatesList = surface.extractPoints()
if type(coordinatesList[0])is not list and type(coordinatesList[0]) is not tuple: coordinatesList = [coordinatesList]
for count in range(len(coordinatesList)):
PVsurfacename = surface.name + '_' + `count`
surface.name = surface.name + '_' + `count`
surface.PVgenlist.append(PV_gen(name,PVsurfacename,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_)) # Last three inputs are for instance method PV_performance
# Not a hb_EPShdSurface
else:
surface.PVgenlist.append(PV_gen(name,surface.name,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_))
# Assign the inverter to each PVgenerator.
for PVgen in surface.PVgenlist:
PVgen.inverter = PVinverter
if sandiaMode_ == []:
message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +message2+"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
if sandiaMode_ != []:
message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
readmedatatree.Add(message,gh.Data.GH_Path(PVgencount))
PVgencount = PVgencount+1
except:
# This catches an error when there is a missing member exception ie length of one of inputs is longer than
# number of Honeybee surfaces not sure how to just catch missing member exception!
warn = "The length of a list of inputs into either _name_,_cellsEfficiency \n" + \
"_integrationMode,_NoParallel,_Noseries,_costPVgen or _powerOutput \n" + \
"is longer than the number of Honeybee surfaces connected to this component!\n" + \
"e.g if you have 2 Honeybee surfaces you cannot have 3 values input into _costPVgen!\n" + \
"Please check the inputs and try again!"
print warn
w = gh.GH_RuntimeMessageLevel.Warning
ghenv.Component.AddRuntimeMessage(w, warn)
return -1
ModifiedHBSurfaces = hb_hive.addToHoneybeeHive(HBSurfacesfromhive, ghenv.Component)
return ModifiedHBSurfaces
0
Example 177
View licensedef main(_name_,_HBSurfaces,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVgen,_powerOutput,PVinverter,sandiaMode_):
""" This function is the heart of this component it takes all the component arguments and writes one PV generator onto each Honeybee surface connected to this component
Args:
The arguements seen in the function definition are the same as the arguements on the panel.
Returns:
The properties of the PV generators connected to this component these properties are then written to an IDF file in Honeybee_ Run Energy Simulation component.
"""
HBSurfacesfromhive = hb_hive.callFromHoneybeeHive(_HBSurfaces) # Call Honeybee surfaces from hive
PVgencount = 0
# CHECK that the same surfaces are not having PV generators added to them twice. If they are stop the component.
class myDict(dict):
def __init__(self):
self = dict()
def add(self, key, value):
self[key] = value
HBsurfaces = myDict()
for HBsurface in HBSurfacesfromhive:
# In order of Honeybee surface, Honeybee surface ID and Brep which corresponds to that Honeybee surface
HBsurfaces.add(HBsurface,HBsurface.ID)
if len(HBsurfaces.values()) != len(set(HBsurfaces.values())):
# Identify duplicateIDs
duplicateIDs = collections.Counter([item for item in HBsurfaces.values()]).keys()
surfaceCount = 0
for surface,surfaceID in HBsurfaces.items():
if surfaceID in duplicateIDs:
warn = "Into the input _HBSurfaces you have input the same surface " + HBsurface.name + " twice. Use Geometry preview to make sure \n"+\
"that the same surfaces are not being input several times. This component will not run until this is fixed."
ghenv.Component.AddRuntimeMessage(gh.GH_RuntimeMessageLevel.Error, warn)
# Draw a preview of the geometry so the user can identify it.
preview = Grasshopper.Kernel.GH_PreviewUtil()
preview.AddBrep(_HBSurfaces[surfaceCount])
surfaceCount =+ 1
return
try:
for name,surface,celleff,mode,parallel,series,modelcost,powerout in itertools.izip_longest(_name_,HBSurfacesfromhive,_cellsEfficiency,_integrationMode,_NoParallel,_Noseries,_costPVPerModule,_powerOutputPerModule):
surface.containsPVgen = True # Set this it true so the surface can be identified in run and write IDF component
surface.PVgenlist = [] # Set the PVgenlist of each surface back to empty otherwise PVgen objects will accumulate on each run
namePVperform = "PVperformance" + str(PVgencount)+ " " + str(surface.name) # Create a name for the PVperformance object for each PV generator - this is always created by automatically here not by the user
# Raise some warnings for the following circumstances
centerPt, normalVector = surface.getSrfCenPtandNormalAlternate()
if normalVector[2] == -1:
w = gh.GH_RuntimeMessageLevel.Warning
warn = "Are you sure that you want to mount PV generators on " + str(surface.name) +" \n"+\
"the surface normal of this surface is facing down!"
ghenv.Component.AddRuntimeMessage(w, warn)
if surface.type == 2.5:
w = gh.GH_RuntimeMessageLevel.Warning
warn = str(surface.name) +" is a on-ground slab you probably don't want to mount PV panels on it \n"+\
"but we will go ahead and do it anyway."
ghenv.Component.AddRuntimeMessage(w, warn)
if surface.type == 2.25:
w = gh.GH_RuntimeMessageLevel.Warning
warn = str(surface.name) +" is a underground slab you probably don't want to mount PV panels on it \n"+\
"but we will go ahead and do it anyway."
ghenv.Component.AddRuntimeMessage(w, warn)
if surface.type == 0.5:
w = gh.GH_RuntimeMessageLevel.Warning
warn = str(surface.name) +" is a underground wall you probably don't want to mount PV panels on it \n"+\
"but we will go ahead and do it anyway."
ghenv.Component.AddRuntimeMessage(w, warn)
try:
name = _name_[PVgencount]
message0 = _name_[PVgencount] + " is mounted on Honeybee surface "+ str(surface.name)
except IndexError:
name = "PVgenerator" + str(PVgencount)
message0 = "For this generator no name has been given so a default name of PVgenerator" + str(PVgencount) + " has been assigned. This generator is mounted on Honeybee surface " + str(surface.name) # If no name given for a PV generator assign one.
if sandiaMode_ == []:
try:
_cellsEfficiency[PVgencount]
message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
except IndexError:
celleff = _cellsEfficiency[0]
message2 = "The solar cell efficiency is "+ str(celleff*100) + " %"
try:
_integrationMode[PVgencount]
message3 = "The integration mode is " + returnmodename(mode)
except IndexError:
mode = _integrationMode[0]
message3 = "The integration mode is "+ returnmodename(mode)
try:
_NoParallel[PVgencount]
message4 = "The number of PV modules in parallel is "+ str(parallel)
except IndexError:
parallel = _NoParallel[0]
message4 = "The number of PV modules in parallel is "+ str(parallel)
try:
_Noseries[PVgencount]
message5 = "The number of PV modules in series is " + str(series)
except IndexError:
series = _Noseries[0]
message5 = "The number of PV modules in series is " + str(series)
message6 = "So " + name + " is made up of "+ str(series*parallel) + " PV modules"
try:
_costPVPerModule[PVgencount]
message7 = "The cost per PV module is " + str(modelcost) + " US dollars" + " \n " +\
"Therefore the total cost of "+ name +" is " + str(modelcost*float(parallel*series)) + " US dollars"
except IndexError:
modelcost = _costPVPerModule[0]
message7 = "The cost per PV module is " + str(_costPVPerModule[0]) + " \n " +\
"Therefore the total cost of "+ name + " is " + str(modelcost*float(parallel*series)) + " in US dollars "
try:
_powerOutputPerModule[PVgencount]
message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
"Therefore the total power output of "+ name+" is " + str(powerout*float(parallel*series)) + " W "
except IndexError:
powerout = _powerOutputPerModule[0]
message8 = "The power output per PV module is " + str(powerout) + " W \n " +\
"Therefore the total power output of "+ name+ " is " + str(powerout*float(parallel*series)) + " W "
# The surface area of the surface that the PV panels cover are calculated from this equation.
if sandiaMode_ == []:
SA_solarcell = (powerout*parallel*series)/(1000*surface.getArea()*celleff)
if sandiaMode_ != []:
# Get Active Area from Sandria
for lineCount,line in enumerate(sandiaMode_):
if "!- Active Area {m2}" in line:
# [^.0-9] remove all non numeric characters from a string using Regex!
areaPerPanel = float(re.sub(r'[^.0-9]', "", line))
SA_solarcell = (parallel*series*areaPerPanel)/(surface.getArea())
if (SA_solarcell > 0.85) and (SA_solarcell < 1):
warn = "According to your inputs solar panels cover more than 85% of the surface area of the Honeybee surface "+ surface.name+ "\n"+\
"Are you sure that the power rating and the number of modules on this surface are correct? \n "+\
"This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on. \n"+ \
"Don't worry though this component will still run!"
w = gh.GH_RuntimeMessageLevel.Warning
ghenv.Component.AddRuntimeMessage(w, warn)
if SA_solarcell >= 1:
warn = "Error! According to your inputs solar panels cover more than 100% of the surface area of the Honeybee surface " + surface.name+"\n"+\
"Are you sure that the power rating and the number of modules on each surface is correct? \n "+\
"This value is calculated from the total power output of each PV generator (The sum of each PV module in that generator) divided by the surface area of the surface they are mounted on."
"This component will not run until the surface area is less than 100%."
w = gh.GH_RuntimeMessageLevel.Error
ghenv.Component.AddRuntimeMessage(w, warn)
return
message1 = "According to the data you entered it is calculated that the modules of this PV generator cover roughly "+str(round(SA_solarcell*100,2))+ " % of the surface area of this surface"
# A hb_EPShdSurface
if surface.type == 6: ## A bit of a hack to get context surface name the same as being produced by EPShdSurface
coordinatesList = surface.extractPoints()
if type(coordinatesList[0])is not list and type(coordinatesList[0]) is not tuple: coordinatesList = [coordinatesList]
for count in range(len(coordinatesList)):
PVsurfacename = surface.name + '_' + `count`
surface.name = surface.name + '_' + `count`
surface.PVgenlist.append(PV_gen(name,PVsurfacename,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_)) # Last three inputs are for instance method PV_performance
# Not a hb_EPShdSurface
else:
surface.PVgenlist.append(PV_gen(name,surface.name,returnmodename(mode),parallel,series,modelcost,powerout,namePVperform,SA_solarcell,celleff,sandiaMode_))
# Assign the inverter to each PVgenerator.
for PVgen in surface.PVgenlist:
PVgen.inverter = PVinverter
if sandiaMode_ == []:
message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +message2+"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
if sandiaMode_ != []:
message = message0 +" \n "+ name + " has the following properties: "+"\n "+message1+"\n " +"\n "+message3+"\n "+message4+"\n "+message5+"\n "+ message6 +"\n "+message7+"\n "+message8
readmedatatree.Add(message,gh.Data.GH_Path(PVgencount))
PVgencount = PVgencount+1
except:
# This catches an error when there is a missing member exception ie length of one of inputs is longer than
# number of Honeybee surfaces not sure how to just catch missing member exception!
warn = "The length of a list of inputs into either _name_,_cellsEfficiency \n" + \
"_integrationMode,_NoParallel,_Noseries,_costPVgen or _powerOutput \n" + \
"is longer than the number of Honeybee surfaces connected to this component!\n" + \
"e.g if you have 2 Honeybee surfaces you cannot have 3 values input into _costPVgen!\n" + \
"Please check the inputs and try again!"
print warn
w = gh.GH_RuntimeMessageLevel.Warning
ghenv.Component.AddRuntimeMessage(w, warn)
return -1
ModifiedHBSurfaces = hb_hive.addToHoneybeeHive(HBSurfacesfromhive, ghenv.Component)
return ModifiedHBSurfaces
0
Example 178
View license def main(self):
database_name = self.config.database_name
if not database_name:
self.config.logger.error(
'"database_name" cannot be an empty string'
)
return 1
# superuser credentials for overall database
superuser_pg_url = self.create_connection_url(
'postgres',
self.config.database_superusername,
self.config.database_superuserpassword
)
# superuser credentials for working database
superuser_normaldb_pg_url = self.create_connection_url(
database_name,
self.config.database_superusername,
self.config.database_superuserpassword
)
# normal user credentials
normal_user_pg_url = self.create_connection_url(
database_name,
self.config.database_username,
self.config.database_password
)
# ensure that if on Heroku the the normal_user_pg_url and the
# superuser_pg_url are the same
if self.config.on_heroku and (normal_user_pg_url != superuser_pg_url):
self.config.logger.error(
'there is no superuser (%s) when using Heroku',
self.config.database_superusername
)
return 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# table logging section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.unlogged:
@compiles(CreateTable)
def create_table(element, compiler, **kw):
text = compiler.visit_create_table(element, **kw)
text = re.sub("^\sCREATE(.*TABLE)",
lambda m: "CREATE UNLOGGED %s" %
m.group(1), text)
return text
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Postgres version check section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info(
'Postgres version check section with %s',
superuser_pg_url
)
with PostgreSQLAlchemyManager(
superuser_pg_url,
self.config.logger,
autocommit=False
) as db:
if not db.min_ver_check(90200):
self.config.logger.error(
'unrecognized PostgreSQL version: %s',
db.version_string()
)
self.config.logger.error('Only 9.2+ is supported at this time')
return 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# drop database section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# We can only do the following if the DB is not Heroku
# XXX Might add the heroku commands for resetting a DB here
if self.config.dropdb and not self.config.on_heroku:
self.config.logger.info(
'drop database section with %s',
superuser_pg_url
)
with PostgreSQLAlchemyManager(
superuser_pg_url,
self.config.logger,
autocommit=False
) as db:
if 'test' not in database_name and not self.config.force:
confirm = raw_input(
'drop database %s [y/N]: ' % database_name)
if not confirm == "y":
self.config.logger.warn('NOT dropping table')
return 2
db.drop_database(database_name)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# create database section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.createdb:
self.config.logger.info(
'create database section with %s',
superuser_pg_url
)
with PostgreSQLAlchemyManager(
superuser_pg_url,
self.config.logger,
autocommit=False
) as db:
db.create_database(database_name)
if self.config.no_roles:
self.config.logger.info("Skipping role creation")
else:
db.create_roles(self.config)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database extensions section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info(
'database extensions section with %s',
superuser_normaldb_pg_url
)
with PostgreSQLAlchemyManager(
superuser_normaldb_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku
) as db:
db.setup_extensions()
db.grant_public_schema_ownership(self.config.database_username)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database schema section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.no_schema:
self.config.logger.info("not adding a schema")
return 0
alembic_cfg = Config(self.config.alembic_config)
alembic_cfg.set_main_option('sqlalchemy.url', normal_user_pg_url)
self.config.logger.info(
'database schema section with %s',
normal_user_pg_url
)
with PostgreSQLAlchemyManager(
normal_user_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku
) as db:
# Order matters below
db.turn_function_body_checks_off()
db.load_raw_sql('types')
db.load_raw_sql('procs')
# We need to commit to make a type visible for table creation
db.commit()
db.create_tables()
db.load_raw_sql('views')
db.commit()
if not self.config.get('no_staticdata'):
self.import_staticdata(db)
if self.config['fakedata']:
self.generate_fakedata(db, self.config['fakedata_days'])
db.commit()
command.stamp(alembic_cfg, "heads")
db.session.close()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database owner section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info(
'database extensions section with %s',
superuser_normaldb_pg_url
)
with PostgreSQLAlchemyManager(
superuser_normaldb_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku
) as db:
db.set_table_owner(self.config.database_username)
db.set_default_owner(database_name, self.config.database_username)
db.set_grants(self.config) # config has user lists
return 0
0
Example 179
View license def main(self):
database_name = self.config.database_name
if not database_name:
self.config.logger.error(
'"database_name" cannot be an empty string'
)
return 1
# superuser credentials for overall database
superuser_pg_url = self.create_connection_url(
'postgres',
self.config.database_superusername,
self.config.database_superuserpassword
)
# superuser credentials for working database
superuser_normaldb_pg_url = self.create_connection_url(
database_name,
self.config.database_superusername,
self.config.database_superuserpassword
)
# normal user credentials
normal_user_pg_url = self.create_connection_url(
database_name,
self.config.database_username,
self.config.database_password
)
# ensure that if on Heroku the the normal_user_pg_url and the
# superuser_pg_url are the same
if self.config.on_heroku and (normal_user_pg_url != superuser_pg_url):
self.config.logger.error(
'there is no superuser (%s) when using Heroku',
self.config.database_superusername
)
return 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# table logging section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.unlogged:
@compiles(CreateTable)
def create_table(element, compiler, **kw):
text = compiler.visit_create_table(element, **kw)
text = re.sub("^\sCREATE(.*TABLE)",
lambda m: "CREATE UNLOGGED %s" %
m.group(1), text)
return text
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Postgres version check section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info(
'Postgres version check section with %s',
superuser_pg_url
)
with PostgreSQLAlchemyManager(
superuser_pg_url,
self.config.logger,
autocommit=False
) as db:
if not db.min_ver_check(90200):
self.config.logger.error(
'unrecognized PostgreSQL version: %s',
db.version_string()
)
self.config.logger.error('Only 9.2+ is supported at this time')
return 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# drop database section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# We can only do the following if the DB is not Heroku
# XXX Might add the heroku commands for resetting a DB here
if self.config.dropdb and not self.config.on_heroku:
self.config.logger.info(
'drop database section with %s',
superuser_pg_url
)
with PostgreSQLAlchemyManager(
superuser_pg_url,
self.config.logger,
autocommit=False
) as db:
if 'test' not in database_name and not self.config.force:
confirm = raw_input(
'drop database %s [y/N]: ' % database_name)
if not confirm == "y":
self.config.logger.warn('NOT dropping table')
return 2
db.drop_database(database_name)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# create database section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.createdb:
self.config.logger.info(
'create database section with %s',
superuser_pg_url
)
with PostgreSQLAlchemyManager(
superuser_pg_url,
self.config.logger,
autocommit=False
) as db:
db.create_database(database_name)
if self.config.no_roles:
self.config.logger.info("Skipping role creation")
else:
db.create_roles(self.config)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database extensions section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info(
'database extensions section with %s',
superuser_normaldb_pg_url
)
with PostgreSQLAlchemyManager(
superuser_normaldb_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku
) as db:
db.setup_extensions()
db.grant_public_schema_ownership(self.config.database_username)
db.commit()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database schema section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if self.config.no_schema:
self.config.logger.info("not adding a schema")
return 0
alembic_cfg = Config(self.config.alembic_config)
alembic_cfg.set_main_option('sqlalchemy.url', normal_user_pg_url)
self.config.logger.info(
'database schema section with %s',
normal_user_pg_url
)
with PostgreSQLAlchemyManager(
normal_user_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku
) as db:
# Order matters below
db.turn_function_body_checks_off()
db.load_raw_sql('types')
db.load_raw_sql('procs')
# We need to commit to make a type visible for table creation
db.commit()
db.create_tables()
db.load_raw_sql('views')
db.commit()
if not self.config.get('no_staticdata'):
self.import_staticdata(db)
if self.config['fakedata']:
self.generate_fakedata(db, self.config['fakedata_days'])
db.commit()
command.stamp(alembic_cfg, "heads")
db.session.close()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# database owner section
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.config.logger.info(
'database extensions section with %s',
superuser_normaldb_pg_url
)
with PostgreSQLAlchemyManager(
superuser_normaldb_pg_url,
self.config.logger,
autocommit=False,
on_heroku=self.config.on_heroku
) as db:
db.set_table_owner(self.config.database_username)
db.set_default_owner(database_name, self.config.database_username)
db.set_grants(self.config) # config has user lists
return 0
0
Example 180
View license def __init__(self, meta, name, data, bases):
"""
Initializes the options object and defaults configuration not
specified.
@param[in] meta
Dictionary of the merged meta attributes.
@param[in] name
Name of the resource class this is being instantiataed for.
"""
#! Whether to allow display of debugging information
#! to the client.
self.debug = meta.get('debug')
if self.debug is None:
self.debug = False
#! Whether to not actualize a resource from the described class.
#! Abstract resources are meant as generic base classes.
#!
#! @note
#! Abstract only counts if it is directly set on the resource.
self.abstract = data.get('abstract')
#! Name of the resource to use in URIs; defaults to the dasherized
#! version of the camel cased class name (eg. SomethingHere becomes
#! something-here). The defaulted version also strips a trailing
#! Resource from the name (eg. SomethingHereResource still becomes
#! something-here).
self.name = meta.get('name')
if self.name is None:
# Generate a name for the resource if one is not provided.
# PascalCaseThis => pascal-case-this
dashed = utils.dasherize(name).strip()
if dashed:
# Strip off a trailing Resource as well.
self.name = re.sub(r'-resource$', '', dashed)
else:
# Something went wrong; just use the class name
self.name = name
elif callable(self.name):
# If name is callable; pass in the class name and use what
# we got back.
self.name = self.name(name)
#! True if the resource is expected to operate asynchronously.
#!
#! The side-effect of setting this to True is that returning from
#! `dispatch` (and by extension `get`, `post`, etc.) does not
#! terminate the connection. You must invoke `response.close()` to
#! terminate the connection.
self.asynchronous = meta.get('asynchronous', False)
#! Connectors to use to connect to the environment.
#!
#! This is a dictionary that maps hooks (keys) to the connector to use
#! for the hook.
#!
#! There is only 1 hook available currently, 'http', and it is
#! required.
#!
#! The available connectors are as follows:
#! - http:
#! > django
#! > flask
#!
#! They may be used as follows:
#!
#! @code
#! from armet import resources
#! class Resource(resources.Resource):
#! class Meta:
#! connectors = {'http': 'django'}
#! @endcode
#!
#! Connectors may also be specified as full paths to the connector
#! module (if a connector is located somewhere other than
#! armet.connectors) as follows:
#!
#! @code
#! from armet import resources
#! class Resource(resources.Resource):
#! class Meta:
#! connectors = {'http': 'some.other.place'}
#! @endcode
self.connectors = connectors = _merge(meta, 'connectors', bases, {})
if not connectors.get('http') and not self.abstract:
raise ImproperlyConfigured('No valid HTTP connector was detected.')
# Pull out the connectors and convert them into module references.
for key in connectors:
connector = connectors[key]
if isinstance(connector, six.string_types):
if connector in getattr(included_connectors, key):
# Included shortname; prepend base.
connectors[key] = 'armet.connectors.{}'.format(connector)
#! Additional options to handle and merge into the meta object
#! at the class object level.
#!
#! This should be a simple set/list/tuple of options.
#!
#! @code
#! from armet import resources
#! class Resource(resources.Resource):
#! class Meta:
#! options = {'color', 'plant'}
#!
#! @endcode
self.options = options = _merge(meta, 'options', bases, {})
for name in options:
# Pull out each option and stick it on the meta.
setattr(self, name, meta.get(name))
#! Regular-expression patterns to apply to the request path
#! and pull arguments and traversal out of it.
#!
#! @code
#! from armet import resources
#! class Resource(resources.Resource):
#! class Meta:
#! patterns = [
#! # Match nothing after the resource name.
#! r'^$',
#!
#! # Match a slug after the resource name.
#! r'^/(?P<slug>[^/]+)/?$',
#! ]
#!
#! @endcode
#!
#! Named parameters get auto-attached to `self` on the resource
#! instance.
#!
#! This may be a list of lists/tuples as well to indicate simple
#! traversal.
#!
#! @code
#! from armet import resources
#! class Resource(resources.Resource):
#! class Meta:
#! patterns = [
#! # Match nothing and don't traverse.
#! (None, r'^$'),
#!
#! # Match the word 'user' and traverse.
#! # The remainder of the path is taken or
#! # the named gruop, "path" (if the whole path was
#! # matched) or the last-matched group (if none are
#! # named "path")
#! (UserResource, r'^/user'),
#! ]
#!
#! @endcode
self.patterns = meta.get('patterns', [])
for index, pattern in enumerate(self.patterns):
# Coerce simple form.
if isinstance(pattern, six.string_types):
pattern = (None, pattern)
# Compile the expression.
self.patterns[index] = (pattern[0], re.compile(pattern[1]))
#! Trailing slash handling.
#! The value indicates which URI is the canonical URI and the
#! alternative URI is then made to redirect (with a 301) to the
#! canonical URI.
self.trailing_slash = meta.get('trailing_slash', True)
#! List of allowed HTTP methods.
#! If not provided and allowed_operations was provided instead
#! the operations are appropriately mapped; else, the default
#! configuration is provided.
self.http_allowed_methods = meta.get('http_allowed_methods')
if self.http_allowed_methods is None:
self.http_allowed_methods = (
'HEAD',
'OPTIONS',
'GET',
'POST',
'PUT',
'PATCH',
'DELETE',
'LINK',
'UNLINK'
)
#! List of allowed HTTP headers.
#! This is used only to request or prevent CORS requests.
self.http_allowed_headers = meta.get('http_allowed_headers')
if self.http_allowed_headers is None:
self.http_allowed_headers = (
'Content-Type',
'Authorization',
'Accept',
'Origin'
)
#! List of exposed HTTP headers.
#! This is used only to show headers to the client.
self.http_exposed_headers = meta.get('http_exposed_headers')
if self.http_exposed_headers is None:
self.http_exposed_headers = (
'Content-Type',
'Authorization',
'Accept',
'Origin'
)
#! List of allowed HTTP origins.
#! This is used to request or prevent CORS requests.
#! No CORS requests will be allowed, at-all, unless this
#! property is set.
#! NOTE: This can be set to '*' to indicate any origin.
self.http_allowed_origins = meta.get('http_allowed_origins')
if self.http_allowed_origins is None:
self.http_allowed_origins = ()
#! Whether to use legacy redirects or not to inform the
#! client the resource is available elsewhere. Legacy redirects
#! require a combination of 301 and 307 in which 307 is not cacheable.
#! Modern redirecting uses 308 and is in effect 307 with cacheing.
#! Unfortunately unknown 3xx codes are treated as
#! a 300 (Multiple choices) in which the user is supposed to chose
#! the alternate link so the client is not supposed to auto follow
#! redirects. Ensure all supported clients understand 308 before
#! turning off legacy redirecting.
#! As of 19 March 2013 only Firefox supports it since a year ago.
self.legacy_redirect = meta.get('legacy_redirect', True)
#! Mapping of serializers known by this resource.
#! Values may either be a string reference to the serializer type
#! or an serializer class object.
self.serializers = serializers = meta.get('serializers')
if not serializers:
self.serializers = {
'json': 'armet.serializers.JSONSerializer',
'url': 'armet.serializers.URLSerializer'
}
# Expand the serializer name references.
for name, serializer in six.iteritems(self.serializers):
if isinstance(serializer, six.string_types):
segments = serializer.split('.')
module = '.'.join(segments[:-1])
module = import_module(module)
self.serializers[name] = getattr(module, segments[-1])
#! List of allowed serializers of the understood serializers.
self.allowed_serializers = meta.get('allowed_serializers')
if not self.allowed_serializers:
self.allowed_serializers = tuple(self.serializers.keys())
# Check to ensure that all allowed serializers are
# understood serializers.
for name in self.allowed_serializers:
if name not in self.serializers:
raise ImproperlyConfigured(
'The allowed serializer, {}, is not one of the '
'understood serializers'.format(name))
#! Name of the default serializer of the list of
#! understood serializers.
self.default_serializer = meta.get('default_serializer')
if not self.default_serializer:
if 'json' in self.allowed_serializers:
self.default_serializer = 'json'
else:
self.default_serializer = self.allowed_serializers[0]
if self.default_serializer not in self.allowed_serializers:
raise ImproperlyConfigured(
'The chosen default serializer, {}, is not one of the '
'allowed serializers'.format(self.default_serializer))
#! Mapping of deserializers known by this resource.
#! Values may either be a string reference to the deserializer type
#! or an deserializer class object.
self.deserializers = deserializers = meta.get('deserializers')
if not deserializers:
self.deserializers = {
'json': 'armet.deserializers.JSONDeserializer',
'url': 'armet.deserializers.URLDeserializer'
}
# Expand the deserializer name references.
for name, deserializer in six.iteritems(self.deserializers):
if isinstance(deserializer, six.string_types):
segments = deserializer.split('.')
module = '.'.join(segments[:-1])
module = import_module(module)
self.deserializers[name] = getattr(module, segments[-1])
#! List of allowed deserializers of the understood deserializers.
self.allowed_deserializers = meta.get('allowed_deserializers')
if not self.allowed_deserializers:
self.allowed_deserializers = tuple(self.deserializers.keys())
# Check to ensure that all allowed deserializers are
# understood deserializers.
for name in self.allowed_deserializers:
if name not in self.deserializers:
raise ImproperlyConfigured(
'The allowed deserializer, {}, is not one of the '
'understood deserializers'.format(name))
#! List of authentication protocols to attempt in sequence
#! to determine the authenticated user.
self.authentication = meta.get('authentication')
if self.authentication is None:
# Default to a single instance of pass-through authentication.
self.authentication = (authentication.Authentication(),)
#! The authorization protocol to attempt to use
#! to determine the if the user can access or is otherwise
#! authorized.
self.authorization = meta.get('authorization')
if self.authorization is None:
# Default is the pass-through authorization.
self.authorization = authorization.Authorization()
0
Example 181
View license def handle_label(self, filename, **options):
missing_options = []
for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']:
if options[k]:
continue
else:
missing_options.append(k)
if missing_options:
message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument "
message = message_start + " ".join('--{0}'.format(k) for k in missing_options)
raise CommandError(message)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
override_name = options['override_name']
name_field = options['name_field']
if not (override_name or name_field):
name_field = 'Name'
override_code = options['override_code']
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if name_field and override_name:
raise CommandError("You must not specify both --name_field and --override_name")
if code_field and override_code:
raise CommandError("You must not specify both --code_field and --override_code")
using_code = (code_field or override_code)
if (using_code and not code_type_code) or (not using_code and code_type_code):
raise CommandError(
"If you want to save a code, specify --code_type and either --code_field or --override_code")
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = input('Please give a description for area type code %s: ' % area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']:
area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = input('Please give a description for name type code %s: ' % name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']:
name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = input('Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']:
country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = input('Please give a description for code type %s: ' % code_type_code)
code_type = CodeType(code=code_type_code, description=code_desc)
if options['commit']:
code_type.save()
self.stdout.write("Importing from %s" % filename)
if not options['commit']:
self.stdout.write('(will not save to db as --commit not specified)')
current_generation = Generation.objects.current()
new_generation = Generation.objects.get(id=generation_id)
def verbose(*args):
if int(options['verbosity']) > 1:
self.stdout.write(" ".join(str(a) for a in args))
ds = DataSource(filename)
layer = ds[0]
if (override_name or override_code) and len(layer) > 1:
message = (
"Warning: you have specified an override %s and this file contains more than one feature; "
"multiple areas with the same %s will be created")
if override_name:
self.stdout.write(message % ('name', 'name'))
if override_code:
self.stdout.write(message % ('code', 'code'))
for feat in layer:
if override_name:
name = override_name
else:
name = None
for nf in name_field.split(','):
try:
name = feat[nf].value
break
except:
pass
if name is None:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find name using name field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --name_field" % (name_field, choices))
try:
if not isinstance(name, six.text_type):
name = name.decode(encoding)
except:
raise CommandError(
"Could not decode name using encoding '%s' - is it in another encoding? "
"Specify one with --encoding" % encoding)
name = re.sub('\s+', ' ', name)
if not name:
if options['ignore_blank']:
continue
raise Exception("Could not find a name to use for area")
code = None
if override_code:
code = override_code
elif code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find code using code field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --code_field" % (code_field, choices))
self.stdout.write(" looking at '%s'%s" % (name, (' (%s)' % code) if code else ''))
g = None
if hasattr(feat, 'geom'):
g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)
try:
if options['new']: # Always want a new area
raise Area.DoesNotExist
if code:
matching_message = "code %s of code type %s" % (code, code_type)
areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
else:
matching_message = "name %s of area type %s" % (name, area_type)
areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
if len(areas) == 0:
verbose(" the area was not found - creating a new one")
raise Area.DoesNotExist
m = areas[0]
verbose(" found the area")
if options['preserve']:
# Find whether we need to create a new Area:
previous_geos_geometry = m.polygons.collect()
if m.generation_high < current_generation.id:
# Then it was missing in current_generation:
verbose(" area existed previously, but was missing from", current_generation)
raise Area.DoesNotExist
elif g is None:
if previous_geos_geometry is not None:
verbose(" area is now empty")
raise Area.DoesNotExist
else:
verbose(" the area has remained empty")
elif previous_geos_geometry is None:
# It was empty in the previous generation:
verbose(" area was empty in", current_generation)
raise Area.DoesNotExist
else:
# Otherwise, create a new Area unless the
# polygons were the same in current_generation:
previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
new_geos_geometry = g.geos.simplify(tolerance=0)
create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
verbose(" change in area is:", "%.03f%%" % (100 * p,))
if create_new_area:
verbose(" the area", m, "has changed, creating a new area due to --preserve")
raise Area.DoesNotExist
else:
verbose(" the area remained the same")
else:
# If --preserve is not specified, the code or the name must be unique:
if len(areas) > 1:
raise Area.MultipleObjectsReturned(
"There was more than one area with %s, and --preserve was not specified" % (
matching_message,))
except Area.DoesNotExist:
m = Area(
name=name,
type=area_type,
country=country,
# parent_area=parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['fix_invalid_polygons'] and g is not None:
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g] if g is not None else []
if options['commit']:
m.save()
m.names.update_or_create(type=name_type, defaults={'name': name})
if code:
m.codes.update_or_create(type=code_type, defaults={'code': code})
save_polygons({m.id: (m, poly)})
0
Example 182
View license def handle_label(self, filename, **options):
missing_options = []
for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']:
if options[k]:
continue
else:
missing_options.append(k)
if missing_options:
message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument "
message = message_start + " ".join('--{0}'.format(k) for k in missing_options)
raise CommandError(message)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
override_name = options['override_name']
name_field = options['name_field']
if not (override_name or name_field):
name_field = 'Name'
override_code = options['override_code']
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if name_field and override_name:
raise CommandError("You must not specify both --name_field and --override_name")
if code_field and override_code:
raise CommandError("You must not specify both --code_field and --override_code")
using_code = (code_field or override_code)
if (using_code and not code_type_code) or (not using_code and code_type_code):
raise CommandError(
"If you want to save a code, specify --code_type and either --code_field or --override_code")
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = input('Please give a description for area type code %s: ' % area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']:
area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = input('Please give a description for name type code %s: ' % name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']:
name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = input('Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']:
country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = input('Please give a description for code type %s: ' % code_type_code)
code_type = CodeType(code=code_type_code, description=code_desc)
if options['commit']:
code_type.save()
self.stdout.write("Importing from %s" % filename)
if not options['commit']:
self.stdout.write('(will not save to db as --commit not specified)')
current_generation = Generation.objects.current()
new_generation = Generation.objects.get(id=generation_id)
def verbose(*args):
if int(options['verbosity']) > 1:
self.stdout.write(" ".join(str(a) for a in args))
ds = DataSource(filename)
layer = ds[0]
if (override_name or override_code) and len(layer) > 1:
message = (
"Warning: you have specified an override %s and this file contains more than one feature; "
"multiple areas with the same %s will be created")
if override_name:
self.stdout.write(message % ('name', 'name'))
if override_code:
self.stdout.write(message % ('code', 'code'))
for feat in layer:
if override_name:
name = override_name
else:
name = None
for nf in name_field.split(','):
try:
name = feat[nf].value
break
except:
pass
if name is None:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find name using name field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --name_field" % (name_field, choices))
try:
if not isinstance(name, six.text_type):
name = name.decode(encoding)
except:
raise CommandError(
"Could not decode name using encoding '%s' - is it in another encoding? "
"Specify one with --encoding" % encoding)
name = re.sub('\s+', ' ', name)
if not name:
if options['ignore_blank']:
continue
raise Exception("Could not find a name to use for area")
code = None
if override_code:
code = override_code
elif code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find code using code field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --code_field" % (code_field, choices))
self.stdout.write(" looking at '%s'%s" % (name, (' (%s)' % code) if code else ''))
g = None
if hasattr(feat, 'geom'):
g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)
try:
if options['new']: # Always want a new area
raise Area.DoesNotExist
if code:
matching_message = "code %s of code type %s" % (code, code_type)
areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
else:
matching_message = "name %s of area type %s" % (name, area_type)
areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
if len(areas) == 0:
verbose(" the area was not found - creating a new one")
raise Area.DoesNotExist
m = areas[0]
verbose(" found the area")
if options['preserve']:
# Find whether we need to create a new Area:
previous_geos_geometry = m.polygons.collect()
if m.generation_high < current_generation.id:
# Then it was missing in current_generation:
verbose(" area existed previously, but was missing from", current_generation)
raise Area.DoesNotExist
elif g is None:
if previous_geos_geometry is not None:
verbose(" area is now empty")
raise Area.DoesNotExist
else:
verbose(" the area has remained empty")
elif previous_geos_geometry is None:
# It was empty in the previous generation:
verbose(" area was empty in", current_generation)
raise Area.DoesNotExist
else:
# Otherwise, create a new Area unless the
# polygons were the same in current_generation:
previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
new_geos_geometry = g.geos.simplify(tolerance=0)
create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
verbose(" change in area is:", "%.03f%%" % (100 * p,))
if create_new_area:
verbose(" the area", m, "has changed, creating a new area due to --preserve")
raise Area.DoesNotExist
else:
verbose(" the area remained the same")
else:
# If --preserve is not specified, the code or the name must be unique:
if len(areas) > 1:
raise Area.MultipleObjectsReturned(
"There was more than one area with %s, and --preserve was not specified" % (
matching_message,))
except Area.DoesNotExist:
m = Area(
name=name,
type=area_type,
country=country,
# parent_area=parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['fix_invalid_polygons'] and g is not None:
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g] if g is not None else []
if options['commit']:
m.save()
m.names.update_or_create(type=name_type, defaults={'name': name})
if code:
m.codes.update_or_create(type=code_type, defaults={'code': code})
save_polygons({m.id: (m, poly)})
0
Example 183
View licensedef process_office(office, commit, start_date, end_date, na_member_lookup, geocode_cache):
global locationsnotfound, personnotfound
# Ensure that all the required kinds and other objects exist:
ok_constituency_office, _ = OrganisationKind.objects.get_or_create(
slug='constituency-office',
name='Constituency Office')
ok_constituency_area, _ = OrganisationKind.objects.get_or_create(
slug='constituency-area',
name='Constituency Area')
pk_constituency_office, _ = PlaceKind.objects.get_or_create(
slug='constituency-office',
name='Constituency Office')
pk_constituency_area, _ = PlaceKind.objects.get_or_create(
slug='constituency-area',
name='Constituency Area')
constituency_kinds = {
'area': ok_constituency_area,
'office': ok_constituency_office
}
constituency_place_kinds = {
'area': pk_constituency_area,
'office': pk_constituency_office
}
ck_address, _ = ContactKind.objects.get_or_create(
slug='address',
name='Address')
ck_postal_address, _ = ContactKind.objects.get_or_create(
slug='postal_address',
name='Postal Address')
ck_email, _ = ContactKind.objects.get_or_create(
slug='email',
name='Email')
ck_fax, _ = ContactKind.objects.get_or_create(
slug='fax',
name='Fax')
ck_telephone, _ = ContactKind.objects.get_or_create(
slug='voice',
name='Voice')
ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
name='has_office'
)
pt_constituency_contact, _ = PositionTitle.objects.get_or_create(
slug='constituency-contact',
name='Constituency Contact')
pt_administrator, _ = PositionTitle.objects.get_or_create(
slug='administrator',
name='Administrator')
pt_administrator_volunteer, _ = PositionTitle.objects.get_or_create(
slug='administrator-volunteer',
name='Administrator (volunteer)')
pt_volunteer, _ = PositionTitle.objects.get_or_create(
slug='volunteer',
name='Volunteer')
pt_coordinator, _ = PositionTitle.objects.get_or_create(
slug='coordinator',
name='Coordinator')
pt_community_development_field_worker, _ = PositionTitle.objects.get_or_create(
slug='community-development-field-worker',
name='Community Development Field Worker')
position_titles = {
'Constituency Contact': pt_constituency_contact,
'Administrator': pt_administrator,
'Administrator (volunteer)': pt_administrator_volunteer,
'Volunteer': pt_volunteer,
'Coordinator': pt_coordinator,
'Community Development Field Worker': pt_community_development_field_worker}
ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
name='has_office')
contact_kinds = {
'E-mail': ck_email,
'Tel': ck_telephone,
'Fax': ck_fax,
'Physical Address': ck_address,
'Postal Address': ck_postal_address
}
print "\n", office['Title']
infosources = []
if 'Sources' in office:
source_url = office['Sources'][0]['Source URL']
first = True
for source in office['Sources']:
print 'Adding InformationSource %s (%s)' % (
source['Source URL'],
source['Source Note']
)
infosources.append({
'source_url': source['Source URL'],
'source_note': source['Source Note']
})
if first:
first = False
continue
source_url += ' and ' + source['Source URL']
else:
source_url = office['Source URL']
source_note = office['Source Note']
infosources.append({
'source_url': source_url,
'source_note': source_note
})
print 'Adding InformationSource %s (%s)' % (source_url, source_note)
if ('Physical Address' in office) and (not 'South Africa' in office['Physical Address']) and (office['Physical Address']!='TBA'):
office['Physical Address'] = office['Physical Address'] + ', South Africa'
if ('Location' in office) and (office['Location']!='TBA') and (not office['Province'].lower() in office['Location'].lower()):
office['Location'] = office['Location'] + ', ' + office['Province'] + ', South Africa'
if ('Location' in office) and (not 'South Africa' in office['Location']) and (office['Location']!='TBA'):
office['Location'] = office['Location'] + ', South Africa'
#first determine whether the office already exists
organisation = None
try:
organisation = Organisation.objects.get(
name=office['Title']
)
except ObjectDoesNotExist:
#check identifiers
try:
if 'identifiers' in office:
for identifier_scheme, party_code in office['identifiers'].items():
identifier = Identifier.objects.get(
identifier=party_code,
scheme=identifier_scheme)
organisation = identifier.content_object
except ObjectDoesNotExist:
pass
if organisation: #existing office
if organisation.name != office['Title']:
print 'Changing name from %s to %s' % (organisation.name, office['Title'])
if commit:
organisation.name = office['Title']
organisation.save()
if organisation.ended != 'future':
print 'Changing ended date from %s to future' % (organisation.ended)
if commit:
organisation.ended = 'future'
organisation.save()
else:
print 'Creating new %s' % (office['Type'])
if commit:
organisation = Organisation.objects.create(
name=office['Title'],
slug=slugify(office['Title']),
kind=constituency_kinds[office['Type']],
started=start_date,
ended='future')
#information source
if commit:
for infosource in infosources:
InformationSource.objects.get_or_create(
source = infosource['source_url'],
note = infosource['source_note'],
entered = True,
content_type=organisation_content_type,
object_id=organisation.id
)
#relationship to party
try:
party = Organisation.objects.get(slug=office['Party'].lower())
OrganisationRelationship.objects.get(
organisation_a=party,
organisation_b=organisation,
kind=ork_has_office
)
#if the relationship exists nothing needs to change
print 'Retaining relationship with %s' % (party)
except (ObjectDoesNotExist, AttributeError):
print 'Adding relationship with %s' % (party)
if commit:
OrganisationRelationship.objects.create(
organisation_a=party,
organisation_b=organisation,
kind=ork_has_office
)
office_fields = [
'E-mail',
'Tel',
'Fax',
'Physical Address',
'Postal Address'
]
for field in office_fields:
if field in office:
try:
if not organisation:
raise ObjectDoesNotExist
contact = Contact.objects.get(
object_id=organisation.id,
content_type=organisation_content_type,
kind=contact_kinds[field])
if office[field] != contact.value:
print 'Changing %s from %s to %s' % (field, contact.value, office[field])
if commit:
contact.value = office[field]
contact.save()
print 'Updating contact source to %s' % (source_url)
if commit:
contact.source = source_url
contact.save()
except ObjectDoesNotExist:
print 'Creating new contact (%s: %s)' % (field, office[field])
if commit:
Contact.objects.create(
object_id=organisation.id,
content_type=organisation_content_type,
kind=contact_kinds[field],
value=office[field],
source=source_url)
else:
try:
if not organisation:
raise ObjectDoesNotExist
contact = Contact.objects.get(
object_id=organisation.id,
content_type=organisation_content_type,
kind=contact_kinds[field])
print 'Deleting', contact
if commit:
contact.delete()
except ObjectDoesNotExist:
pass
if 'Municipality' in office:
mapit_municipality = get_mapit_municipality(office['Municipality'], office.get('Province', ''))
if mapit_municipality:
place_name = u'Municipality associated with ' + office['Title']
try:
if not organisation:
raise ObjectDoesNotExist
place = Place.objects.get(
name__startswith=u'Municipality associated with ',
organisation=organisation)
if place.name != place_name:
'Changing municipality association name from %s to %s' % (place.name, place_name)
if commit:
place.name = place_name
place.save()
if place.mapit_area != mapit_municipality:
print 'Changing municipality mapit association from %s to %s' % (place.mapit_area, mapit_municipality)
if commit:
place.mapit_area = mapit_municipality
place.save()
except ObjectDoesNotExist:
print 'Create municipality association'
to_add = {
'name': place_name,
'slug': slugify(place_name),
'kind': constituency_place_kinds[office['Type']],
'mapit_area': mapit_municipality,}
print to_add
if commit:
Place.objects.create(
name=to_add['name'],
slug=to_add['slug'],
kind=to_add['kind'],
mapit_area=to_add['mapit_area'],
organisation=organisation)
if 'manual_lonlat' in office or 'Physical Address' in office or 'Location' in office:
reference_location = ''
try:
if 'manual_lonlat' in office:
#FIXME implement
print 'manual'
elif 'Location' in office:
reference_location = office['Location']
lon, lat, geocode_cache = geocode(office['Location'], geocode_cache, VERBOSE)
elif 'Physical Address' in office:
reference_location = office['Physical Address']
#geocode physical address
lon, lat, geocode_cache = geocode(office['Physical Address'], geocode_cache, VERBOSE)
location = Point(lon, lat)
if office['Type']=='area':
name = u'Unknown sub-area of %s known as %s' % (office['Province'], office['Title'])
else:
name = u'Approximate position of ' + office['Title']
try:
if not organisation:
raise ObjectDoesNotExist
if office['Type'] == 'area':
place = Place.objects.get(
name__startswith=u'Unknown sub-area of',
organisation=organisation)
else:
place = Place.objects.get(
name__startswith=u'Approximate position of ',
organisation=organisation)
if place.location != location:
print 'Changing location from %s to %s' % (place.location, location)
debug_location_change(place.location, location)
if commit:
place.location = location
place.save()
if place.name != name:
print 'Changing location name from %s to %s' % (place.name, name)
if commit:
place.name = name
place.save()
except ObjectDoesNotExist:
print 'Create constituency location'
if commit:
Place.objects.create(
name=name,
slug=slugify(name),
organisation=organisation,
location=location,
kind=constituency_place_kinds[office['Type']])
except LocationNotFound:
locationsnotfound.append([office['Title'], reference_location])
print "XXX no results found for: " + reference_location
else:
print 'No office/area location specified'
people_to_keep = []
if 'People' in office:
for person in office['People']:
#person matching needs to be improved - for now attempt
#find_pombola_person (from
#south_africa_import_constituency_offices command) otherwise
#direct match.
pombola_person = find_pombola_person(person['Name'], na_member_lookup, VERBOSE)
if not pombola_person:
#use filter().distinct() instead of get due to multiple
#rows being returned
pombola_person = Person.objects.filter(
Q(legal_name=person['Name']) |
Q(alternative_names__alternative_name=person['Name'])).distinct()
if len(pombola_person)==0:
pombola_person = None
else:
pombola_person = pombola_person[0]
#check person currently holds office
accept_person = True
if pombola_person and person['Position']=='Constituency Contact':
position_check = Position.objects.filter(
person=pombola_person,
organisation__kind__slug__in=['parliament', 'provincial-legislature'])
if len(position_check)==0:
accept_person=False
print '%s is not an MP or MPL' % (pombola_person.name)
if pombola_person and accept_person:
#check if the position already exists
positions = Position.objects.filter(
person=pombola_person,
organisation=organisation,
title=position_titles[person['Position']]
).currently_active()
if not positions:
print 'Creating position (%s) for %s' % (person['Position'], pombola_person)
if commit:
positiontitle, _ = PositionTitle.objects.get_or_create(
name=person['Position'])
position = Position.objects.create(
person=pombola_person,
organisation=organisation,
title=positiontitle,
start_date=start_date,
end_date='future')
people_to_keep.append(position.id)
#information source
for infosource in infosources:
InformationSource.objects.get_or_create(
source = infosource['source_url'],
note = infosource['source_note'],
entered = True,
content_type=position_content_type,
object_id=position.id
)
for position in positions:
people_to_keep.append(position.id)
print 'Retaining %s' % (position)
#check cell number
if 'Cell' in person:
contacts = Contact.objects.filter(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_telephone)
#if only one cell exists replace
if len(contacts)==1:
print 'Updating tel number for', pombola_person, 'from', contacts[0].value, 'to', person['Cell']
if commit:
contacts[0].value = person['Cell']
contacts[0].save()
else:
#otherwise check if the cell
#has already been loaded
add = True
#check if already exists
for contact in contacts:
existing = re.sub('[^0-9]', '', contact.value)
new = re.sub('[^0-9]', '', person['Cell'])
if existing==new:
add = False
if add:
print pombola_person
print person['Cell']
print 'Adding tel number for', pombola_person, '-', person['Cell']
if commit:
Contact.objects.create(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_telephone,
value=person['Cell'],
source=source_url)
print 'Updating contact source to %s' % (source_url)
if commit:
for contact in contacts:
contact.source = source_url
contact.save()
#check email
if 'Email' in person:
contacts = Contact.objects.filter(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_email)
#if only one email exists replace
if len(contacts)==1:
print 'Updating email for %s from %s to %s' % (pombola_person, contacts[0].value, person['Email'])
if commit:
contacts[0].value = person['Email']
contacts[0].save()
else:
#otherwise check if the email has already been
#loaded
add = True
#check if already exists
for contact in contacts:
existing = contact.value
new = person['Email']
if existing==new:
add = False
if add:
print 'Adding email for %s: %s' % (pombola_person, person['Email'])
if commit:
Contact.objects.create(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_email,
value=person['Email'],
source=source_url)
print 'Updating contact source to %s' % (source_url)
if commit:
for contact in contacts:
contact.source = source_url
contact.save()
#check alternative name
if 'Alternative Name' in person:
try:
AlternativePersonName.objects.get(
person=pombola_person,
alternative_name=person['Alternative Name'])
except ObjectDoesNotExist:
print 'Adding alternative name for %s: %s' % (pombola_person, person['Alternative Name'].encode('utf-8'))
if commit:
AlternativePersonName.objects.create(
person=pombola_person,
alternative_name=person['Alternative Name'])
if not pombola_person:
if person['Position'] == 'Constituency Contact':
personnotfound.append([office['Title'], person['Name']])
print 'Failed to match representative', person['Name']
else:
print 'Creating person (%s) with position (%s)' % (person['Name'], person['Position'])
if commit:
create_person = Person.objects.create(
legal_name=person['Name'],
slug=slugify(person['Name']))
Position.objects.create(
person=create_person,
organisation=organisation,
title=position_titles[person['Position']],
start_date=start_date,
end_date='future')
if 'Cell' in person:
print 'Adding cell number %s' % (person['Cell'])
if commit:
Contact.objects.create(
object_id=create_person.id,
content_type=person_content_type,
kind=ck_telephone,
value=person['Cell'],
source=source_url)
if 'Alternative Name' in person:
print 'Adding alternative name %s' % (unicode(person['Alternative Name'], 'utf-8'))
if commit:
AlternativePersonName.objects.create(
person=create_person,
alternative_name=person['Alternative Name'])
#find the positions to end
if organisation:
positions = Position.objects.filter(
organisation=organisation
).exclude(id__in=people_to_keep).currently_active()
for position in positions:
print 'Ending %s' % (position)
if commit:
position.end_date = end_date
position.save()
#FIXME: check summary, kind, started, ended,
#identifiers (not expected at present)
return organisation
0
Example 184
View licensedef process_office(office, commit, start_date, end_date, na_member_lookup, geocode_cache):
global locationsnotfound, personnotfound
# Ensure that all the required kinds and other objects exist:
ok_constituency_office, _ = OrganisationKind.objects.get_or_create(
slug='constituency-office',
name='Constituency Office')
ok_constituency_area, _ = OrganisationKind.objects.get_or_create(
slug='constituency-area',
name='Constituency Area')
pk_constituency_office, _ = PlaceKind.objects.get_or_create(
slug='constituency-office',
name='Constituency Office')
pk_constituency_area, _ = PlaceKind.objects.get_or_create(
slug='constituency-area',
name='Constituency Area')
constituency_kinds = {
'area': ok_constituency_area,
'office': ok_constituency_office
}
constituency_place_kinds = {
'area': pk_constituency_area,
'office': pk_constituency_office
}
ck_address, _ = ContactKind.objects.get_or_create(
slug='address',
name='Address')
ck_postal_address, _ = ContactKind.objects.get_or_create(
slug='postal_address',
name='Postal Address')
ck_email, _ = ContactKind.objects.get_or_create(
slug='email',
name='Email')
ck_fax, _ = ContactKind.objects.get_or_create(
slug='fax',
name='Fax')
ck_telephone, _ = ContactKind.objects.get_or_create(
slug='voice',
name='Voice')
ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
name='has_office'
)
pt_constituency_contact, _ = PositionTitle.objects.get_or_create(
slug='constituency-contact',
name='Constituency Contact')
pt_administrator, _ = PositionTitle.objects.get_or_create(
slug='administrator',
name='Administrator')
pt_administrator_volunteer, _ = PositionTitle.objects.get_or_create(
slug='administrator-volunteer',
name='Administrator (volunteer)')
pt_volunteer, _ = PositionTitle.objects.get_or_create(
slug='volunteer',
name='Volunteer')
pt_coordinator, _ = PositionTitle.objects.get_or_create(
slug='coordinator',
name='Coordinator')
pt_community_development_field_worker, _ = PositionTitle.objects.get_or_create(
slug='community-development-field-worker',
name='Community Development Field Worker')
position_titles = {
'Constituency Contact': pt_constituency_contact,
'Administrator': pt_administrator,
'Administrator (volunteer)': pt_administrator_volunteer,
'Volunteer': pt_volunteer,
'Coordinator': pt_coordinator,
'Community Development Field Worker': pt_community_development_field_worker}
ork_has_office, _ = OrganisationRelationshipKind.objects.get_or_create(
name='has_office')
contact_kinds = {
'E-mail': ck_email,
'Tel': ck_telephone,
'Fax': ck_fax,
'Physical Address': ck_address,
'Postal Address': ck_postal_address
}
print "\n", office['Title']
infosources = []
if 'Sources' in office:
source_url = office['Sources'][0]['Source URL']
first = True
for source in office['Sources']:
print 'Adding InformationSource %s (%s)' % (
source['Source URL'],
source['Source Note']
)
infosources.append({
'source_url': source['Source URL'],
'source_note': source['Source Note']
})
if first:
first = False
continue
source_url += ' and ' + source['Source URL']
else:
source_url = office['Source URL']
source_note = office['Source Note']
infosources.append({
'source_url': source_url,
'source_note': source_note
})
print 'Adding InformationSource %s (%s)' % (source_url, source_note)
if ('Physical Address' in office) and (not 'South Africa' in office['Physical Address']) and (office['Physical Address']!='TBA'):
office['Physical Address'] = office['Physical Address'] + ', South Africa'
if ('Location' in office) and (office['Location']!='TBA') and (not office['Province'].lower() in office['Location'].lower()):
office['Location'] = office['Location'] + ', ' + office['Province'] + ', South Africa'
if ('Location' in office) and (not 'South Africa' in office['Location']) and (office['Location']!='TBA'):
office['Location'] = office['Location'] + ', South Africa'
#first determine whether the office already exists
organisation = None
try:
organisation = Organisation.objects.get(
name=office['Title']
)
except ObjectDoesNotExist:
#check identifiers
try:
if 'identifiers' in office:
for identifier_scheme, party_code in office['identifiers'].items():
identifier = Identifier.objects.get(
identifier=party_code,
scheme=identifier_scheme)
organisation = identifier.content_object
except ObjectDoesNotExist:
pass
if organisation: #existing office
if organisation.name != office['Title']:
print 'Changing name from %s to %s' % (organisation.name, office['Title'])
if commit:
organisation.name = office['Title']
organisation.save()
if organisation.ended != 'future':
print 'Changing ended date from %s to future' % (organisation.ended)
if commit:
organisation.ended = 'future'
organisation.save()
else:
print 'Creating new %s' % (office['Type'])
if commit:
organisation = Organisation.objects.create(
name=office['Title'],
slug=slugify(office['Title']),
kind=constituency_kinds[office['Type']],
started=start_date,
ended='future')
#information source
if commit:
for infosource in infosources:
InformationSource.objects.get_or_create(
source = infosource['source_url'],
note = infosource['source_note'],
entered = True,
content_type=organisation_content_type,
object_id=organisation.id
)
#relationship to party
try:
party = Organisation.objects.get(slug=office['Party'].lower())
OrganisationRelationship.objects.get(
organisation_a=party,
organisation_b=organisation,
kind=ork_has_office
)
#if the relationship exists nothing needs to change
print 'Retaining relationship with %s' % (party)
except (ObjectDoesNotExist, AttributeError):
print 'Adding relationship with %s' % (party)
if commit:
OrganisationRelationship.objects.create(
organisation_a=party,
organisation_b=organisation,
kind=ork_has_office
)
office_fields = [
'E-mail',
'Tel',
'Fax',
'Physical Address',
'Postal Address'
]
for field in office_fields:
if field in office:
try:
if not organisation:
raise ObjectDoesNotExist
contact = Contact.objects.get(
object_id=organisation.id,
content_type=organisation_content_type,
kind=contact_kinds[field])
if office[field] != contact.value:
print 'Changing %s from %s to %s' % (field, contact.value, office[field])
if commit:
contact.value = office[field]
contact.save()
print 'Updating contact source to %s' % (source_url)
if commit:
contact.source = source_url
contact.save()
except ObjectDoesNotExist:
print 'Creating new contact (%s: %s)' % (field, office[field])
if commit:
Contact.objects.create(
object_id=organisation.id,
content_type=organisation_content_type,
kind=contact_kinds[field],
value=office[field],
source=source_url)
else:
try:
if not organisation:
raise ObjectDoesNotExist
contact = Contact.objects.get(
object_id=organisation.id,
content_type=organisation_content_type,
kind=contact_kinds[field])
print 'Deleting', contact
if commit:
contact.delete()
except ObjectDoesNotExist:
pass
if 'Municipality' in office:
mapit_municipality = get_mapit_municipality(office['Municipality'], office.get('Province', ''))
if mapit_municipality:
place_name = u'Municipality associated with ' + office['Title']
try:
if not organisation:
raise ObjectDoesNotExist
place = Place.objects.get(
name__startswith=u'Municipality associated with ',
organisation=organisation)
if place.name != place_name:
'Changing municipality association name from %s to %s' % (place.name, place_name)
if commit:
place.name = place_name
place.save()
if place.mapit_area != mapit_municipality:
print 'Changing municipality mapit association from %s to %s' % (place.mapit_area, mapit_municipality)
if commit:
place.mapit_area = mapit_municipality
place.save()
except ObjectDoesNotExist:
print 'Create municipality association'
to_add = {
'name': place_name,
'slug': slugify(place_name),
'kind': constituency_place_kinds[office['Type']],
'mapit_area': mapit_municipality,}
print to_add
if commit:
Place.objects.create(
name=to_add['name'],
slug=to_add['slug'],
kind=to_add['kind'],
mapit_area=to_add['mapit_area'],
organisation=organisation)
if 'manual_lonlat' in office or 'Physical Address' in office or 'Location' in office:
reference_location = ''
try:
if 'manual_lonlat' in office:
#FIXME implement
print 'manual'
elif 'Location' in office:
reference_location = office['Location']
lon, lat, geocode_cache = geocode(office['Location'], geocode_cache, VERBOSE)
elif 'Physical Address' in office:
reference_location = office['Physical Address']
#geocode physical address
lon, lat, geocode_cache = geocode(office['Physical Address'], geocode_cache, VERBOSE)
location = Point(lon, lat)
if office['Type']=='area':
name = u'Unknown sub-area of %s known as %s' % (office['Province'], office['Title'])
else:
name = u'Approximate position of ' + office['Title']
try:
if not organisation:
raise ObjectDoesNotExist
if office['Type'] == 'area':
place = Place.objects.get(
name__startswith=u'Unknown sub-area of',
organisation=organisation)
else:
place = Place.objects.get(
name__startswith=u'Approximate position of ',
organisation=organisation)
if place.location != location:
print 'Changing location from %s to %s' % (place.location, location)
debug_location_change(place.location, location)
if commit:
place.location = location
place.save()
if place.name != name:
print 'Changing location name from %s to %s' % (place.name, name)
if commit:
place.name = name
place.save()
except ObjectDoesNotExist:
print 'Create constituency location'
if commit:
Place.objects.create(
name=name,
slug=slugify(name),
organisation=organisation,
location=location,
kind=constituency_place_kinds[office['Type']])
except LocationNotFound:
locationsnotfound.append([office['Title'], reference_location])
print "XXX no results found for: " + reference_location
else:
print 'No office/area location specified'
people_to_keep = []
if 'People' in office:
for person in office['People']:
#person matching needs to be improved - for now attempt
#find_pombola_person (from
#south_africa_import_constituency_offices command) otherwise
#direct match.
pombola_person = find_pombola_person(person['Name'], na_member_lookup, VERBOSE)
if not pombola_person:
#use filter().distinct() instead of get due to multiple
#rows being returned
pombola_person = Person.objects.filter(
Q(legal_name=person['Name']) |
Q(alternative_names__alternative_name=person['Name'])).distinct()
if len(pombola_person)==0:
pombola_person = None
else:
pombola_person = pombola_person[0]
#check person currently holds office
accept_person = True
if pombola_person and person['Position']=='Constituency Contact':
position_check = Position.objects.filter(
person=pombola_person,
organisation__kind__slug__in=['parliament', 'provincial-legislature'])
if len(position_check)==0:
accept_person=False
print '%s is not an MP or MPL' % (pombola_person.name)
if pombola_person and accept_person:
#check if the position already exists
positions = Position.objects.filter(
person=pombola_person,
organisation=organisation,
title=position_titles[person['Position']]
).currently_active()
if not positions:
print 'Creating position (%s) for %s' % (person['Position'], pombola_person)
if commit:
positiontitle, _ = PositionTitle.objects.get_or_create(
name=person['Position'])
position = Position.objects.create(
person=pombola_person,
organisation=organisation,
title=positiontitle,
start_date=start_date,
end_date='future')
people_to_keep.append(position.id)
#information source
for infosource in infosources:
InformationSource.objects.get_or_create(
source = infosource['source_url'],
note = infosource['source_note'],
entered = True,
content_type=position_content_type,
object_id=position.id
)
for position in positions:
people_to_keep.append(position.id)
print 'Retaining %s' % (position)
#check cell number
if 'Cell' in person:
contacts = Contact.objects.filter(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_telephone)
#if only one cell exists replace
if len(contacts)==1:
print 'Updating tel number for', pombola_person, 'from', contacts[0].value, 'to', person['Cell']
if commit:
contacts[0].value = person['Cell']
contacts[0].save()
else:
#otherwise check if the cell
#has already been loaded
add = True
#check if already exists
for contact in contacts:
existing = re.sub('[^0-9]', '', contact.value)
new = re.sub('[^0-9]', '', person['Cell'])
if existing==new:
add = False
if add:
print pombola_person
print person['Cell']
print 'Adding tel number for', pombola_person, '-', person['Cell']
if commit:
Contact.objects.create(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_telephone,
value=person['Cell'],
source=source_url)
print 'Updating contact source to %s' % (source_url)
if commit:
for contact in contacts:
contact.source = source_url
contact.save()
#check email
if 'Email' in person:
contacts = Contact.objects.filter(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_email)
#if only one email exists replace
if len(contacts)==1:
print 'Updating email for %s from %s to %s' % (pombola_person, contacts[0].value, person['Email'])
if commit:
contacts[0].value = person['Email']
contacts[0].save()
else:
#otherwise check if the email has already been
#loaded
add = True
#check if already exists
for contact in contacts:
existing = contact.value
new = person['Email']
if existing==new:
add = False
if add:
print 'Adding email for %s: %s' % (pombola_person, person['Email'])
if commit:
Contact.objects.create(
object_id=pombola_person.id,
content_type=person_content_type,
kind=ck_email,
value=person['Email'],
source=source_url)
print 'Updating contact source to %s' % (source_url)
if commit:
for contact in contacts:
contact.source = source_url
contact.save()
#check alternative name
if 'Alternative Name' in person:
try:
AlternativePersonName.objects.get(
person=pombola_person,
alternative_name=person['Alternative Name'])
except ObjectDoesNotExist:
print 'Adding alternative name for %s: %s' % (pombola_person, person['Alternative Name'].encode('utf-8'))
if commit:
AlternativePersonName.objects.create(
person=pombola_person,
alternative_name=person['Alternative Name'])
if not pombola_person:
if person['Position'] == 'Constituency Contact':
personnotfound.append([office['Title'], person['Name']])
print 'Failed to match representative', person['Name']
else:
print 'Creating person (%s) with position (%s)' % (person['Name'], person['Position'])
if commit:
create_person = Person.objects.create(
legal_name=person['Name'],
slug=slugify(person['Name']))
Position.objects.create(
person=create_person,
organisation=organisation,
title=position_titles[person['Position']],
start_date=start_date,
end_date='future')
if 'Cell' in person:
print 'Adding cell number %s' % (person['Cell'])
if commit:
Contact.objects.create(
object_id=create_person.id,
content_type=person_content_type,
kind=ck_telephone,
value=person['Cell'],
source=source_url)
if 'Alternative Name' in person:
print 'Adding alternative name %s' % (unicode(person['Alternative Name'], 'utf-8'))
if commit:
AlternativePersonName.objects.create(
person=create_person,
alternative_name=person['Alternative Name'])
#find the positions to end
if organisation:
positions = Position.objects.filter(
organisation=organisation
).exclude(id__in=people_to_keep).currently_active()
for position in positions:
print 'Ending %s' % (position)
if commit:
position.end_date = end_date
position.save()
#FIXME: check summary, kind, started, ended,
#identifiers (not expected at present)
return organisation
0
Example 185
View licensedef get_settings(conf_file_leafname, election_app=None, tests=False):
conf = get_conf(conf_file_leafname)
debug = bool(int(conf.get('STAGING')))
# Get the requested ELECTION_APP:
if election_app is None:
election_app = conf['ELECTION_APP']
election_app_fully_qualified = 'elections.' + election_app
election_settings_module = election_app_fully_qualified + '.settings'
elections_module = importlib.import_module(election_settings_module)
language_code = conf.get('LANGUAGE_CODE', 'en-gb')
# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/
locale_paths = [
join(BASE_DIR, 'locale')
]
# The code below sets LANGUAGES to only those we have translations
# for, so at the time of writing that will be:
# [('en', 'English'), ('es-ar', 'Argentinian Spanish')]
# whereas the default setting is a long list of languages which
# includes:
# ('es', 'Spanish').
# If someone's browser sends 'Accept-Language: es', that means that it
# will be found in this list, but since there are no translations for 'es'
# it'll fall back to LANGUAGE_CODE. However, if there is no 'es' in
# LANGUAGES, then Django will attempt to do a best match, so if
# Accept-Language is 'es' then it will use the 'es-ar' translation. We think
# this is generally desirable (e.g. so someone can see YourNextMP in Spanish
# if their browser asks for Spanish).
languages = [
l for l in LANGUAGES
if exists(join(locale_paths[0], to_locale(l[0])))
]
languages.append(('cy-gb', 'Welsh'))
languages.append(('es-cr', 'Costa Rican Spanish'))
# we need this to make the language switcher work because Django doesn't
# have any language information for these so if you try to tell it to
# switch to them it cannot and falls back to the existing/default language
EXTRA_LANG_INFO = {
'cy-gb': {
'bidi': False,
'code': 'cy-gb',
'name': 'Welsh',
'name_local': u'Cymraeg',
},
'es-cr': {
'bidi': False,
'code': 'es-cr',
'name': 'Spanish',
'name_local': u'español de Costa Rica',
},
}
locale.LANG_INFO.update(EXTRA_LANG_INFO)
# The language selection has been slightly complicated now that we
# have two es- languages: es-ar and es-cr. Chrome doesn't offer
# Costa Rican Spanish as one of its language choices, so the best
# you can do is choose 'Spanish - español'. (This might well be
# the case in other browsers too.) Since 'es-ar' comes first in
# 'languages' after the preceding code, this means that someone
# viewing the Costa Rica site with Chrome's preferred language set
# to Spanish (i.e. with 'es' first in Accept-Language) will get
# the Argentinian Spanish translations instead of Costa Rican
# Spanish. To get around this, look for the default language code
# for the site, and if that's present, move it to the front of
# 'languages'. This should be generally helpful behaviour: the
# default language code of the site should take precedence over
# another language that happens to match based on the generic part
# of the language code.
language_code_index = next(
(i for i, l in enumerate(languages) if l[0] == language_code),
None
)
if language_code_index is not None:
languages.insert(0, languages.pop(language_code_index))
# Make sure the MEDIA_ROOT directory actually exists:
media_root = conf.get('MEDIA_ROOT') or join(BASE_DIR, 'media')
# Make sure that the MEDIA_ROOT and subdirectory for archived CSV
# files exist:
mkdir_p(join(media_root, 'csv-archives'))
# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases
if conf.get('DATABASE_SYSTEM') == 'postgresql':
databases = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': conf.get('YNMP_DB_NAME'),
'USER': conf.get('YNMP_DB_USER'),
'PASSWORD': conf.get('YNMP_DB_PASS'),
'HOST': conf.get('YNMP_DB_HOST'),
'PORT': conf.get('YNMP_DB_PORT'),
}
}
else:
databases = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': join(BASE_DIR, 'db.sqlite3'),
}
}
# Setup caches depending on DEBUG:
if debug:
cache = {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}
cache_thumbnails = {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}
else:
cache = {
'TIMEOUT': None, # cache keys never expire; we invalidate them
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'KEY_PREFIX': databases['default']['NAME'],
}
cache_thumbnails = {
'TIMEOUT': 60 * 60 * 24 * 2, # expire after two days
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
'KEY_PREFIX': databases['default']['NAME'] + "-thumbnails",
}
# Create a dictionary with these settings and other simpler ones:
result = {
'BASE_DIR': BASE_DIR,
'ALLOWED_HOSTS': conf.get('ALLOWED_HOSTS'),
'DEBUG': debug,
'RUNNING_TESTS': tests,
# Email addresses that error emails are sent to when DEBUG = False
'ADMINS': conf['ADMINS'],
# SECURITY WARNING: keep the secret key used in production secret!
'SECRET_KEY': conf['SECRET_KEY'],
'TEMPLATE_DEBUG': True,
'TEMPLATE_DIRS': (
join(BASE_DIR, 'mysite', 'templates'),
),
'TEMPLATE_CONTEXT_PROCESSORS': TEMPLATE_CONTEXT_PROCESSORS + (
# Required by allauth template tags
"django.core.context_processors.request",
# allauth specific context processors
"allauth.account.context_processors.account",
"allauth.socialaccount.context_processors.socialaccount",
"django.contrib.messages.context_processors.messages",
"mysite.context_processors.add_settings",
"mysite.context_processors.election_date",
"mysite.context_processors.add_group_permissions",
"mysite.context_processors.add_notification_data",
"mysite.context_processors.locale",
"mysite.context_processors.add_site",
),
'ELECTION_APP': election_app,
'ELECTION_APP_FULLY_QUALIFIED': election_app_fully_qualified,
# The Django applications in use:
'INSTALLED_APPS': (
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.humanize',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.sites',
'django_nose',
'django_extensions',
'pipeline',
'statici18n',
'sorl.thumbnail',
'rest_framework',
'rest_framework.authtoken',
'images',
'haystack',
'elections',
'popolo',
election_app_fully_qualified,
'candidates',
'tasks',
'alerts',
'cached_counts',
'moderation_queue',
'auth_helpers',
'debug_toolbar',
'template_timings_panel',
'official_documents',
'results',
'notifications',
'allauth',
'allauth.account',
'allauth.socialaccount',
'allauth.socialaccount.providers.google',
'allauth.socialaccount.providers.facebook',
'allauth.socialaccount.providers.twitter',
'corsheaders',
'crispy_forms',
'usersettings',
),
'SITE_ID': 1,
'USERSETTINGS_MODEL': 'candidates.SiteSettings',
'MIDDLEWARE_CLASSES': (
'debug_toolbar.middleware.DebugToolbarMiddleware',
'corsheaders.middleware.CorsMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.locale.LocaleMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'candidates.middleware.CopyrightAssignmentMiddleware',
'candidates.middleware.DisallowedUpdateMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'usersettings.middleware.CurrentUserSettingsMiddleware',
),
# django-allauth settings:
'AUTHENTICATION_BACKENDS': (
# Needed to login by username in Django admin, regardless of `allauth`
"django.contrib.auth.backends.ModelBackend",
# `allauth` specific authentication methods, such as login by e-mail
"allauth.account.auth_backends.AuthenticationBackend",
),
'SOCIALACCOUNT_PROVIDERS': {
'google': {'SCOPE': ['https://www.googleapis.com/auth/userinfo.profile'],
'AUTH_PARAMS': {'access_type': 'online'}},
'facebook': {'SCOPE': ['email',]},
},
'LOGIN_REDIRECT_URL': '/',
'ACCOUNT_EMAIL_VERIFICATION': 'mandatory',
'ACCOUNT_EMAIL_REQUIRED': True,
'ACCOUNT_USERNAME_REQUIRED': True,
'SOCIALACCOUNT_AUTO_SIGNUP': True,
# use our own adapter that checks if user signup has been disabled
'ACCOUNT_ADAPTER': 'mysite.account_adapter.CheckIfAllowedNewUsersAccountAdapter',
'ROOT_URLCONF': 'mysite.urls',
'WSGI_APPLICATION': 'mysite.wsgi.application',
# Django Debug Toolbar settings:
'DEBUG_TOOLBAR_PATCH_SETTINGS': False,
'DEBUG_TOOLBAR_PANELS': [
'debug_toolbar.panels.versions.VersionsPanel',
'debug_toolbar.panels.timer.TimerPanel',
'debug_toolbar.panels.settings.SettingsPanel',
'debug_toolbar.panels.headers.HeadersPanel',
'debug_toolbar.panels.request.RequestPanel',
'debug_toolbar.panels.sql.SQLPanel',
'debug_toolbar.panels.staticfiles.StaticFilesPanel',
'debug_toolbar.panels.templates.TemplatesPanel',
'debug_toolbar.panels.cache.CachePanel',
'debug_toolbar.panels.signals.SignalsPanel',
'debug_toolbar.panels.logging.LoggingPanel',
'debug_toolbar.panels.redirects.RedirectsPanel',
'template_timings_panel.panels.TemplateTimings.TemplateTimings',
],
'INTERNAL_IPS': ['127.0.0.1'],
# Language settings (calculated above):
'LOCALE_PATHS': locale_paths,
'LANGUAGES': languages,
'LANGUAGE_CODE': language_code,
'TIME_ZONE': conf.get('TIME_ZONE', 'Europe/London'),
'USE_I18N': True,
'USE_L10N': True,
'USE_TZ': True,
# The media and static file settings:
'MEDIA_ROOT': media_root,
'MEDIA_URL': '/media/',
# Settings for staticfiles and Django pipeline:
'STATIC_URL': '/static/',
'STATIC_ROOT': join(BASE_DIR, 'static'),
'STATICI18N_ROOT': join(BASE_DIR, 'mysite', 'static'),
'STATICFILES_DIRS': (
join(BASE_DIR, 'mysite', 'static'),
),
'STATICFILES_FINDERS': (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
'pipeline.finders.PipelineFinder',
),
'PIPELINE': {
'STYLESHEETS': {
'image-review': {
'source_filenames': (
'moderation_queue/css/jquery.Jcrop.css',
'moderation_queue/css/crop.scss',
),
'output_filename': 'css/image-review.css',
},
'official_documents': {
'source_filenames': (
'official_documents/css/official_documents.scss',
),
'output_filename': 'css/official_documents.css',
},
'all': {
'source_filenames': (
'candidates/style.scss',
'cached_counts/style.scss',
'select2/select2.css',
'jquery/jquery-ui.css',
'jquery/jquery-ui.structure.css',
'jquery/jquery-ui.theme.css',
'moderation_queue/css/photo-upload.scss',
),
'output_filename': 'css/all.css',
}
},
'JAVASCRIPT': {
'image-review': {
'source_filenames': (
'moderation_queue/js/jquery.color.js',
'moderation_queue/js/jquery.Jcrop.js',
'moderation_queue/js/crop.js',
),
'output_filename': 'js/image-review.js',
},
'all': {
'source_filenames': (
'js/vendor/custom.modernizr.js',
'jquery/jquery-1.11.1.js',
'jquery/jquery-ui.js',
'foundation/js/foundation/foundation.js',
'foundation/js/foundation/foundation.equalizer.js',
'foundation/js/foundation/foundation.dropdown.js',
'foundation/js/foundation/foundation.tooltip.js',
'foundation/js/foundation/foundation.offcanvas.js',
'foundation/js/foundation/foundation.accordion.js',
'foundation/js/foundation/foundation.joyride.js',
'foundation/js/foundation/foundation.alert.js',
'foundation/js/foundation/foundation.topbar.js',
'foundation/js/foundation/foundation.reveal.js',
'foundation/js/foundation/foundation.slider.js',
'foundation/js/foundation/foundation.magellan.js',
'foundation/js/foundation/foundation.clearing.js',
'foundation/js/foundation/foundation.orbit.js',
'foundation/js/foundation/foundation.interchange.js',
'foundation/js/foundation/foundation.abide.js',
'foundation/js/foundation/foundation.tab.js',
'select2/select2.js',
'js/constituency.js',
'js/person_form.js',
'js/home_geolocation_form.js',
'js/versions.js',
'js/language-switcher.js',
),
'output_filename': 'js/all.js'
}
},
'COMPILERS': (
'pipeline.compilers.sass.SASSCompiler',
),
'SASS_BINARY': 'sassc',
'CSS_COMPRESSOR': 'pipeline.compressors.yui.YUICompressor',
'JS_COMPRESSOR': 'pipeline.compressors.yui.YUICompressor',
# On some platforms this might be called "yuicompressor", so it may be
# necessary to symlink it into your PATH as "yui-compressor".
'YUI_BINARY': '/usr/bin/env yui-compressor',
},
'TEST_RUNNER': 'django_nose.NoseTestSuiteRunner',
'SOURCE_HINTS': _(
u"Please don't quote third-party candidate sites \u2014 "
u"we prefer URLs of news stories or official candidate pages."
),
# By default, cache successful results from MapIt for a day
'MAPIT_CACHE_SECONDS': 86400,
'DATABASES': databases,
'CACHES': {
'default': cache,
'thumbnails': cache_thumbnails,
},
# sorl-thumbnail settings:
'THUMBNAIL_CACHE': 'thumbnails',
'THUMBNAIL_DEBUG': debug,
# Django Rest Framework settings:
'REST_FRAMEWORK': {
'DEFAULT_PERMISSION_CLASSES': ('candidates.api_permissions.ReadOnly',),
'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.URLPathVersioning',
'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
'PAGE_SIZE': 10,
},
# allow attaching extra data to notifications:
'NOTIFICATIONS_USE_JSONFIELD': True,
'HAYSTACK_SIGNAL_PROCESSOR': 'haystack.signals.RealtimeSignalProcessor',
'HAYSTACK_CONNECTIONS': {
'default': {
'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
'URL': 'http://127.0.0.1:9200/',
'INDEX_NAME': '{0}_{1}'.format(conf.get('YNMP_DB_NAME'), conf.get('YNMP_DB_HOST')),
},
},
# CORS config
'CORS_ORIGIN_ALLOW_ALL': True,
'CORS_URLS_REGEX': r'^/api/.*$',
'CORS_ALLOW_METHODS': (
'GET',
'OPTIONS',
),
}
if tests:
result['NOSE_ARGS'] = [
'--nocapture',
'--with-yanc',
# There are problems with OpenCV on Travis, so don't even try to
# import moderation_queue/faces.py
'--ignore-files=faces',
]
if election_app == 'example':
result['NOSE_ARGS'].append('--with-doctest')
else:
# If we're not testing, use PipelineCachedStorage
result['STATICFILES_STORAGE'] = \
'pipeline.storage.PipelineCachedStorage'
if conf.get('NGINX_SSL'):
result['SECURE_PROXY_SSL_HEADER'] = ('HTTP_X_FORWARDED_PROTO', 'https')
result['ACCOUNT_DEFAULT_HTTP_PROTOCOL'] = 'https'
for optional_election_app_setting, default in (
('AREAS_TO_ALWAYS_RETURN', []),
):
try:
result[optional_election_app_setting] = \
getattr(elections_module, optional_election_app_setting)
except AttributeError:
result[optional_election_app_setting] = default
# Make sure there's a trailing slash at the end of base MapIt URL:
result['MAPIT_BASE_URL'] = \
re.sub(r'/*$', '/', elections_module.MAPIT_BASE_URL)
return result
0
Example 186
View license def build_meta(self):
"""
Ensures the guarantees about metadata for documents are valid.
`page.title` - Will be a string.
`page.slug` - Will be a string.
`page.author` - Will have fields `name` and `email`.
`page.authors` - Will be a list of Authors.
`page.category` - Will be a list.
`page.published` - Will exist.
`page.datetime` - Will be a datetime, or None.
`page.date` - Will be a date, or None.
`page.time` - Will be a time, or None.
`page.tags` - Will be a list.
`page.url` - Will be the url of the page, relative to the web root.
`page.subpages` - Will be a list containing every sub page of this page
"""
self.engine.run_hook('page.meta.pre', self)
if not self.meta:
self.meta = {}
# title
if not 'title' in self.meta:
if self.filename:
# Take off the last file extension.
self.meta['title'] = '.'.join(self.filename.split('.')[:-1])
if (self.meta['title'] == ''):
self.meta['title'] = self.filename
logging.warning("You didn't specify a title in {0}. Using the "
"file name as a title.".format(self.path))
elif 'slug' in self.meta:
self.meta['title'] = self.meta['slug']
logging.warning("You didn't specify a title in {0}, which was "
"not generated from a file. Using the slug as a title."
.format(self.meta['slug']))
else:
logging.error("A page was generated that is not from a file, "
"has no title, and no slug. I don't know what to do. "
"Not using this page.")
logging.info("Bad Meta's keys: {0}".format(self.meta.keys()))
logging.debug("Bad Meta: {0}".format(self.meta))
raise BadMetaException()
# slug
if not 'slug' in self.meta:
if self.filename and self.options['slug_from_filename']:
filename_no_ext = '.'.join(self.filename.split('.')[:-1])
if filename_no_ext == '':
filename_no_ext = self.filename
self.meta['slug'] = slugify(filename_no_ext)
logging.info("You didn't specify a slug, generating it from the "
"filename.")
else:
self.meta['slug'] = slugify(unicode(self.meta['title']))
logging.info("You didn't specify a slug, and no filename "
"exists. Generating the slug from the title.")
elif self.meta['slug'] != slugify(self.meta['slug']):
logging.warning('Your slug should probably be all lower case, and '
'match "[a-z0-9-]*"')
# authors and author
authors = self.meta.get('authors', self.meta.get('author', None))
if isinstance(authors, list):
self.meta['authors'] = [Author.parse(a) for a in authors]
elif isinstance(authors, str):
self.meta['authors'] = [Author.parse(a) for a in authors.split(',')]
if len(self.meta['authors']) > 1:
logging.warn('Deprecation Warning: Use YAML lists instead of '
'CSV for multiple authors. i.e. ["John Doe", "Jane '
'Smith"] instead of "John Doe, Jane Smith". In '
'{0}.'.format(self.path))
elif authors is None:
self.meta['authors'] = self.options.get('authors', [])
else:
# wait, what? Authors is of wrong type.
self.meta['authors'] = []
logging.error(('Authors in {0} is an unknown type. Valid types '
'are string or list. Instead it is a {1}')
.format(self.meta['slug']), authors.type)
if self.meta['authors']:
self.meta['author'] = self.meta['authors'][0]
else:
self.meta['author'] = Author()
# category
if 'category' in self.meta:
if isinstance(self.meta['category'], str):
self.meta['category'] = self.meta['category'].split('/')
elif isinstance(self.meta['category'], list):
pass
else:
# category is of wrong type.
logging.error('Category in {0} is an unknown type. Valid '
'types are string or list. Instead it is a {1}'
.format(self.meta['slug'], type(self.meta['category'])))
self.meta['category'] = []
else:
self.meta['category'] = []
if self.meta['category'] == None:
self.meta = []
# published
if not 'published' in self.meta:
self.meta['published'] = True
# make_file
if not 'make_file' in self.meta:
self.meta['make_file'] = True
# datetime, date, time
util.date_and_times(self.meta)
# tags
if 'tags' in self.meta:
if isinstance(self.meta['tags'], list):
# good
pass
elif isinstance(self.meta['tags'], str):
self.meta['tags'] = [t.strip() for t in
self.meta['tags'].split(',')]
if len(self.meta['tags']) > 1:
logging.warn('Deprecation Warning: Use YAML lists instead '
'of CSV for multiple tags. i.e. tags: [guide, '
'howto] instead of tags: guide, howto. In {0}.'
.format(self.path))
else:
self.meta['tags'] = []
logging.debug('Tags for {0}: {1}'.
format(self.meta['slug'], self.meta['tags']))
# pagination
if 'pagination' not in self.meta:
self.meta['pagination'] = {}
if 'cur_page' not in self.meta['pagination']:
self.meta['pagination']['cur_page'] = 1
if 'num_pages' not in self.meta['pagination']:
self.meta['pagination']['num_pages'] = 1
# template
try:
template_type = str(self.meta.get('type', 'default'))
self.template = self.tmpl_env.get_template(template_type + '.*')
except jinja2.loaders.TemplateNotFound:
logging.error('No template "{0}.*" found in template directory. Aborting.'
.format(template_type))
sys.exit()
except AmbiguousTemplate:
logging.error(('Ambiguous template found. There are two files that '
'match "{0}.*". Aborting.').format(template_type))
sys.exit()
# url
parts = {
'slug': self.meta['slug'],
'category': '/'.join(self.meta['category']),
'page': self.meta['pagination']['cur_page'],
'date': self.meta['date'],
'datetime': self.meta['datetime'],
'time': self.meta['time'],
}
logging.debug('current page: ' + repr(parts['page']))
# Pull extensions from the template's real file name.
parts['ext'] = os.path.splitext(self.template.filename)[1]
if parts['ext']:
parts['ext'] = parts['ext'][1:] # remove leading dot
# Deprecated
parts['type'] = parts['ext']
self.meta['ext'] = parts['ext']
if parts['page'] == 1:
parts['page'] = ''
if 'url' in self.meta:
logging.debug('Using page url pattern')
self.url_pattern = self.meta['url']
else:
logging.debug('Using global url pattern')
self.url_pattern = self.options['url_pattern']
self.meta['url'] = self.url_pattern.format(**parts)
logging.info('URL pattern is: {0}'.format(self.url_pattern))
logging.info('URL parts are: {0}'.format(parts))
# Get rid of extra slashes
self.meta['url'] = re.sub(r'//+', '/', self.meta['url'])
# If we have been asked to, rip out any plain "index.html"s
if not self.options['url_include_index']:
self.meta['url'] = re.sub(r'/index\.html$', '/', self.meta['url'])
# To be used for writing page content
self.meta['path'] = self.meta['url']
# If site is going to be in a subdirectory
if self.options.get('url_subdir'):
self.meta['url'] = self.options['url_subdir'] + self.meta['url']
# Some urls should start with /, some should not.
if self.options['relative_urls'] and self.meta['url'][0] == '/':
self.meta['url'] = self.meta['url'][1:]
if not self.options['relative_urls'] and self.meta['url'][0] != '/':
self.meta['url'] = '/' + self.meta['url']
logging.debug('url is: ' + self.meta['url'])
# subpages
self.meta['subpages'] = []
self.engine.run_hook('page.meta.post', self)
0
Example 187
View licensedef _make_jsmin(python_only=False):
"""
Generate JS minifier based on `jsmin.c by Douglas Crockford`_
.. _jsmin.c by Douglas Crockford:
http://www.crockford.com/javascript/jsmin.c
:Parameters:
`python_only` : ``bool``
Use only the python variant. If true, the c extension is not even
tried to be loaded.
:Return: Minifier
:Rtype: ``callable``
"""
# pylint: disable = unused-variable
# pylint: disable = too-many-locals
if not python_only:
try:
import _rjsmin
except ImportError:
pass
else:
return _rjsmin.jsmin
try:
xrange
except NameError:
xrange = range # pylint: disable = redefined-builtin
space_chars = r'[\000-\011\013\014\016-\040]'
line_comment = r'(?://[^\r\n]*)'
space_comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
space_comment_nobang = r'(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*\*+)*/)'
bang_comment = r'(?:/\*![^*]*\*+(?:[^/*][^*]*\*+)*/)'
string1 = \
r'(?:\047[^\047\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)'
string2 = r'(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^"\\\r\n]*)*")'
strings = r'(?:%s|%s)' % (string1, string2)
charclass = r'(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\])'
nospecial = r'[^/\\\[\r\n]'
regex = r'(?:/(?![\r\n/*])%s*(?:(?:\\[^\r\n]|%s)%s*)*/)' % (
nospecial, charclass, nospecial
)
space = r'(?:%s|%s)' % (space_chars, space_comment)
newline = r'(?:%s?[\r\n])' % line_comment
def fix_charclass(result):
""" Fixup string of chars to fit into a regex char class """
pos = result.find('-')
if pos >= 0:
result = r'%s%s-' % (result[:pos], result[pos + 1:])
def sequentize(string):
"""
Notate consecutive characters as sequence
(1-4 instead of 1234)
"""
first, last, result = None, None, []
for char in map(ord, string):
if last is None:
first = last = char
elif last + 1 == char:
last = char
else:
result.append((first, last))
first = last = char
if last is not None:
result.append((first, last))
return ''.join(['%s%s%s' % (
chr(first),
last > first + 1 and '-' or '',
last != first and chr(last) or ''
) for first, last in result]) # noqa
return _re.sub(
r'([\000-\040\047])', # \047 for better portability
lambda m: '\\%03o' % ord(m.group(1)), (
sequentize(result)
.replace('\\', '\\\\')
.replace('[', '\\[')
.replace(']', '\\]')
)
)
def id_literal_(what):
""" Make id_literal like char class """
match = _re.compile(what).match
result = ''.join([
chr(c) for c in xrange(127) if not match(chr(c))
])
return '[^%s]' % fix_charclass(result)
def not_id_literal_(keep):
""" Make negated id_literal like char class """
match = _re.compile(id_literal_(keep)).match
result = ''.join([
chr(c) for c in xrange(127) if not match(chr(c))
])
return r'[%s]' % fix_charclass(result)
not_id_literal = not_id_literal_(r'[a-zA-Z0-9_$]')
preregex1 = r'[(,=:\[!&|?{};\r\n]'
preregex2 = r'%(not_id_literal)sreturn' % locals()
id_literal = id_literal_(r'[a-zA-Z0-9_$]')
id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]')
id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047+-]')
post_regex_off = id_literal_(r'[^\000-\040}\])?:|,;.&=+-]')
dull = r'[^\047"/\000-\040]'
space_sub_simple = _re.compile((
# noqa pylint: disable = bad-continuation
r'(%(dull)s+)' # 0
r'|(%(strings)s%(dull)s*)' # 1
r'|(?<=%(preregex1)s)'
r'%(space)s*(?:%(newline)s%(space)s*)*'
r'(%(regex)s)' # 2
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 3
r'(?=%(post_regex_off)s))?'
r'|(?<=%(preregex2)s)'
r'%(space)s*(?:(%(newline)s)%(space)s*)*' # 4
r'(%(regex)s)' # 5
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 6
r'(?=%(post_regex_off)s))?'
r'|(?<=%(id_literal_close)s)'
r'%(space)s*(?:(%(newline)s)%(space)s*)+' # 7
r'(?=%(id_literal_open)s)'
r'|(?<=%(id_literal)s)(%(space)s)+(?=%(id_literal)s)' # 8
r'|(?<=\+)(%(space)s)+(?=\+)' # 9
r'|(?<=-)(%(space)s)+(?=-)' # 10
r'|%(space)s+'
r'|(?:%(newline)s%(space)s*)+'
) % locals()).sub
# print space_sub_simple.__self__.pattern
def space_subber_simple(match):
""" Substitution callback """
# pylint: disable = too-many-return-statements
groups = match.groups()
if groups[0]:
return groups[0]
elif groups[1]:
return groups[1]
elif groups[2]:
if groups[3]:
return groups[2] + '\n'
return groups[2]
elif groups[5]:
return "%s%s%s" % (
groups[4] and '\n' or '',
groups[5],
groups[6] and '\n' or '',
)
elif groups[7]:
return '\n'
elif groups[8] or groups[9] or groups[10]:
return ' '
else:
return ''
space_sub_banged = _re.compile((
# noqa pylint: disable = bad-continuation
r'(%(dull)s+)' # 0
r'|(%(strings)s%(dull)s*)' # 1
r'|(?<=%(preregex1)s)'
r'(%(space)s*(?:%(newline)s%(space)s*)*)' # 2
r'(%(regex)s)' # 3
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 4
r'(?=%(post_regex_off)s))?'
r'|(?<=%(preregex2)s)'
r'(%(space)s*(?:(%(newline)s)%(space)s*)*)' # 5, 6
r'(%(regex)s)' # 7
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 8
r'(?=%(post_regex_off)s))?'
r'|(?<=%(id_literal_close)s)'
r'(%(space)s*(?:%(newline)s%(space)s*)+)' # 9
r'(?=%(id_literal_open)s)'
r'|(?<=%(id_literal)s)(%(space)s+)(?=%(id_literal)s)' # 10
r'|(?<=\+)(%(space)s+)(?=\+)' # 11
r'|(?<=-)(%(space)s+)(?=-)' # 12
r'|(%(space)s+)' # 13
r'|((?:%(newline)s%(space)s*)+)' # 14
) % locals()).sub
# print space_sub_banged.__self__.pattern
keep = _re.compile((
r'%(space_chars)s+|%(space_comment_nobang)s+|%(newline)s+'
r'|(%(bang_comment)s+)'
) % locals()).sub
keeper = lambda m: m.groups()[0] or ''
# print keep.__self__.pattern
def space_subber_banged(match):
""" Substitution callback """
# pylint: disable = too-many-return-statements
groups = match.groups()
if groups[0]:
return groups[0]
elif groups[1]:
return groups[1]
elif groups[3]:
return "%s%s%s%s" % (
keep(keeper, groups[2]),
groups[3],
keep(keeper, groups[4] or ''),
groups[4] and '\n' or '',
)
elif groups[7]:
return "%s%s%s%s%s" % (
keep(keeper, groups[5]),
groups[6] and '\n' or '',
groups[7],
keep(keeper, groups[8] or ''),
groups[8] and '\n' or '',
)
elif groups[9]:
return keep(keeper, groups[9]) + '\n'
elif groups[10] or groups[11] or groups[12]:
return keep(keeper, groups[10] or groups[11] or groups[12]) or ' '
else:
return keep(keeper, groups[13] or groups[14])
def jsmin(script, keep_bang_comments=False):
r"""
Minify javascript based on `jsmin.c by Douglas Crockford`_\.
Instead of parsing the stream char by char, it uses a regular
expression approach which minifies the whole script with one big
substitution regex.
.. _jsmin.c by Douglas Crockford:
http://www.crockford.com/javascript/jsmin.c
:Parameters:
`script` : ``str``
Script to minify
`keep_bang_comments` : ``bool``
Keep comments starting with an exclamation mark? (``/*!...*/``)
:Return: Minified script
:Rtype: ``str``
"""
# pylint: disable = redefined-outer-name
if keep_bang_comments:
return space_sub_banged(
space_subber_banged, '\n%s\n' % script
).strip()
else:
return space_sub_simple(
space_subber_simple, '\n%s\n' % script
).strip()
return jsmin
0
Example 188
View licensedef _make_jsmin(python_only=False):
"""
Generate JS minifier based on `jsmin.c by Douglas Crockford`_
.. _jsmin.c by Douglas Crockford:
http://www.crockford.com/javascript/jsmin.c
:Parameters:
`python_only` : ``bool``
Use only the python variant. If true, the c extension is not even
tried to be loaded.
:Return: Minifier
:Rtype: ``callable``
"""
# pylint: disable = unused-variable
# pylint: disable = too-many-locals
if not python_only:
try:
import _rjsmin
except ImportError:
pass
else:
return _rjsmin.jsmin
try:
xrange
except NameError:
xrange = range # pylint: disable = redefined-builtin
space_chars = r'[\000-\011\013\014\016-\040]'
line_comment = r'(?://[^\r\n]*)'
space_comment = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
space_comment_nobang = r'(?:/\*(?!!)[^*]*\*+(?:[^/*][^*]*\*+)*/)'
bang_comment = r'(?:/\*![^*]*\*+(?:[^/*][^*]*\*+)*/)'
string1 = \
r'(?:\047[^\047\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^\047\\\r\n]*)*\047)'
string2 = r'(?:"[^"\\\r\n]*(?:\\(?:[^\r\n]|\r?\n|\r)[^"\\\r\n]*)*")'
strings = r'(?:%s|%s)' % (string1, string2)
charclass = r'(?:\[[^\\\]\r\n]*(?:\\[^\r\n][^\\\]\r\n]*)*\])'
nospecial = r'[^/\\\[\r\n]'
regex = r'(?:/(?![\r\n/*])%s*(?:(?:\\[^\r\n]|%s)%s*)*/)' % (
nospecial, charclass, nospecial
)
space = r'(?:%s|%s)' % (space_chars, space_comment)
newline = r'(?:%s?[\r\n])' % line_comment
def fix_charclass(result):
""" Fixup string of chars to fit into a regex char class """
pos = result.find('-')
if pos >= 0:
result = r'%s%s-' % (result[:pos], result[pos + 1:])
def sequentize(string):
"""
Notate consecutive characters as sequence
(1-4 instead of 1234)
"""
first, last, result = None, None, []
for char in map(ord, string):
if last is None:
first = last = char
elif last + 1 == char:
last = char
else:
result.append((first, last))
first = last = char
if last is not None:
result.append((first, last))
return ''.join(['%s%s%s' % (
chr(first),
last > first + 1 and '-' or '',
last != first and chr(last) or ''
) for first, last in result]) # noqa
return _re.sub(
r'([\000-\040\047])', # \047 for better portability
lambda m: '\\%03o' % ord(m.group(1)), (
sequentize(result)
.replace('\\', '\\\\')
.replace('[', '\\[')
.replace(']', '\\]')
)
)
def id_literal_(what):
""" Make id_literal like char class """
match = _re.compile(what).match
result = ''.join([
chr(c) for c in xrange(127) if not match(chr(c))
])
return '[^%s]' % fix_charclass(result)
def not_id_literal_(keep):
""" Make negated id_literal like char class """
match = _re.compile(id_literal_(keep)).match
result = ''.join([
chr(c) for c in xrange(127) if not match(chr(c))
])
return r'[%s]' % fix_charclass(result)
not_id_literal = not_id_literal_(r'[a-zA-Z0-9_$]')
preregex1 = r'[(,=:\[!&|?{};\r\n]'
preregex2 = r'%(not_id_literal)sreturn' % locals()
id_literal = id_literal_(r'[a-zA-Z0-9_$]')
id_literal_open = id_literal_(r'[a-zA-Z0-9_${\[(!+-]')
id_literal_close = id_literal_(r'[a-zA-Z0-9_$}\])"\047+-]')
post_regex_off = id_literal_(r'[^\000-\040}\])?:|,;.&=+-]')
dull = r'[^\047"/\000-\040]'
space_sub_simple = _re.compile((
# noqa pylint: disable = bad-continuation
r'(%(dull)s+)' # 0
r'|(%(strings)s%(dull)s*)' # 1
r'|(?<=%(preregex1)s)'
r'%(space)s*(?:%(newline)s%(space)s*)*'
r'(%(regex)s)' # 2
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 3
r'(?=%(post_regex_off)s))?'
r'|(?<=%(preregex2)s)'
r'%(space)s*(?:(%(newline)s)%(space)s*)*' # 4
r'(%(regex)s)' # 5
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 6
r'(?=%(post_regex_off)s))?'
r'|(?<=%(id_literal_close)s)'
r'%(space)s*(?:(%(newline)s)%(space)s*)+' # 7
r'(?=%(id_literal_open)s)'
r'|(?<=%(id_literal)s)(%(space)s)+(?=%(id_literal)s)' # 8
r'|(?<=\+)(%(space)s)+(?=\+)' # 9
r'|(?<=-)(%(space)s)+(?=-)' # 10
r'|%(space)s+'
r'|(?:%(newline)s%(space)s*)+'
) % locals()).sub
# print space_sub_simple.__self__.pattern
def space_subber_simple(match):
""" Substitution callback """
# pylint: disable = too-many-return-statements
groups = match.groups()
if groups[0]:
return groups[0]
elif groups[1]:
return groups[1]
elif groups[2]:
if groups[3]:
return groups[2] + '\n'
return groups[2]
elif groups[5]:
return "%s%s%s" % (
groups[4] and '\n' or '',
groups[5],
groups[6] and '\n' or '',
)
elif groups[7]:
return '\n'
elif groups[8] or groups[9] or groups[10]:
return ' '
else:
return ''
space_sub_banged = _re.compile((
# noqa pylint: disable = bad-continuation
r'(%(dull)s+)' # 0
r'|(%(strings)s%(dull)s*)' # 1
r'|(?<=%(preregex1)s)'
r'(%(space)s*(?:%(newline)s%(space)s*)*)' # 2
r'(%(regex)s)' # 3
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 4
r'(?=%(post_regex_off)s))?'
r'|(?<=%(preregex2)s)'
r'(%(space)s*(?:(%(newline)s)%(space)s*)*)' # 5, 6
r'(%(regex)s)' # 7
r'(%(space)s*(?:%(newline)s%(space)s*)+' # 8
r'(?=%(post_regex_off)s))?'
r'|(?<=%(id_literal_close)s)'
r'(%(space)s*(?:%(newline)s%(space)s*)+)' # 9
r'(?=%(id_literal_open)s)'
r'|(?<=%(id_literal)s)(%(space)s+)(?=%(id_literal)s)' # 10
r'|(?<=\+)(%(space)s+)(?=\+)' # 11
r'|(?<=-)(%(space)s+)(?=-)' # 12
r'|(%(space)s+)' # 13
r'|((?:%(newline)s%(space)s*)+)' # 14
) % locals()).sub
# print space_sub_banged.__self__.pattern
keep = _re.compile((
r'%(space_chars)s+|%(space_comment_nobang)s+|%(newline)s+'
r'|(%(bang_comment)s+)'
) % locals()).sub
keeper = lambda m: m.groups()[0] or ''
# print keep.__self__.pattern
def space_subber_banged(match):
""" Substitution callback """
# pylint: disable = too-many-return-statements
groups = match.groups()
if groups[0]:
return groups[0]
elif groups[1]:
return groups[1]
elif groups[3]:
return "%s%s%s%s" % (
keep(keeper, groups[2]),
groups[3],
keep(keeper, groups[4] or ''),
groups[4] and '\n' or '',
)
elif groups[7]:
return "%s%s%s%s%s" % (
keep(keeper, groups[5]),
groups[6] and '\n' or '',
groups[7],
keep(keeper, groups[8] or ''),
groups[8] and '\n' or '',
)
elif groups[9]:
return keep(keeper, groups[9]) + '\n'
elif groups[10] or groups[11] or groups[12]:
return keep(keeper, groups[10] or groups[11] or groups[12]) or ' '
else:
return keep(keeper, groups[13] or groups[14])
def jsmin(script, keep_bang_comments=False):
r"""
Minify javascript based on `jsmin.c by Douglas Crockford`_\.
Instead of parsing the stream char by char, it uses a regular
expression approach which minifies the whole script with one big
substitution regex.
.. _jsmin.c by Douglas Crockford:
http://www.crockford.com/javascript/jsmin.c
:Parameters:
`script` : ``str``
Script to minify
`keep_bang_comments` : ``bool``
Keep comments starting with an exclamation mark? (``/*!...*/``)
:Return: Minified script
:Rtype: ``str``
"""
# pylint: disable = redefined-outer-name
if keep_bang_comments:
return space_sub_banged(
space_subber_banged, '\n%s\n' % script
).strip()
else:
return space_sub_simple(
space_subber_simple, '\n%s\n' % script
).strip()
return jsmin
0
Example 189
View licensedef process_comment(comment, is_edit=False):
#Anime/Manga requests that are found go into separate arrays
animeArray = []
mangaArray = []
lnArray = []
#ignores all "code" markup (i.e. anything between backticks)
comment.body = re.sub(r"\`(?s)(.*?)\`", "", comment.body)
#This checks for requests. First up we check all known tags for the !stats request
if re.search('({!stats.*?}|{{!stats.*?}}|<!stats.*?>|<<!stats.*?>>)', comment.body, re.S) is not None:
username = re.search('[uU]\/([A-Za-z0-9_-]+?)(>|}|$)', comment.body, re.S)
subreddit = re.search('[rR]\/([A-Za-z0-9_]+?)(>|}|$)', comment.body, re.S)
if username:
commentReply = CommentBuilder.buildStatsComment(username=username.group(1))
elif subreddit:
commentReply = CommentBuilder.buildStatsComment(subreddit=subreddit.group(1))
else:
commentReply = CommentBuilder.buildStatsComment()
else:
#The basic algorithm here is:
#If it's an expanded request, build a reply using the data in the braces, clear the arrays, add the reply to the relevant array and ignore everything else.
#If it's a normal request, build a reply using the data in the braces, add the reply to the relevant array.
#Counts the number of expanded results vs total results. If it's not just a single expanded result, they all get turned into normal requests.
numOfRequest = 0
numOfExpandedRequest = 0
forceNormal = False
for match in re.finditer("\{{2}([^}]*)\}{2}|\<{2}([^>]*)\>{2}", comment.body, re.S):
numOfRequest += 1
numOfExpandedRequest += 1
for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))|(?<=(?<!\<)\<)([^\<\>]*)(?=\>(?!\>))", comment.body, re.S):
numOfRequest += 1
if (numOfExpandedRequest >= 1) and (numOfRequest > 1):
forceNormal = True
#Expanded Anime
for match in re.finditer("\{{2}([^}]*)\}{2}", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildAnimeReply(match.group(1), False, comment)
else:
reply = Search.buildAnimeReply(match.group(1), True, comment)
if (reply is not None):
animeArray.append(reply)
#Normal Anime
for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))", comment.body, re.S):
reply = Search.buildAnimeReply(match.group(1), False, comment)
if (reply is not None):
animeArray.append(reply)
#Expanded Manga
#NORMAL EXPANDED
for match in re.finditer("\<{2}([^>]*)\>{2}(?!(:|\>))", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildMangaReply(match.group(1), False, comment)
else:
reply = Search.buildMangaReply(match.group(1), True, comment)
if (reply is not None):
mangaArray.append(reply)
#AUTHOR SEARCH EXPANDED
for match in re.finditer("\<{2}([^>]*)\>{2}:\(([^)]+)\)", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)
else:
reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), True, comment)
if (reply is not None):
mangaArray.append(reply)
#Normal Manga
#NORMAL
for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]+)\>(?!(:|\>))", comment.body, re.S):
reply = Search.buildMangaReply(match.group(1), False, comment)
if (reply is not None):
mangaArray.append(reply)
#AUTHOR SEARCH
for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]*)\>:\(([^)]+)\)", comment.body, re.S):
reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)
if (reply is not None):
mangaArray.append(reply)
#Expanded LN
for match in re.finditer("\]{2}([^]]*)\[{2}", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildLightNovelReply(match.group(1), False, comment)
else:
reply = Search.buildLightNovelReply(match.group(1), True, comment)
if (reply is not None):
lnArray.append(reply)
#Normal LN
for match in re.finditer("(?<=(?<!\])\])([^\]\[]*)(?=\[(?!\[))", comment.body, re.S):
reply = Search.buildLightNovelReply(match.group(1), False, comment)
if (reply is not None):
lnArray.append(reply)
#Here is where we create the final reply to be posted
#The final comment reply. We add stuff to this progressively.
commentReply = ''
#Basically just to keep track of people posting the same title multiple times (e.g. {Nisekoi}{Nisekoi}{Nisekoi})
postedAnimeTitles = []
postedMangaTitles = []
postedLNTitles = []
#Adding all the anime to the final comment. If there's manga too we split up all the paragraphs and indent them in Reddit markup by adding a '>', then recombine them
for i, animeReply in enumerate(animeArray):
if not (i is 0):
commentReply += '\n\n'
if not (animeReply['title'] in postedAnimeTitles):
postedAnimeTitles.append(animeReply['title'])
commentReply += animeReply['comment']
if mangaArray:
commentReply += '\n\n'
#Adding all the manga to the final comment
for i, mangaReply in enumerate(mangaArray):
if not (i is 0):
commentReply += '\n\n'
if not (mangaReply['title'] in postedMangaTitles):
postedMangaTitles.append(mangaReply['title'])
commentReply += mangaReply['comment']
if lnArray:
commentReply += '\n\n'
#Adding all the manga to the final comment
for i, lnReply in enumerate(lnArray):
if not (i is 0):
commentReply += '\n\n'
if not (lnReply['title'] in postedLNTitles):
postedLNTitles.append(lnReply['title'])
commentReply += lnReply['comment']
#If there are more than 10 requests, shorten them all
if not (commentReply is '') and (len(animeArray) + len(mangaArray)+ len(lnArray) >= 10):
commentReply = re.sub(r"\^\((.*?)\)", "", commentReply, flags=re.M)
#If there was actually something found, add the signature and post the comment to Reddit. Then, add the comment to the "already seen" database.
if commentReply is not '':
'''if (comment.author.name == 'treborabc'):
commentReply = '[No.](https://www.reddit.com/r/anime_irl/comments/4sba1n/anime_irl/d58xkha)'''
commentReply += Config.getSignature(comment.permalink)
commentReply += Reference.get_bling(comment.author.name)
if is_edit:
return commentReply
else:
try:
comment.reply(commentReply)
print("Comment made.\n")
except praw.errors.Forbidden:
print('Request from banned subreddit: ' + str(comment.subreddit) + '\n')
except Exception:
traceback.print_exc()
try:
DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, True)
except:
traceback.print_exc()
else:
try:
if is_edit:
return None
else:
DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, False)
except:
traceback.print_exc()
0
Example 190
View licensedef process_comment(comment, is_edit=False):
#Anime/Manga requests that are found go into separate arrays
animeArray = []
mangaArray = []
lnArray = []
#ignores all "code" markup (i.e. anything between backticks)
comment.body = re.sub(r"\`(?s)(.*?)\`", "", comment.body)
#This checks for requests. First up we check all known tags for the !stats request
if re.search('({!stats.*?}|{{!stats.*?}}|<!stats.*?>|<<!stats.*?>>)', comment.body, re.S) is not None:
username = re.search('[uU]\/([A-Za-z0-9_-]+?)(>|}|$)', comment.body, re.S)
subreddit = re.search('[rR]\/([A-Za-z0-9_]+?)(>|}|$)', comment.body, re.S)
if username:
commentReply = CommentBuilder.buildStatsComment(username=username.group(1))
elif subreddit:
commentReply = CommentBuilder.buildStatsComment(subreddit=subreddit.group(1))
else:
commentReply = CommentBuilder.buildStatsComment()
else:
#The basic algorithm here is:
#If it's an expanded request, build a reply using the data in the braces, clear the arrays, add the reply to the relevant array and ignore everything else.
#If it's a normal request, build a reply using the data in the braces, add the reply to the relevant array.
#Counts the number of expanded results vs total results. If it's not just a single expanded result, they all get turned into normal requests.
numOfRequest = 0
numOfExpandedRequest = 0
forceNormal = False
for match in re.finditer("\{{2}([^}]*)\}{2}|\<{2}([^>]*)\>{2}", comment.body, re.S):
numOfRequest += 1
numOfExpandedRequest += 1
for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))|(?<=(?<!\<)\<)([^\<\>]*)(?=\>(?!\>))", comment.body, re.S):
numOfRequest += 1
if (numOfExpandedRequest >= 1) and (numOfRequest > 1):
forceNormal = True
#Expanded Anime
for match in re.finditer("\{{2}([^}]*)\}{2}", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildAnimeReply(match.group(1), False, comment)
else:
reply = Search.buildAnimeReply(match.group(1), True, comment)
if (reply is not None):
animeArray.append(reply)
#Normal Anime
for match in re.finditer("(?<=(?<!\{)\{)([^\{\}]*)(?=\}(?!\}))", comment.body, re.S):
reply = Search.buildAnimeReply(match.group(1), False, comment)
if (reply is not None):
animeArray.append(reply)
#Expanded Manga
#NORMAL EXPANDED
for match in re.finditer("\<{2}([^>]*)\>{2}(?!(:|\>))", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildMangaReply(match.group(1), False, comment)
else:
reply = Search.buildMangaReply(match.group(1), True, comment)
if (reply is not None):
mangaArray.append(reply)
#AUTHOR SEARCH EXPANDED
for match in re.finditer("\<{2}([^>]*)\>{2}:\(([^)]+)\)", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)
else:
reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), True, comment)
if (reply is not None):
mangaArray.append(reply)
#Normal Manga
#NORMAL
for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]+)\>(?!(:|\>))", comment.body, re.S):
reply = Search.buildMangaReply(match.group(1), False, comment)
if (reply is not None):
mangaArray.append(reply)
#AUTHOR SEARCH
for match in re.finditer("(?<=(?<!\<)\<)([^\<\>]*)\>:\(([^)]+)\)", comment.body, re.S):
reply = Search.buildMangaReplyWithAuthor(match.group(1), match.group(2), False, comment)
if (reply is not None):
mangaArray.append(reply)
#Expanded LN
for match in re.finditer("\]{2}([^]]*)\[{2}", comment.body, re.S):
reply = ''
if (forceNormal) or (str(comment.subreddit).lower() in disableexpanded):
reply = Search.buildLightNovelReply(match.group(1), False, comment)
else:
reply = Search.buildLightNovelReply(match.group(1), True, comment)
if (reply is not None):
lnArray.append(reply)
#Normal LN
for match in re.finditer("(?<=(?<!\])\])([^\]\[]*)(?=\[(?!\[))", comment.body, re.S):
reply = Search.buildLightNovelReply(match.group(1), False, comment)
if (reply is not None):
lnArray.append(reply)
#Here is where we create the final reply to be posted
#The final comment reply. We add stuff to this progressively.
commentReply = ''
#Basically just to keep track of people posting the same title multiple times (e.g. {Nisekoi}{Nisekoi}{Nisekoi})
postedAnimeTitles = []
postedMangaTitles = []
postedLNTitles = []
#Adding all the anime to the final comment. If there's manga too we split up all the paragraphs and indent them in Reddit markup by adding a '>', then recombine them
for i, animeReply in enumerate(animeArray):
if not (i is 0):
commentReply += '\n\n'
if not (animeReply['title'] in postedAnimeTitles):
postedAnimeTitles.append(animeReply['title'])
commentReply += animeReply['comment']
if mangaArray:
commentReply += '\n\n'
#Adding all the manga to the final comment
for i, mangaReply in enumerate(mangaArray):
if not (i is 0):
commentReply += '\n\n'
if not (mangaReply['title'] in postedMangaTitles):
postedMangaTitles.append(mangaReply['title'])
commentReply += mangaReply['comment']
if lnArray:
commentReply += '\n\n'
#Adding all the manga to the final comment
for i, lnReply in enumerate(lnArray):
if not (i is 0):
commentReply += '\n\n'
if not (lnReply['title'] in postedLNTitles):
postedLNTitles.append(lnReply['title'])
commentReply += lnReply['comment']
#If there are more than 10 requests, shorten them all
if not (commentReply is '') and (len(animeArray) + len(mangaArray)+ len(lnArray) >= 10):
commentReply = re.sub(r"\^\((.*?)\)", "", commentReply, flags=re.M)
#If there was actually something found, add the signature and post the comment to Reddit. Then, add the comment to the "already seen" database.
if commentReply is not '':
'''if (comment.author.name == 'treborabc'):
commentReply = '[No.](https://www.reddit.com/r/anime_irl/comments/4sba1n/anime_irl/d58xkha)'''
commentReply += Config.getSignature(comment.permalink)
commentReply += Reference.get_bling(comment.author.name)
if is_edit:
return commentReply
else:
try:
comment.reply(commentReply)
print("Comment made.\n")
except praw.errors.Forbidden:
print('Request from banned subreddit: ' + str(comment.subreddit) + '\n')
except Exception:
traceback.print_exc()
try:
DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, True)
except:
traceback.print_exc()
else:
try:
if is_edit:
return None
else:
DatabaseHandler.addComment(comment.id, comment.author.name, comment.subreddit, False)
except:
traceback.print_exc()
0
Example 191
View licensedef ipa(text): # Generates narrow transcription of Tamil texts
text=" " + text + " "
# text = """ %s """ % text
# Move Punctuations
repl = lambda m: " " + m.group(1) + " "
text = re.sub("([\\,\\\,\.\!\?\"\'\"\(\)])", repl, text)
# text = re.sub("/(?<=[\w])([\\,\.\!\?\"\'\"\(\)])/",repl,text)
# text = re.sub("/([\\,\.\!\?\"\'\"\(\)])(?=[\w])/",repl,text)
# The comments below refer to the implementation of transcription rules as described in the
# book - Panniru Thirumurai Olipeyarppu by Punal K Murugaiyan
# Dipthongs
text = text.replace("ai","ay") # 5.3.3.1 ii Palatal Approximant ... # ai
text = text.replace("au","av") # au - dipthong replacement
# Grantha
text = text.replace("j","ʤ")
text = text.replace("h","ɦ")
text = text.replace("S","ʂ")
text = text.replace("srI","ʂrI")
# Mey
# Vallinam
# pa
text = re.sub(r"(?<=[aAiIuUeEoO])p(?=[aAiIuUeEoO])","β",text) # 5.3.1.6 iii Voiced Bilabial Fricative
text = re.sub(r"(?<=[yrlvZL])p","β",text) # 5.3.1.6 iii
text = re.sub(r"(?<=[GJnVmN])p","b",text) #5.3.1.6 ii voiced bilabial plosive
# 5.3.1.6 i no replacement
# ta
text = re.sub(r"(?<=[aAiIuUeEoOyrlvZL])t(?=[aAiIuUeEoO])","ð",text) # 5.3.1.5 iii Voiced dental Fricative
text = re.sub(r"(?<=[nV])t","d̪",text) # 5.3.1.5 ii Voiced dental plosive
text = re.sub(r"t","t̪",text) # 5.3.1.5 i Voiceless dental plosive
# Ra
text = text.replace("XX","t̺t̺ʳ") # 5.3.1.4 ii & ii find correct name
text = re.sub(r"(?<=V)X","d̺ʳ",text) # 5.3.1.4 iii
# 5.3.1.4 iv & v implemented in idaiyinam section
# Ta
text = re.sub(r"(?<=[aAiIuUeEoO])T(?=[aAiIuUeEoO])","ɽ",text) # 5.3.1.3 iii Retroflex Flap
text = re.sub(r"(?<=[N])T","ɖ",text) # 5.3.1.3 ii Voiced Retroflex Plosive | VT ?
text = text.replace("T","ʈ") # 5.3.1.3 i Voiceless Retroflex Plosive
# ca
text = re.sub(r"(?<=[aAiIuUeEoOl])c(?=[aAiIuUeEoO])","s",text) # 5.3.1.2 iii voiceless alveolar fricatives
repl = lambda m: m.group(1) + "s"
text = re.sub(r"(\s)c",repl,text) # 5.3.1.2 iii
text = re.sub(r"(V)c",repl,text)
text = re.sub(r"(?<=[J])c","ʤ",text) # 5.3.1.2 ii Voiced Post Alveolar affricate - Symbol Changed : d͡ʒ
text = re.sub(r"c","ʧ",text) # 5.3.1.2 i Voicless Post Alveolar Affrivate - Symbol Changed : t͡ʃ
# ka
text = re.sub(r"Gk(?=[iI])","ŋʲgʲ",text) # 5.3.2.1 ii Palatized Velar Nasal
text = text.replace("Gk","ŋg") # 5.3.2.1 Velar Nasal
text = re.sub(r"(?<=[aAiIuUeEoO])k(?=[iI])","ç",text) # 5.3.1.1 viii voiceless palatal fricative
#yrZlLv assumed above. Missing in definiation : eykiya -> eyçiya aarkiya -> aarçiya....
text = re.sub(r"(?<=r)k(?=[aAuUeEoO])","ɣ",text) # 5.3.1.1 Vii Voiced Velar Fricative
text = re.sub(r"(?<=[aAiIuUeEoO])k(?=[aAuUeEoO])","x",text) # 5.3.1.1 vi Voicless Velar Fricative
text = re.sub(r"(?<=[ylvZL])k(?=[aAuUeEoO])","x",text) # above
text = re.sub(r"ykk","jcc",text) # 5.3.1.1 v voiceless palatal plosive
text = re.sub(r"jkk","jcc",text) # above
text = re.sub(r"(?<=[rylvZLGVNaAiIuUeEoO])k(?=[iI])","gʲ",text) # 5.3.1.1 iv Voiced Palatized Velar Plosive
text = re.sub(r"(?<=[NVmn])k(?=[aAuUeEoO])","g",text) # 5.3.1.1 iii voiced velar plosive
text = re.sub(r"(?<=k)k(?=[iI])","kʲ",text) # 5.3.1.1 ii Voiceless velar plosive
# 5.3.1.1 i no relacement #
# Idaiyinam
text = text.replace("Z","ɻ") #5.3.3.6 Retroflex Approximant
text = re.sub(r"(?<=[aAiIuUeEoO])L(?=[aAiIuUeEoO])","ɭʼ",text) #5.3.3.5 i Lateral Approximant - Ejective
text = text.replace("L","ɭ") # 5.3.3.5 ii Lateral Approximant
# 5.3.3.4 no change
text = re.sub(r"(?<=[aAiIuUeEoO])[rX](?=[aAiIuUeEoO])","ɾ",text) # 5.3.3.3 i Alveolar Tap
# 5.3.3.3 ii - pure consonant r - no replacement
text = re.sub(r"X(?!=[aAiIuUeEoO])","r",text) # 5.3.3.3 ii Trill
text = re.sub(r"(?<=[aAiIuUeEoO])v(?=[aAiIuUeEoO])","ʋ",text) # 5.3.3.2 ii labio-dental approximant
text = re.sub(r"(\s)v(?=[aAiIuUeEoO])",lambda m: m.group(1)+"ʋ",text) # 5.3.3.2 ii
text = text.replace("vv","ʊ̯ʋ") # 5.3.3.2 i near-close near-back rounded vowel - part of a dipthong
text = text.replace("v","ʋ")
text = re.sub(r"yy","jɪ̯",text) # 5.3.3.1 i near-close near-front unrounded vowel - part of a dipthong
text = re.sub(r"y","ɪ̯",text) # 5.3.3.1 i near-close near-front unrounded vowel - part of a dipthong
# Mellinam
# 5.3.2.6 no replacement
text = re.sub(r"[Vn]","n̺",text) # 5.3.2.4 Alveolar Nasal (Check Actual name in Wikipedia)
text = text.replace("n̺d̪","n̪d̪") # 5.3.2.5 Dental Nasal
text = re.sub(r"(?<=[aAiIuUeEoO])N(?=[aAiIuUeEoO])","ɳʼ",text) # 5.3.2.3 ii Retroflex Nasal Ejective
text = text.replace("N","ɳ") # 5.3.2.3 Retroflex Nasal
text = text.replace("J","ɲ") # 5.3.2.3 Palatal Nasal
text = re.sub(r"GG(?=[iI])","ŋʲŋʲ",text) # Assumed based on above
text = text.replace("GG","ŋŋ") # Assumed based on above
text = text.replace("G","ŋ") # Assumed based on above
# Uyir
# Seperate Pure Vowel Combinations
text = re.sub(r"([aAiIuUeEoO])([aAiIuUeEoO])",
lambda m: m.group(1)+"_"+m.group(2),text)
#return text
# Long O
text=re.sub(r"o(\s)",lambda m: "o·"+m.group(1),text) # 5.2.5.2 v
text=re.sub(r"(\s)o(?!·)",lambda m: m.group(1)+"ʷoː",text) # 5.2.5.2 iii
text=re.sub(r"_o(?!·)","ʷoː",text) # 5.2.5.2 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·])o(?![ː·])","oː",text) # 5.2.5.2 i
# Short o
text=re.sub(r"(\s)O(?!·)",lambda m: m.group(1)+"ʷo̞",text) # 5.2.5.1 iii
text=re.sub(r"_O(?!·)","ʷo̞",text) # 5.2.5.1 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞])O(?![ː·])","o̞",text) # 5.2.5.1 i
# Adding extra symbol for Retroflex Consonants
retroflex = ["ɽ","ɖ","ʈ","ɳ","ɭ","ɻ"]
for rf in retroflex:
text = re.sub("/̞(?=" + rf + ")","̞˞",text)
# Long e
text=re.sub(r"e(\s)",lambda m: "e·"+m.group(1),text) # 5.2.4.2 v
text=re.sub(r"(\s)e(?!·)",lambda m: m.group(1)+"ʲeː",text) # 5.2.4.2 iii
text=re.sub(r"_e(?!·)","ʲeː",text) # 5.2.4.2 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞])e(?![ː·])","eː",text) # 5.2.5.2 i
# short e
text=re.sub(r"(\s)E(?!·)",lambda m: m.group(1)+"ʲɛ̝",text) # 5.2.4.1 iii
text=re.sub(r"_E(?!·)","ʲɛ̝",text) # 5.2.4.1 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])E(?![ː·])","ɛ̝",text) # 5.2.5.4 i
# Adding extra symbol for Retroflex Consonants
for rf in retroflex:
text = re.sub("/̝(?=" + rf + ")","̝˞",text)
# short u
text = re.sub(r"(?<!u)(\S)(?<![bʋpmβ])u",lambda m: m.group(1)+"ɨ",text) # 5.2.3.1 v
text=re.sub(r"(\s)u(?!·)",lambda m: m.group(1)+"ʷʊ",text) # 5.2.3.1 iii
text=re.sub(r"_u(?!·)","ʷʊ",text) # 5.2.3.1 iii - puththi_Ottum
text = re.sub(r"(?<!u\S)([bʋpmβ])u",lambda m: m.group(1)+"ʉ̩",text) # 5.2.3.1 Vii
text = re.sub(r"([bʋpβm])u",lambda m: m.group(1)+"ʉ̩",text) # 5.2.3.1 Vii
text = re.sub(r"(?<![bʋpβm])u(\s)",lambda m: "ɨ"+m.group(1),text) # 5.2.3.1 v
repl = lambda m: m.group(1) + m.group(2) + "ʊ"
text= re.sub(r"(\s)(\S)(ɨ|ʉ̩)",repl,text) # 5.2.5.2 i
text= re.sub(r"(U)(\S{1,2})ɨ",repl,text) # 5.2.5.2 i
text= re.sub(r"(ʊ)(\S{1,2})ɨ",repl,text)
text= re.sub(r"(ʊ)(\S{1,2})ʉ̩",repl,text)
text = re.sub(r"(?<![bʋβpm])ʊ(\s)",lambda m: "ɨ"+m.group(1),text) # 5.2.3.1 v
text = re.sub(r"(?<=[bʋβpm])ʊ(\s)",lambda m: "ʉ̩"+m.group(1),text)
for rf in retroflex:
text = re.sub(r"ʊ(?=" + rf + ")","ʊ˞",text)
for rf in retroflex:
text = re.sub(r"ʉ̩(?=" + rf + ")","ʉ̩˞",text)
for rf in retroflex:
text = re.sub(r"ɨ(?=" + rf + ")","ɨ˞",text)
#
text = re.sub(r"\S(<=!u)\Su","ɨ",text) # 5.2.3.1 v
# Long u
text=re.sub(r"U(\s)",lambda m: "u·"+m.group(1),text) # 5.2.3.2 v
text=re.sub(r"(\s)U(?!·)",lambda m: m.group(1)+"ʷuː",text) # 5.2.3.2 iii
text=re.sub(r"_U(?!·)","ʷuː",text) # 5.2.3.2 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])U(?![ː·])","uː",text) # 5.2.3.2 i
# short i
text=re.sub(r"i(\s)",lambda m: "ɪ·"+m.group(1),text) # 5.2.2.1 iii
text=re.sub(r"(\s)i(?!·)",lambda m: m.group(1)+"ʲɪ",text) # 5.2.4.2 iii
text=re.sub(r"_i(?!·)","ʲɪ",text) # 5.2.4.2 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])i(?![ː·])","ɪ",text) # 5.2.5.2 i
for rf in retroflex:
text = re.sub(r"ɪ(?=" + rf + ")","ɪ˞",text)
# Long i
text=re.sub(r"I(\s)",lambda m: "i·"+m.group(1),text) # 5.2.2.2 v
text=re.sub(r"(\s)I(?!·)",lambda m: m.group(1)+"ʲiː",text) # 5.2.2.2 iii
text=re.sub(r"_I(?!·)","ʲiː",text) # 5.2.2.2 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])I(?![ː·])","iː",text) # 5.2.2.2 i
# Long A
text=re.sub(r"(\s)A(?!·)",lambda m: m.group(1)+"ˀɑː",text) # 5.2.1.2 iii
text=re.sub(r"_A(?!·)","ˀɑː",text) # 5.2.1.2 iii - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])A(?![ː·])","ɑː",text) # 5.2.1.2 i
# short a
# Transcription of Abbreviation Ignored
text=re.sub(r"(\s)a(?!·)",lambda m: m.group(1)+"ˀʌ",text) # 5.2.1.1 vi
text=re.sub(r"_a(?!·)","ˀʌ",text) # 5.2.1.1 vi - puththi_Ottum
text=re.sub(r"(?<![aAiIuUeEoOː·̞̝])a(?![ː·])","ʌ",text) # 5.2.1.1 i
text=re.sub(r"ʌ(\s)",lambda m: "ə"+m.group(1),text) # 5.2.1.1 iii
for rf in retroflex:
text = re.sub(r"ʌ(?=" + rf + ")","ʌ˞",text)
# Aytham
text = text.replace("K","x")
text = text.replace("xt̪","xð")
text = text.replace("xk","xx")
# Adding extra symbol for Retroflex Consonants Common of U & O
# Regex won't accept (?=[Multiple Unicode Chars]) so separating each charcter
for rf in retroflex:
text = re.sub(r"ː(?=" + rf + ")","˞ː",text)
# Fixing kaaZBbu kaLaip kaLaippu bugs
text = text.replace("βp","pp")
text = text.replace("β ","p ")
text = re.sub(r"ʧ([ \n])s",lambda m: "ʧ"+m.group(1)+"ʧ",text) # ac samayam -> ac camayam \\ Check with Newlines ac \n samayam
# New IPA Convensions
text = text.replace("ː","ː")
text = text.replace("·","ˑ") #
text = text.replace("ʧ","tʃ")
text = text.replace("ʤ","dʒ")
#ɨɭ (i- + Refelex -> <u> <Retroflex>
text = text.lstrip()
return text
0
Example 192
View licensedef twistranet_project():
"""
Copies the contents of the project_template directory to a new directory
specified as an argument to the command line.
"""
# That is just to check that we can import the TN package.
# If something goes wrong here, just add a --pythonpath option.
parser = OptionParser(
usage = "usage: %prog [options] [<template>] <project_path>\n"
" where project_name is the name of the directory that will be created for your site,\n"
" <path> is the (optional) path where you want to install it.\n"
)
parser.add_option("-n", "--no-bootstrap",
action="store_false", dest="bootstrap", default=True,
help="Don't bootstrap project immediately. Use this if you want to review your settings before bootstraping.",
)
parser.add_option("-d", "--develmode",
action="store_true", dest="develmode", default=False,
help="Use this option if you want to work on sources in your project. eg: static files",
)
(options, args) = parser.parse_args()
# Check template and path args
if len(args) < 1:
parser.error("project_path must be specified.")
elif len(args) == 1:
project_path = args[0]
project_template = 'default'
elif len(args) == 2:
project_path = args[1]
project_template = args[0]
# Check if we can import TN.
import twistranet
# We decompose the given path to split installdir and project_name
project_path = os.path.abspath(project_path)
if os.path.lexists(project_path):
parser.error("Directory '%s' already exists. "
"Please specify a non-existing directory." % (project_path, ))
project_name = os.path.split(project_path)[1]
# Check if project_name is correct
try:
eval("%s is True" % project_name)
except SyntaxError:
parser.error("Directory '%s' must be a valid python identifier." % project_name)
except:
pass
# Ensure the given directory name doesn't clash with an existing Python
# package/module.
try:
__import__(project_name)
except ImportError:
pass
except ValueError:
pass # It's ok to install in the same directory ;)
else:
parser.error("'%s' conflicts with the name of an existing "
"Python module and cannot be used as a project name. "
"Please try another name." % project_name)
# Build the project up copying over the twistranet project_template
twist_package = __import__('twistranet')
twist_package_path = os.path.dirname(os.path.abspath(twist_package.__file__))
template_dir = os.path.join(twist_package_path, "project_templates", "default")
if not os.path.isdir(template_dir):
parser.error("Template '%s' is invalid." % project_template)
try:
shutil.copytree(template_dir, project_path, ignore=shutil.ignore_patterns(*IGNORE_PATTERNS))
except AttributeError:
print "shutil.copytree is likely not to have the 'ignore' attribute available.\n"
shutil.copytree(template_dir, project_path)
replaceable_files = [(os.path.join(project_path, "local_settings.py")),]
# project variables replaced in the project files
replacement = {
"SECRET_KEY.*$": "SECRET_KEY = '%s'" % (uuid1(), ),
"__INSTANCE_NAME__": '%s' % project_name,
}
# If project_template <> default, we copy the project_template-specific files as well
if project_template != "default":
source_root = os.path.join(twist_package_path, "project_templates", project_template)
if not os.path.isdir(source_root):
source_root = os.path.abspath(os.path.join(os.path.curdir, project_template))
if not os.path.isdir(source_root):
raise ValueError("Invalid template directory: '%s'" % source_root)
dest_root = project_path
for root, dirs, files in os.walk(source_root):
# Ugly wat to deal with dotted dirs
if '/.' in root:
continue
relative_root = root[len(source_root) + 1:]
for d in dirs:
if d.startswith('.'):
continue
dest_dir = os.path.join(dest_root, relative_root, d)
if not os.path.isdir(dest_dir):
os.mkdir(dest_dir)
for fname in files:
# Ignore doted files, and rename if it contains any replacement string
if fname.startswith('.'):
continue
# if a file is named with __INSTANCE_NAME__.ext rename it with project_name
dname = fname
for regex, repl in replacement.items():
dname = re.sub(regex, repl, dname)
# Actually copy
dest_file = os.path.join(dest_root, relative_root, dname)
shutil.copy(
os.path.join(source_root, root, fname),
dest_file,
)
# in project files some files must be changed (as done for local_settings, see bellow)
replaceable_files.append(dest_file)
# replace some settings.
for replaceable_path in replaceable_files:
with open(replaceable_path, "r") as f:
data = f.read()
with open(replaceable_path, "w") as f:
for regex, repl in replacement.items():
data = re.sub(regex, repl, data)
f.write(data)
f.close()
# we append project_path to sys.path, used for bootstrap and for devel mode configuration (-d).
sys.path.insert(0, project_path) # Here is how we're gonna find the 'settings' module from here.
# XXX NOT VERY DJANGOISH TO USE JUST 'settings' HERE !
os.environ["DJANGO_SETTINGS_MODULE"] = 'settings'
os.environ["TWISTRANET_NOMAIL"] = "1" # Disable emails
import settings
# update settings.TWISTRANET_STATIC_PATH in devel mode
# to use theme from product itself
if options.develmode:
from django.utils.importlib import import_module
theme_app = import_module(settings.TWISTRANET_THEME_APP)
theme_app_dir = os.path.split(theme_app.__file__)[0]
DEVEL_TWISTRANET_STATIC_PATH = os.path.abspath(os.path.join(theme_app_dir, 'static'))
DEVEL_TWISTRANET_LOCALE_PATHS = "("
for twapp in ('core', 'notifier', 'search', 'sharing', 'tagging', 'twistapp', ):
DEVEL_TWISTRANET_LOCALE_PATHS += "r'%s', " %os.path.abspath(os.path.join(twist_package_path, twapp, 'locale'))
DEVEL_TWISTRANET_LOCALE_PATHS += ")"
settings_path = os.path.join(project_path, "settings.py")
f = open(settings_path, "r")
data = f.read()
f.close()
f = open(settings_path, "w")
# XXX TODO : change with a replace
data += '''
# ADDED FOR DEVEL MODE ONLY
TWISTRANET_STATIC_PATH = r"%(static_path)s"
STATICFILES_DIRS = ( r"%(static_path)s",)
COMPRESS_ROOT = r"%(static_path)s"
LOCALE_PATHS = %(locale_paths)s
''' % { 'static_path': DEVEL_TWISTRANET_STATIC_PATH ,
'locale_paths': DEVEL_TWISTRANET_LOCALE_PATHS,
}
f.write(data)
f.close()
# fix settings for the first server start
settings.TWISTRANET_STATIC_PATH = DEVEL_TWISTRANET_STATIC_PATH
settings.STATICFILES_DIRS = (DEVEL_TWISTRANET_STATIC_PATH ,)
settings.COMPRESS_ROOT = DEVEL_TWISTRANET_STATIC_PATH
# As we use a standard sqlite configuration, we can boostrap quite safely just now.
# then we start the server
if options.bootstrap:
from django.core.management import call_command
from django import conf
# update static files,
# excepted in devel mode
if not options.develmode:
call_command('twistranet_update')
call_command('twistranet_bootstrap')
# Now we can start the server!
os.environ["TWISTRANET_NOMAIL"] = "" # Re-enable emails
call_command("runserver", "0.0.0.0:8000", use_reloader = False, )
0
Example 193
View license def receipt(self,xml):
"""
Prints an xml based receipt definition
"""
def strclean(string):
if not string:
string = ''
string = string.strip()
string = re.sub('\s+',' ',string)
return string
def format_value(value, decimals=3, width=0, decimals_separator='.', thousands_separator=',', autoint=False, symbol='', position='after'):
decimals = max(0,int(decimals))
width = max(0,int(width))
value = float(value)
if autoint and math.floor(value) == value:
decimals = 0
if width == 0:
width = ''
if thousands_separator:
formatstr = "{:"+str(width)+",."+str(decimals)+"f}"
else:
formatstr = "{:"+str(width)+"."+str(decimals)+"f}"
ret = formatstr.format(value)
ret = ret.replace(',','COMMA')
ret = ret.replace('.','DOT')
ret = ret.replace('COMMA',thousands_separator)
ret = ret.replace('DOT',decimals_separator)
if symbol:
if position == 'after':
ret = ret + symbol
else:
ret = symbol + ret
return ret
def print_elem(stylestack, serializer, elem, indent=0):
elem_styles = {
'h1': {'bold': 'on', 'size':'double'},
'h2': {'size':'double'},
'h3': {'bold': 'on', 'size':'double-height'},
'h4': {'size': 'double-height'},
'h5': {'bold': 'on'},
'em': {'font': 'b'},
'b': {'bold': 'on'},
}
stylestack.push()
if elem.tag in elem_styles:
stylestack.set(elem_styles[elem.tag])
stylestack.set(elem.attrib)
if elem.tag in ('p','div','section','article','receipt','header','footer','li','h1','h2','h3','h4','h5'):
serializer.start_block(stylestack)
serializer.text(elem.text)
for child in elem:
print_elem(stylestack,serializer,child)
serializer.start_inline(stylestack)
serializer.text(child.tail)
serializer.end_entity()
serializer.end_entity()
elif elem.tag in ('span','em','b','left','right'):
serializer.start_inline(stylestack)
serializer.text(elem.text)
for child in elem:
print_elem(stylestack,serializer,child)
serializer.start_inline(stylestack)
serializer.text(child.tail)
serializer.end_entity()
serializer.end_entity()
elif elem.tag == 'value':
serializer.start_inline(stylestack)
serializer.pre(format_value(
elem.text,
decimals=stylestack.get('value-decimals'),
width=stylestack.get('value-width'),
decimals_separator=stylestack.get('value-decimals-separator'),
thousands_separator=stylestack.get('value-thousands-separator'),
autoint=(stylestack.get('value-autoint') == 'on'),
symbol=stylestack.get('value-symbol'),
position=stylestack.get('value-symbol-position')
))
serializer.end_entity()
elif elem.tag == 'line':
width = stylestack.get('width')
if stylestack.get('size') in ('double', 'double-width'):
width = width / 2
lineserializer = XmlLineSerializer(stylestack.get('indent')+indent,stylestack.get('tabwidth'),width,stylestack.get('line-ratio'))
serializer.start_block(stylestack)
for child in elem:
if child.tag == 'left':
print_elem(stylestack,lineserializer,child,indent=indent)
elif child.tag == 'right':
lineserializer.start_right()
print_elem(stylestack,lineserializer,child,indent=indent)
serializer.pre(lineserializer.get_line())
serializer.end_entity()
elif elem.tag == 'ul':
serializer.start_block(stylestack)
bullet = stylestack.get('bullet')
for child in elem:
if child.tag == 'li':
serializer.style(stylestack)
serializer.raw(' ' * indent * stylestack.get('tabwidth') + bullet)
print_elem(stylestack,serializer,child,indent=indent+1)
serializer.end_entity()
elif elem.tag == 'ol':
cwidth = len(str(len(elem))) + 2
i = 1
serializer.start_block(stylestack)
for child in elem:
if child.tag == 'li':
serializer.style(stylestack)
serializer.raw(' ' * indent * stylestack.get('tabwidth') + ' ' + (str(i)+')').ljust(cwidth))
i = i + 1
print_elem(stylestack,serializer,child,indent=indent+1)
serializer.end_entity()
elif elem.tag == 'pre':
serializer.start_block(stylestack)
serializer.pre(elem.text)
serializer.end_entity()
elif elem.tag == 'hr':
width = stylestack.get('width')
if stylestack.get('size') in ('double', 'double-width'):
width = width / 2
serializer.start_block(stylestack)
serializer.text('-'*width)
serializer.end_entity()
elif elem.tag == 'br':
serializer.linebreak()
elif elem.tag == 'img':
if 'src' in elem.attrib and 'data:' in elem.attrib['src']:
self.print_base64_image(elem.attrib['src'])
elif elem.tag == 'barcode' and 'encoding' in elem.attrib:
serializer.start_block(stylestack)
self.barcode(strclean(elem.text),elem.attrib['encoding'])
serializer.end_entity()
elif elem.tag == 'cut':
self.cut()
elif elem.tag == 'partialcut':
self.cut(mode='part')
elif elem.tag == 'cashdraw':
self.cashdraw(2)
self.cashdraw(5)
stylestack.pop()
try:
stylestack = StyleStack()
serializer = XmlSerializer(self)
root = ET.fromstring(xml.encode('utf-8'))
self._raw(stylestack.to_escpos())
print_elem(stylestack,serializer,root)
if 'open-cashdrawer' in root.attrib and root.attrib['open-cashdrawer'] == 'true':
self.cashdraw(2)
self.cashdraw(5)
if not 'cut' in root.attrib or root.attrib['cut'] == 'true' :
self.cut()
except Exception as e:
errmsg = str(e)+'\n'+'-'*48+'\n'+traceback.format_exc() + '-'*48+'\n'
self.text(errmsg)
self.cut()
raise e
0
Example 194
View licensedef trans_generate(lang, modules, cr):
dbname = cr.dbname
registry = openerp.registry(dbname)
trans_obj = registry['ir.translation']
model_data_obj = registry['ir.model.data']
uid = 1
query = 'SELECT name, model, res_id, module' \
' FROM ir_model_data'
query_models = """SELECT m.id, m.model, imd.module
FROM ir_model AS m, ir_model_data AS imd
WHERE m.id = imd.res_id AND imd.model = 'ir.model' """
if 'all_installed' in modules:
query += ' WHERE module IN ( SELECT name FROM ir_module_module WHERE state = \'installed\') '
query_models += " AND imd.module in ( SELECT name FROM ir_module_module WHERE state = 'installed') "
query_param = None
if 'all' not in modules:
query += ' WHERE module IN %s'
query_models += ' AND imd.module in %s'
query_param = (tuple(modules),)
else:
query += ' WHERE module != %s'
query_models += ' AND imd.module != %s'
query_param = ('__export__',)
query += ' ORDER BY module, model, name'
query_models += ' ORDER BY module, model'
cr.execute(query, query_param)
_to_translate = set()
def push_translation(module, type, name, id, source, comments=None):
# empty and one-letter terms are ignored, they probably are not meant to be
# translated, and would be very hard to translate anyway.
sanitized_term = (source or '').strip()
try:
# verify the minimal size without eventual xml tags
# wrap to make sure html content like '<a>b</a><c>d</c>' is accepted by lxml
wrapped = "<div>%s</div>" % sanitized_term
node = etree.fromstring(wrapped)
sanitized_term = etree.tostring(node, encoding='UTF-8', method='text')
except etree.ParseError:
pass
# remove non-alphanumeric chars
sanitized_term = re.sub(r'\W+', '', sanitized_term)
if not sanitized_term or len(sanitized_term) <= 1:
return
tnx = (module, source, name, id, type, tuple(comments or ()))
_to_translate.add(tnx)
def push(mod, type, name, res_id, term):
term = (term or '').strip()
if len(term) > 2 or term in ENGLISH_SMALL_WORDS:
push_translation(mod, type, name, res_id, term)
def get_root_view(xml_id):
view = model_data_obj.xmlid_to_object(cr, uid, xml_id)
if view:
while view.mode != 'primary':
view = view.inherit_id
xml_id = view.get_external_id(cr, uid).get(view.id, xml_id)
return xml_id
for (xml_name,model,res_id,module) in cr.fetchall():
module = encode(module)
model = encode(model)
xml_name = "%s.%s" % (module, encode(xml_name))
if model not in registry:
_logger.error("Unable to find object %r", model)
continue
Model = registry[model]
if not Model._translate:
# explicitly disabled
continue
obj = Model.browse(cr, uid, res_id)
if not obj.exists():
_logger.warning("Unable to find object %r with id %d", model, res_id)
continue
if model=='ir.model.fields':
try:
field_name = encode(obj.name)
except AttributeError, exc:
_logger.error("name error in %s: %s", xml_name, str(exc))
continue
field_model = registry.get(obj.model)
if (field_model is None or not field_model._translate or
field_name not in field_model._fields):
continue
field_def = field_model._fields[field_name]
if hasattr(field_def, 'selection') and isinstance(field_def.selection, (list, tuple)):
name = "%s,%s" % (encode(obj.model), field_name)
for dummy, val in field_def.selection:
push_translation(module, 'selection', name, 0, encode(val))
elif model=='ir.actions.report.xml':
name = encode(obj.report_name)
fname = ""
if obj.report_rml:
fname = obj.report_rml
parse_func = trans_parse_rml
report_type = "report"
elif obj.report_xsl:
continue
if fname and obj.report_type in ('pdf', 'xsl'):
try:
report_file = misc.file_open(fname)
try:
d = etree.parse(report_file)
for t in parse_func(d.iter()):
push_translation(module, report_type, name, 0, t)
finally:
report_file.close()
except (IOError, etree.XMLSyntaxError):
_logger.exception("couldn't export translation for report %s %s %s", name, report_type, fname)
for field_name, field_def in obj._fields.iteritems():
if getattr(field_def, 'translate', None):
name = model + "," + field_name
try:
value = obj[field_name] or ''
except Exception:
continue
for term in set(field_def.get_trans_terms(value)):
push_translation(module, 'model', name, xml_name, encode(term))
# End of data for ir.model.data query results
cr.execute(query_models, query_param)
def push_constraint_msg(module, term_type, model, msg):
if not hasattr(msg, '__call__'):
push_translation(encode(module), term_type, encode(model), 0, encode(msg))
def push_local_constraints(module, model, cons_type='sql_constraints'):
"""Climb up the class hierarchy and ignore inherited constraints
from other modules"""
term_type = 'sql_constraint' if cons_type == 'sql_constraints' else 'constraint'
msg_pos = 2 if cons_type == 'sql_constraints' else 1
for cls in model.__class__.__mro__:
if getattr(cls, '_module', None) != module:
continue
constraints = getattr(cls, '_local_' + cons_type, [])
for constraint in constraints:
push_constraint_msg(module, term_type, model._name, constraint[msg_pos])
for (_, model, module) in cr.fetchall():
if model not in registry:
_logger.error("Unable to find object %r", model)
continue
model_obj = registry[model]
if model_obj._constraints:
push_local_constraints(module, model_obj, 'constraints')
if model_obj._sql_constraints:
push_local_constraints(module, model_obj, 'sql_constraints')
installed_modules = map(
lambda m: m['name'],
registry['ir.module.module'].search_read(cr, uid, [('state', '=', 'installed')], fields=['name']))
path_list = [(path, True) for path in openerp.modules.module.ad_paths]
# Also scan these non-addon paths
for bin_path in ['osv', 'report', 'modules', 'service', 'tools']:
path_list.append((os.path.join(config.config['root_path'], bin_path), True))
# non-recursive scan for individual files in root directory but without
# scanning subdirectories that may contain addons
path_list.append((config.config['root_path'], False))
_logger.debug("Scanning modules at paths: %s", path_list)
def get_module_from_path(path):
for (mp, rec) in path_list:
mp = os.path.join(mp, '')
if rec and path.startswith(mp) and os.path.dirname(path) != mp:
path = path[len(mp):]
return path.split(os.path.sep)[0]
return 'base' # files that are not in a module are considered as being in 'base' module
def verified_module_filepaths(fname, path, root):
fabsolutepath = join(root, fname)
frelativepath = fabsolutepath[len(path):]
display_path = "addons%s" % frelativepath
module = get_module_from_path(fabsolutepath)
if ('all' in modules or module in modules) and module in installed_modules:
if os.path.sep != '/':
display_path = display_path.replace(os.path.sep, '/')
return module, fabsolutepath, frelativepath, display_path
return None, None, None, None
def babel_extract_terms(fname, path, root, extract_method="python", trans_type='code',
extra_comments=None, extract_keywords={'_': None}):
module, fabsolutepath, _, display_path = verified_module_filepaths(fname, path, root)
extra_comments = extra_comments or []
if not module: return
src_file = open(fabsolutepath, 'r')
try:
for extracted in extract.extract(extract_method, src_file,
keywords=extract_keywords):
# Babel 0.9.6 yields lineno, message, comments
# Babel 1.3 yields lineno, message, comments, context
lineno, message, comments = extracted[:3]
push_translation(module, trans_type, display_path, lineno,
encode(message), comments + extra_comments)
except Exception:
_logger.exception("Failed to extract terms from %s", fabsolutepath)
finally:
src_file.close()
for (path, recursive) in path_list:
_logger.debug("Scanning files of modules at %s", path)
for root, dummy, files in osutil.walksymlinks(path):
for fname in fnmatch.filter(files, '*.py'):
babel_extract_terms(fname, path, root)
# mako provides a babel extractor: http://docs.makotemplates.org/en/latest/usage.html#babel
for fname in fnmatch.filter(files, '*.mako'):
babel_extract_terms(fname, path, root, 'mako', trans_type='report')
# Javascript source files in the static/src/js directory, rest is ignored (libs)
if fnmatch.fnmatch(root, '*/static/src/js*'):
for fname in fnmatch.filter(files, '*.js'):
babel_extract_terms(fname, path, root, 'javascript',
extra_comments=[WEB_TRANSLATION_COMMENT],
extract_keywords={'_t': None, '_lt': None})
# QWeb template files
if fnmatch.fnmatch(root, '*/static/src/xml*'):
for fname in fnmatch.filter(files, '*.xml'):
babel_extract_terms(fname, path, root, 'openerp.tools.translate:babel_extract_qweb',
extra_comments=[WEB_TRANSLATION_COMMENT])
if not recursive:
# due to topdown, first iteration is in first level
break
out = []
# translate strings marked as to be translated
for module, source, name, id, type, comments in sorted(_to_translate):
trans = '' if not lang else trans_obj._get_source(cr, uid, name, type, lang, source)
out.append((module, type, name, id, source, encode(trans) or '', comments))
return out
0
Example 195
View license def scrape_bill_type(self, chamber, session, bill_type, type_abbr,
committee_abbr_regex=get_committee_name_regex()):
if chamber == 'upper':
chamber_name = 'SENATE'
else:
chamber_name = 'ASSEMBLY'
bills = self.session.query(CABill).filter_by(
session_year=session).filter_by(
measure_type=type_abbr)
for bill in bills:
bill_session = session
if bill.session_num != '0':
bill_session += ' Special Session %s' % bill.session_num
bill_id = bill.short_bill_id
fsbill = Bill(bill_session, chamber, bill_id, '')
# # Construct session for web query, going from '20092010' to '0910'
# source_session = session[2:4] + session[6:8]
# # Turn 'AB 10' into 'ab_10'
# source_num = "%s_%s" % (bill.measure_type.lower(),
# bill.measure_num)
# Construct a fake source url
source_url = ('http://leginfo.legislature.ca.gov/faces/'
'billNavClient.xhtml?bill_id=%s') % bill.bill_id
fsbill.add_source(source_url)
fsbill.add_version(bill_id, source_url, 'text/html')
title = ''
type_ = ['bill']
subject = ''
all_titles = set()
# Get digest test (aka "summary") from latest version.
if bill.versions:
version = bill.versions[-1]
nsmap = version.xml.nsmap
xpath = '//caml:DigestText/xhtml:p'
els = version.xml.xpath(xpath, namespaces=nsmap)
chunks = []
for el in els:
t = etree_text_content(el)
t = re.sub(r'\s+', ' ', t)
t = re.sub(r'\)(\S)', lambda m: ') %s' % m.group(1), t)
chunks.append(t)
summary = '\n\n'.join(chunks)
for version in bill.versions:
if not version.bill_xml:
continue
# CA is inconsistent in that some bills have a short title
# that is longer, more descriptive than title.
if bill.measure_type in ('AB', 'SB'):
impact_clause = clean_title(version.title)
title = clean_title(version.short_title)
else:
impact_clause = None
if len(version.title) < len(version.short_title) and \
not version.title.lower().startswith('an act'):
title = clean_title(version.short_title)
else:
title = clean_title(version.title)
if title:
all_titles.add(title)
type_ = [bill_type]
if version.appropriation == 'Yes':
type_.append('appropriation')
if version.fiscal_committee == 'Yes':
type_.append('fiscal committee')
if version.local_program == 'Yes':
type_.append('local program')
if version.urgency == 'Yes':
type_.append('urgency')
if version.taxlevy == 'Yes':
type_.append('tax levy')
if version.subject:
subject = clean_title(version.subject)
if not title:
self.warning("Couldn't find title for %s, skipping" % bill_id)
continue
fsbill['title'] = title
fsbill['summary'] = summary
fsbill['type'] = type_
fsbill['subjects'] = filter(None, [subject])
fsbill['impact_clause'] = impact_clause
# We don't want the current title in alternate_titles
all_titles.remove(title)
fsbill['alternate_titles'] = list(all_titles)
for author in version.authors:
if author.house == chamber_name:
fsbill.add_sponsor(SPONSOR_TYPES[author.contribution],
author.name,
official_type=author.contribution)
seen_actions = set()
for action in bill.actions:
if not action.action:
# NULL action text seems to be an error on CA's part,
# unless it has some meaning I'm missing
continue
actor = action.actor or chamber
actor = actor.strip()
match = re.match(r'(Assembly|Senate)($| \(Floor)', actor)
if match:
actor = {'Assembly': 'lower',
'Senate': 'upper'}[match.group(1)]
elif actor.startswith('Governor'):
actor = 'other'
else:
def replacer(matchobj):
if matchobj:
return {'Assembly': 'lower',
'Senate': 'upper'}[matchobj.group()]
else:
return matchobj.group()
actor = re.sub(r'^(Assembly|Senate)', replacer, actor)
type_ = []
act_str = action.action
act_str = re.sub(r'\s+', ' ', act_str)
attrs = self.categorizer.categorize(act_str)
# Add in the committee strings of the related committees, if any.
kwargs = attrs
matched_abbrs = committee_abbr_regex.findall(action.action)
if re.search(r'Com[s]?. on', action.action) and not matched_abbrs:
msg = 'Failed to extract committee abbr from %r.'
self.logger.warning(msg % action.action)
if matched_abbrs:
committees = []
for abbr in matched_abbrs:
try:
name = self.committee_abbr_to_name(chamber, abbr)
committees.append(name)
except KeyError:
msg = ('Mapping contains no committee name for '
'abbreviation %r. Action text was %r.')
args = (abbr, action.action)
raise KeyError(msg % args)
committees = filter(None, committees)
kwargs['committees'] = committees
code = re.search(r'C[SXZ]\d+', actor)
if code is not None:
code = code.group()
kwargs['actor_info'] = {'committee_code': code}
assert len(committees) == len(matched_abbrs)
for committee, abbr in zip(committees, matched_abbrs):
act_str = act_str.replace('Coms. on ', '')
act_str = act_str.replace('Com. on ' + abbr, committee)
act_str = act_str.replace(abbr, committee)
if not act_str.endswith('.'):
act_str = act_str + '.'
# Determine which chamber the action originated from.
changed = False
for committee_chamber in ['upper', 'lower', 'joint']:
if actor.startswith(committee_chamber):
actor = committee_chamber
changed = True
break
if not changed:
actor = 'other'
if actor != action.actor:
actor_info = kwargs.get('actor_info', {})
actor_info['details'] = action.actor
kwargs['actor_info'] = actor_info
# Add strings for related legislators, if any.
rgx = '(?:senator|assembly[mwp][^ .,:;]+)\s+[^ .,:;]+'
legislators = re.findall(rgx, action.action, re.I)
if legislators:
kwargs['legislators'] = legislators
date = action.action_date
date = self._tz.localize(date)
date = date.date()
if (actor, act_str, date) in seen_actions:
continue
fsbill.add_action(actor, act_str, date, **kwargs)
seen_actions.add((actor, act_str, date))
for vote in bill.votes:
if vote.vote_result == '(PASS)':
result = True
else:
result = False
full_loc = vote.location.description
first_part = full_loc.split(' ')[0].lower()
if first_part in ['asm', 'assembly']:
vote_chamber = 'lower'
vote_location = ' '.join(full_loc.split(' ')[1:])
elif first_part.startswith('sen'):
vote_chamber = 'upper'
vote_location = ' '.join(full_loc.split(' ')[1:])
else:
raise ScrapeError("Bad location: %s" % full_loc)
if vote.motion:
motion = vote.motion.motion_text or ''
else:
motion = ''
if "Third Reading" in motion or "3rd Reading" in motion:
vtype = 'passage'
elif "Do Pass" in motion:
vtype = 'passage'
else:
vtype = 'other'
motion = motion.strip()
# Why did it take until 2.7 to get a flags argument on re.sub?
motion = re.compile(r'(\w+)( Extraordinary)? Session$',
re.IGNORECASE).sub('', motion)
motion = re.compile(r'^(Senate|Assembly) ',
re.IGNORECASE).sub('', motion)
motion = re.sub(r'^(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.? ',
'', motion)
motion = re.sub(r' \(\w+\)$', '', motion)
motion = re.sub(r'(SCR|SB|AB|AJR|ACR)\s?\d+ \w+\.?$',
'', motion)
motion = re.sub(r'(SCR|SJR|SB|AB|AJR|ACR)\s?\d+ \w+\.? '
r'Urgency Clause$',
'(Urgency Clause)', motion)
motion = re.sub(r'\s+', ' ', motion)
if not motion:
self.warning("Got blank motion on vote for %s" % bill_id)
continue
fsvote = Vote(vote_chamber,
self._tz.localize(vote.vote_date_time),
motion,
result,
int(vote.ayes),
int(vote.noes),
int(vote.abstain),
threshold=vote.threshold,
type_=vtype)
if vote_location != 'Floor':
fsvote['committee'] = vote_location
for record in vote.votes:
if record.vote_code == 'AYE':
fsvote.yes(record.legislator_name)
elif record.vote_code.startswith('NO'):
fsvote.no(record.legislator_name)
else:
fsvote.other(record.legislator_name)
for s in ('yes', 'no', 'other'):
# Kill dupe votes.
key = s + '_votes'
fsvote[key] = list(set(fsvote[key]))
# In a small percentage of bills, the integer vote counts
# are inaccurate, so let's ignore them.
for k in ('yes', 'no', 'other'):
fsvote[k + '_count'] = len(fsvote[k + '_votes'])
fsbill.add_vote(fsvote)
self.save_bill(fsbill)
self.session.expire_all()
0
Example 196
View license def parse_bill(self, chamber, session, bill_id, url):
page = self.lxmlize(url)
short_bill_id = re.sub(r'(H|S)([JC])R', r'\1\2', bill_id)
version_link_node = self.get_node(
page,
'//a[contains(@href, "{bill_id}/bill.doc") or contains(@href,'
'"{bill_id}/bill.pdf")]'.format(bill_id=short_bill_id))
if version_link_node is None:
# Bill withdrawn
self.logger.warning('Bill withdrawn.')
return
else:
source_url = version_link_node.attrib['href']
if source_url.endswith('.doc'):
mimetype='application/msword'
elif source_url.endswith('.pdf'):
mimetype='application/pdf'
if self._is_post_2016:
title_texts = self.get_nodes(
page,
'//div[@class="StandardText leftDivMargin"]/text()')
title_texts = filter(None, [text.strip() for text in title_texts])\
[1:]
title_texts = [s for s in title_texts if s != ',']
title = ' '.join(title_texts)
actions = self.get_nodes(
page,
'//div[@class="StandardText leftDivMargin"]/'
'div[@class="StandardText"][last()]//text()[normalize-space()]')
else:
pars = version_link_node.xpath("following-sibling::p")
if len(pars) == 2:
title = pars[0].xpath("string()")
action_p = pars[1]
else:
title = pars[0].getprevious().tail
if not title:
self.warning('walking backwards to get bill title, error prone!')
title = pars[0].getprevious().getprevious()
while not title.tail:
title = title.getprevious()
title = title.tail
self.warning('got title the dangerous way: %s' % title)
action_p = pars[0]
title = re.sub(ur'[\s\xa0]+', ' ', title).strip()
actions = action_p.xpath("string()").split("\n")
if 'CR' in bill_id:
bill_type = 'concurrent resolution'
elif 'JR' in bill_id:
bill_type = 'joint resolution'
elif 'R' in bill_id:
bill_type = 'resolution'
else:
bill_type = 'bill'
bill = Bill(session, chamber, bill_id, title, type=bill_type)
bill['subjects'] = self._subjects[bill_id]
bill.add_source(url)
bill.add_version("Most Recent Version",
source_url,
mimetype=mimetype)
for link in page.xpath("//a[contains(@href, 'legislator/')]"):
bill.add_sponsor('primary', link.text.strip())
for line in actions:
line_actions = line.strip().split(';')
for index, action in enumerate(line_actions):
action = action.strip()
if not action:
continue
action_date_text = line.split('-')[0].strip()
if self._is_post_2016:
action_date_string = action_date_text.replace(',', '')
else:
action_date_string = '{} {}'.format(action_date_text,
session[0:4])
# This patch is super hacky, but allows us to better
# capture actions that screw up the formatting such as
# veto document links.
try:
action_date = datetime.datetime.strptime(
action_date_string, '%b %d %Y')
cached_action_date = action_date
used_cached_action_date = False
except:
action_date = cached_action_date
used_cached_action_date = True
# Separate out the date if first action on the line.
if index == 0 and not used_cached_action_date:
action = '-'.join(action.split('-')[1:]).strip()
if not action:
continue
if action.endswith('House') or action.endswith('(H)'):
actor = 'lower'
elif action.endswith('Senate') or action.endswith('(S)'):
actor = 'upper'
else:
actor = chamber
atype = []
if 'introduced in' in action:
atype.append('bill:introduced')
if 'to ' in action:
atype.append('committee:referred')
elif 'signed by Governor' in action:
atype.append('governor:signed')
elif 'vetoed' in action:
atype.append('governor:vetoed')
# Get the accompanying veto message document. There
# should only be one.
veto_document_link = self.get_node(page,
'//div[@class="StandardText leftDivMargin"]/'
'div[@class="StandardText"][last()]/a[contains(@href,'
'"veto.pdf")]')
if veto_document_link is not None:
bill.add_document("Veto Message",
veto_document_link.attrib['href'])
elif re.match(r'^to [A-Z]', action):
atype.append('committee:referred')
elif action == 'adopted by voice vote':
atype.append('bill:passed')
if '1st reading' in action:
atype.append('bill:reading:1')
if '3rd reading' in action:
atype.append('bill:reading:3')
if 'passed' in action:
atype.append('bill:passed')
if '2nd reading' in action:
atype.append('bill:reading:2')
if 'R' in bill_id and 'adopted by voice vote' in action:
atype.append('bill:passed')
amendment_re = (r'floor amendments?( \([a-z\d\-]+\))*'
r'( and \([a-z\d\-]+\))? filed')
if re.search(amendment_re, action):
atype.append('amendment:introduced')
if not atype:
atype = ['other']
# Capitalize the first letter of the action for nicer
# display. capitalize() won't work for this because it
# lowercases all other letters.
action = (action[0].upper() + action[1:])
if action:
bill.add_action(actor, action, action_date, type=atype)
try:
votes_link = page.xpath(
"//a[contains(@href, 'vote_history.pdf')]")[0]
bill.add_document("Vote History",
votes_link.attrib['href'])
except IndexError:
# No votes
self.logger.warning(u'No votes found for {}'.format(title))
pass
# Ugly Hack Alert!
# find actions before introduction date and subtract 1 from the year
# if the date is after introduction
intro_date = None
for i, action in enumerate(bill['actions']):
if 'bill:introduced' in action['type']:
intro_date = action['date']
break
for action in bill['actions'][:i]:
if action['date'] > intro_date:
action['date'] = action['date'].replace(year=action['date'].year-1)
self.debug('corrected year for %s', action['action'])
self.save_bill(bill)
0
Example 197
View license def scrape_bills(self, session, year_abr):
#Main Bill information
main_bill_csv = self.access_to_csv('MainBill')
# keep a dictionary of bills (mapping bill_id to Bill obj)
bill_dict = {}
for rec in main_bill_csv:
bill_type = rec["BillType"].strip()
bill_number = int(rec["BillNumber"])
bill_id = bill_type + str(bill_number)
title = rec["Synopsis"]
if bill_type[0] == 'A':
chamber = "lower"
else:
chamber = "upper"
# some bills have a blank title.. just skip it
if not title:
continue
bill = Bill(str(session), chamber, bill_id, title,
type=self._bill_types[bill_type[1:]])
if rec['IdenticalBillNumber'].strip():
bill.add_companion(rec['IdenticalBillNumber'].split()[0])
# TODO: last session info is in there too
bill_dict[bill_id] = bill
#Sponsors
bill_sponsors_csv = self.access_to_csv('BillSpon')
for rec in bill_sponsors_csv:
bill_type = rec["BillType"].strip()
bill_number = int(rec["BillNumber"])
bill_id = bill_type + str(bill_number)
if bill_id not in bill_dict:
self.warning('unknown bill %s in sponsor database' % bill_id)
continue
bill = bill_dict[bill_id]
name = rec["Sponsor"]
sponsor_type = rec["Type"]
if sponsor_type == 'P':
sponsor_type = "primary"
else:
sponsor_type = "cosponsor"
bill.add_sponsor(sponsor_type, name)
#Documents
bill_document_csv = self.access_to_csv('BillWP')
for rec in bill_document_csv:
bill_type = rec["BillType"].strip()
bill_number = int(rec["BillNumber"])
bill_id = bill_type + str(bill_number)
if bill_id not in bill_dict:
self.warning('unknown bill %s in document database' % bill_id)
continue
bill = bill_dict[bill_id]
document = rec["Document"]
document = document.split('\\')
document = document[-2] + "/" + document[-1]
year = str(year_abr) + str((year_abr + 1))
#doc_url = "ftp://www.njleg.state.nj.us/%s/%s" % (year, document)
htm_url = 'http://www.njleg.state.nj.us/%s/Bills/%s' % (year_abr,
document.replace('.DOC', '.HTM'))
# name document based _doctype
try:
doc_name = self._doctypes[rec['DocType']]
except KeyError:
raise Exception('unknown doctype %s on %s' %
(rec['DocType'], bill_id))
if rec['Comment']:
doc_name += ' ' + rec['Comment']
if rec['DocType'] in self._version_types:
# Clean HTMX links.
if htm_url.endswith('HTMX'):
htm_url = re.sub('X$', '', htm_url)
if htm_url.endswith('HTM'):
mimetype = 'text/html'
elif htm_url.endswith('wpd'):
mimetype = 'application/vnd.wordperfect'
bill.add_version(doc_name, htm_url, mimetype=mimetype)
else:
bill.add_document(doc_name, htm_url)
# Votes
next_year = int(year_abr)+1
vote_info_list = ['A%s' % year_abr,
'A%s' % next_year,
'S%s' % year_abr,
'S%s' % next_year,
'CA%s-%s' % (year_abr, next_year),
'CS%s-%s' % (year_abr, next_year),
]
for filename in vote_info_list:
s_vote_url = 'ftp://www.njleg.state.nj.us/votes/%s.zip' % filename
try:
s_vote_zip, resp = self.urlretrieve(s_vote_url)
except scrapelib.FTPError:
self.warning('could not find %s' % s_vote_url)
continue
zipedfile = zipfile.ZipFile(s_vote_zip)
for vfile in ["%s.txt" % (filename), "%sEnd.txt" % (filename)]:
try:
vote_file = zipedfile.open(vfile, 'U')
except KeyError:
#
# Right, so, 2011 we have an "End" file with more
# vote data than was in the original dump.
#
self.warning("No such file: %s" % (vfile))
continue
vdict_file = csv.DictReader(vote_file)
votes = {}
if filename.startswith('A') or filename.startswith('CA'):
chamber = "lower"
else:
chamber = "upper"
if filename.startswith('C'):
vote_file_type = 'committee'
else:
vote_file_type = 'chamber'
for rec in vdict_file:
if vote_file_type == 'chamber':
bill_id = rec["Bill"].strip()
leg = rec["Full_Name"]
date = rec["Session_Date"]
action = rec["Action"]
leg_vote = rec["Legislator_Vote"]
else:
bill_id = '%s%s' % (rec['Bill_Type'], rec['Bill_Number'])
leg = rec['Name']
# drop time portion
date = rec['Agenda_Date'].split()[0]
# make motion readable
action = self._com_vote_motions[rec['BillAction']]
# first char (Y/N) use [0:1] to ignore ''
leg_vote = rec['LegislatorVote'][0:1]
date = datetime.strptime(date, "%m/%d/%Y")
vote_id = '_'.join((bill_id, chamber, action))
vote_id = vote_id.replace(" ", "_")
if vote_id not in votes:
votes[vote_id] = Vote(chamber, date, action, None, None,
None, None, bill_id=bill_id)
if vote_file_type == 'committee':
votes[vote_id]['committee'] = self._committees[
rec['Committee_House']]
if leg_vote == "Y":
votes[vote_id].yes(leg)
elif leg_vote == "N":
votes[vote_id].no(leg)
else:
votes[vote_id].other(leg)
# remove temp file
os.remove(s_vote_zip)
#Counts yes/no/other votes and saves overall vote
for vote in votes.itervalues():
vote_yes_count = len(vote["yes_votes"])
vote_no_count = len(vote["no_votes"])
vote_other_count = len(vote["other_votes"])
vote["yes_count"] = vote_yes_count
vote["no_count"] = vote_no_count
vote["other_count"] = vote_other_count
# Veto override.
if vote['motion'] == 'OVERRIDE':
# Per the NJ leg's glossary, a veto override requires
# 2/3ds of each chamber. 27 in the senate, 54 in the house.
# http://www.njleg.state.nj.us/legislativepub/glossary.asp
vote['passed'] = False
if vote['chamber'] == 'lower':
if vote_yes_count >= 54:
vote['passed'] = True
elif vote['chamber'] == 'upper':
if vote_yes_count >= 27:
vote['passed'] = True
# Regular vote.
elif vote_yes_count > vote_no_count:
vote["passed"] = True
else:
vote["passed"] = False
vote_bill_id = vote["bill_id"]
bill = bill_dict[vote_bill_id]
bill.add_vote(vote)
#Actions
bill_action_csv = self.access_to_csv('BillHist')
actor_map = {'A': 'lower', 'G': 'executive', 'S': 'upper'}
for rec in bill_action_csv:
bill_type = rec["BillType"].strip()
bill_number = int(rec["BillNumber"])
bill_id = bill_type + str(bill_number)
if bill_id not in bill_dict:
self.warning('unknown bill %s in action database' % bill_id)
continue
bill = bill_dict[bill_id]
action = rec["Action"]
date = rec["DateAction"]
date = datetime.strptime(date, "%m/%d/%y %H:%M:%S")
actor = actor_map[rec["House"]]
comment = rec["Comment"]
action, atype = self.categorize_action(action, bill_id)
if comment:
action += (' ' + comment)
bill.add_action(actor, action, date, type=atype)
# Subjects
subject_csv = self.access_to_csv('BillSubj')
for rec in subject_csv:
bill_id = rec['BillType'].strip() + str(int(rec['BillNumber']))
if bill_id not in bill_dict:
self.warning('unknown bill %s in subject database' % bill_id)
continue
bill = bill_dict.get(bill_id)
if bill:
bill.setdefault('subjects', []).append(rec['SubjectKey'])
else:
self.warning('invalid bill id in BillSubj: %s' % bill_id)
phony_bill_count = 0
# save all bills at the end
for bill in bill_dict.itervalues():
# add sources
if not bill['actions'] and not bill['versions']:
self.warning('probable phony bill detected %s',
bill['bill_id'])
phony_bill_count += 1
else:
bill.add_source('http://www.njleg.state.nj.us/downloads.asp')
self.save_bill(bill)
if phony_bill_count:
self.warning('%s total phony bills detected', phony_bill_count)
0
Example 198
View licensedef main():
"""top main loop"""
collection_interval=DEFAULT_COLLECTION_INTERVAL
collect_every_cpu=True
if(sysload_conf):
config = sysload_conf.get_config()
collection_interval=config['collection_interval']
collect_every_cpu=config['collect_every_cpu']
global signal_received
signal.signal(signal.SIGTERM, handlesignal)
signal.signal(signal.SIGINT, handlesignal)
try:
if platform.system() == "FreeBSD":
if collect_every_cpu:
p_top = subprocess.Popen(
["top", "-S", "-P", "-n", "-s"+str(collection_interval), "-dinfinity", "0"],
stdout=subprocess.PIPE,
)
else:
p_top = subprocess.Popen(
["top", "-S", "-n", "-s"+str(collection_interval), "-dinfinity", "0"],
stdout=subprocess.PIPE,
)
else:
if collect_every_cpu:
p_top = subprocess.Popen(
["mpstat", "-P", "ALL", str(collection_interval)],
stdout=subprocess.PIPE,
)
else:
p_top = subprocess.Popen(
["mpstat", str(collection_interval)],
stdout=subprocess.PIPE,
)
except OSError, e:
if e.errno == errno.ENOENT:
# it makes no sense to run this collector here
sys.exit(13) # we signal tcollector to not run us
raise
timestamp = 0
while signal_received is None:
try:
line = p_top.stdout.readline()
except (IOError, OSError), e:
if e.errno in (errno.EINTR, errno.EAGAIN):
break
raise
if not line:
# end of the program, die
break
# CPU: --> CPU all: : FreeBSD, to match the all CPU
# %( [uni][a-z]+,?)? : FreeBSD, so that top output matches mpstat output
# AM : Linux, mpstat output depending on locale
# PM : Linux, mpstat output depending on locale
# .* load : FreeBSD, to correctly match load averages
# , : FreeBSD, to correctly match processes: Mem: ARC: and Swap:
fields = re.sub("CPU:", "CPU all:", re.sub(r"%( [uni][a-z]+,?)?| AM | PM |.* load |,", "", line)).split()
if len(fields) <= 0:
continue
if (((fields[0] == "CPU") or (re.match("[0-9][0-9]:[0-9][0-9]:[0-9][0-9]",fields[0]))) and ((collect_every_cpu and re.match("[0-9]+:?",fields[1])) or ((not collect_every_cpu) and re.match("all:?",fields[1])))):
if((fields[1] == "all") or (fields[1] == "0")):
timestamp = int(time.time())
cpuid=fields[1].replace(":","")
cpuuser=fields[2]
cpunice=fields[3]
cpusystem=fields[4]
cpuinterrupt=fields[6]
cpuidle=fields[-1]
print ("cpu.usr %s %s cpu=%s" % (timestamp, float(cpuuser), cpuid))
print ("cpu.nice %s %s cpu=%s" % (timestamp, float(cpunice), cpuid))
print ("cpu.sys %s %s cpu=%s" % (timestamp, float(cpusystem), cpuid))
print ("cpu.irq %s %s cpu=%s" % (timestamp, float(cpuinterrupt), cpuid))
print ("cpu.idle %s %s cpu=%s" % (timestamp, float(cpuidle), cpuid))
elif(fields[0] == "averages:"):
timestamp = int(time.time())
print ("load.1m %s %s" % (timestamp, fields[1]))
print ("load.5m %s %s" % (timestamp, fields[2]))
print ("load.15m %s %s" % (timestamp, fields[3]))
elif (re.match("[0-9]+ processes:",line)):
starting=0
running=0
sleeping=0
stopped=0
zombie=0
waiting=0
lock=0
for i in range(len(fields)):
if(fields[i] == "starting"):
starting=fields[i-1]
if(fields[i] == "running"):
running=fields[i-1]
if(fields[i] == "sleeping"):
sleeping=fields[i-1]
if(fields[i] == "stopped"):
stopped=fields[i-1]
if(fields[i] == "zombie"):
zombie=fields[i-1]
if(fields[i] == "waiting"):
waiting=fields[i-1]
if(fields[i] == "lock"):
lock=fields[i-1]
print ("ps.all %s %s" % (timestamp, fields[0]))
print ("ps.start %s %s" % (timestamp, starting))
print ("ps.run %s %s" % (timestamp, running))
print ("ps.sleep %s %s" % (timestamp, sleeping))
print ("ps.stop %s %s" % (timestamp, stopped))
print ("ps.zomb %s %s" % (timestamp, zombie))
print ("ps.wait %s %s" % (timestamp, waiting))
print ("ps.lock %s %s" % (timestamp, lock))
elif(fields[0] == "Mem:"):
active=0
inact=0
wired=0
cache=0
buf=0
free=0
for i in range(len(fields)):
if(fields[i] == "Active"):
active=convert_to_bytes(fields[i-1])
if(fields[i] == "Inact"):
inact=convert_to_bytes(fields[i-1])
if(fields[i] == "Wired"):
wired=convert_to_bytes(fields[i-1])
if(fields[i] == "Cache"):
cache=convert_to_bytes(fields[i-1])
if(fields[i] == "Buf"):
buf=convert_to_bytes(fields[i-1])
if(fields[i] == "Free"):
free=convert_to_bytes(fields[i-1])
print ("mem.active %s %s" % (timestamp, active))
print ("mem.inact %s %s" % (timestamp, inact))
print ("mem.wired %s %s" % (timestamp, wired))
print ("mem.cache %s %s" % (timestamp, cache))
print ("mem.buf %s %s" % (timestamp, buf))
print ("mem.free %s %s" % (timestamp, free))
elif(fields[0] == "ARC:"):
total=0
mru=0
mfu=0
anon=0
header=0
other=0
for i in range(len(fields)):
if(fields[i] == "Total"):
total=convert_to_bytes(fields[i-1])
if(fields[i] == "MRU"):
mru=convert_to_bytes(fields[i-1])
if(fields[i] == "MFU"):
mfu=convert_to_bytes(fields[i-1])
if(fields[i] == "Anon"):
anon=convert_to_bytes(fields[i-1])
if(fields[i] == "Header"):
header=convert_to_bytes(fields[i-1])
if(fields[i] == "Other"):
other=convert_to_bytes(fields[i-1])
print ("arc.total %s %s" % (timestamp, total))
print ("arc.mru %s %s" % (timestamp, mru))
print ("arc.mfu %s %s" % (timestamp, mfu))
print ("arc.anon %s %s" % (timestamp, anon))
print ("arc.header %s %s" % (timestamp, header))
print ("arc.other %s %s" % (timestamp, other))
elif(fields[0] == "Swap:"):
total=0
free=0
inuse=0
inps=0
outps=0
for i in range(len(fields)):
if(fields[i] == "Total"):
total=convert_to_bytes(fields[i-1])
if(fields[i] == "Free"):
free=convert_to_bytes(fields[i-1])
if(fields[i] == "Inuse"):
inuse=convert_to_bytes(fields[i-1])
if(fields[i] == "In"):
inps=convert_to_bytes(fields[i-1])/collection_interval
if(fields[i] == "Out"):
outps=convert_to_bytes(fields[i-1])/collection_interval
print ("swap.total %s %s" % (timestamp, total))
print ("swap.free %s %s" % (timestamp, free))
print ("swap.inuse %s %s" % (timestamp, inuse))
print ("swap.inps %s %s" % (timestamp, inps))
print ("swap.outps %s %s" % (timestamp, outps))
sys.stdout.flush()
if signal_received is None:
signal_received = signal.SIGTERM
try:
os.kill(p_top.pid, signal_received)
except Exception:
pass
p_top.wait()
0
Example 199
View licensedef convert(buildingsFile, osmOut):
with open(buildingsFile) as f:
buildings = json.load(f)
buildingShapes = []
buildingIdx = index.Index()
for building in buildings:
shape = asShape(building['geometry'])
buildingShapes.append(shape)
buildingIdx.add(len(buildingShapes) - 1, shape.bounds)
# Generates a new osm id.
osmIds = dict(node = -1, way = -1, rel = -1)
def newOsmId(type):
osmIds[type] = osmIds[type] - 1
return osmIds[type]
## Formats multi part house numbers
def formatHousenumber(p):
def suffix(part1, part2, hyphen_type=None):
part1 = stripZeroes(part1)
if not part2:
return str(part1)
part2 = stripZeroes(part2)
if hyphen_type == 'U': # unit numbers
return part1 + '-' + part2
if len(part2) == 1 and part2.isalpha(): # single letter extensions
return part1 + part2
return part1 + ' ' + part2 # All others
def stripZeroes(addr): # strip leading zeroes from numbers
if addr.isdigit():
addr = str(int(addr))
if '-' in addr:
try:
addr2 = addr.split('-')
if len(addr2) == 2:
addr = str(int(addr2[0])) + '-' + str(int(addr2[1])).zfill(2)
except:
pass
return addr
number = suffix(p['HOUSE_NUMB'], p['HOUSE_NU_1'], p['HYPHEN_TYP'])
return number
# Converts an address
def convertAddress(address):
result = dict()
if all (k in address for k in ('HOUSE_NUMB', 'STREET_NAM')):
if address['HOUSE_NUMB']:
result['addr:housenumber'] = formatHousenumber(address)
if address['STREET_NAM']:
streetname = address['STREET_NAM'].title()
streetname = streetname.replace('F D R ', 'FDR ')
# Expand Service Road
# See https://github.com/osmlab/nycbuildings/issues/30
streetname = re.sub(r"(.*)\bSr\b(.*)", r"\1Service Road\2", streetname)
# Expand cardinal directions on Service Roads
streetname = re.sub(r"(.*\bService Road\s)\bN\b(.*)", r"\1North\2", streetname)
streetname = re.sub(r"(.*\bService Road\s)\bE\b(.*)", r"\1East\2", streetname)
streetname = re.sub(r"(.*\bService Road\s)\bS\b(.*)", r"\1South\2", streetname)
streetname = re.sub(r"(.*\bService Road\s)\bW\b(.*)", r"\1West\2", streetname)
# Expand Expressway on Service Roads
streetname = re.sub(r"(.*)Expwy\s\bN\b(.*)", r"\1Expressway North\2", streetname)
streetname = re.sub(r"(.*)Expwy\s\bE\b(.*)", r"\1Expressway East\2", streetname)
streetname = re.sub(r"(.*)Expwy\s\bS\b(.*)", r"\1Expressway South\2", streetname)
streetname = re.sub(r"(.*)Expwy\s\bW\b(.*)", r"\1Expressway West\2", streetname)
streetname = re.sub(r"(.*)Expwy(.*)", r"\1Expressway\2", streetname)
# Add ordinal suffixes to numerals
streetname = re.sub(r"(.*)(\d*11)\s+(.*)", r"\1\2th \3", streetname)
streetname = re.sub(r"(.*)(\d*12)\s+(.*)", r"\1\2th \3", streetname)
streetname = re.sub(r"(.*)(\d*13)\s+(.*)", r"\1\2th \3", streetname)
streetname = re.sub(r"(.*)(\d*1)\s+(.*)", r"\1\2st \3", streetname)
streetname = re.sub(r"(.*)(\d*2)\s+(.*)", r"\1\2nd \3", streetname)
streetname = re.sub(r"(.*)(\d*3)\s+(.*)", r"\1\2rd \3", streetname)
streetname = re.sub(r"(.*)(\d+)\s+(.*)", r"\1\2th \3", streetname)
# Expand 'Ft' -> 'Fort'
if streetname[0:3] == 'Ft ': streetname = 'Fort ' + streetname[3:]
# Expand 'St ' -> 'Saint'
if streetname[0:3] == 'St ': streetname = 'Saint ' + streetname[3:]
# Expand 'Rev ' -> 'Reverend '
if streetname[0:4] == 'Rev ': streetname = 'Reverend ' + streetname[3:]
# Expand middlename ' St John' fix
streetname = streetname.replace('St John', 'Saint John')
# Middle name expansions
streetname = streetname.replace(' St ', ' Street ')
streetname = streetname.replace(' Rd ', ' Road ')
streetname = streetname.replace(' Blvd ', ' Boulevard ')
result['addr:street'] = streetname
if address['ZIPCODE']:
result['addr:postcode'] = str(int(address['ZIPCODE']))
return result
# Appends new node or returns existing if exists.
nodes = {}
def appendNewNode(coords, osmXml):
rlon = int(float(coords[0]*10**7))
rlat = int(float(coords[1]*10**7))
if (rlon, rlat) in nodes:
return nodes[(rlon, rlat)]
node = etree.Element('node', visible = 'true', id = str(newOsmId('node')))
node.set('lon', str(Decimal(coords[0])*Decimal(1)))
node.set('lat', str(Decimal(coords[1])*Decimal(1)))
nodes[(rlon, rlat)] = node
osmXml.append(node)
return node
def appendNewWay(coords, intersects, osmXml):
way = etree.Element('way', visible='true', id=str(newOsmId('way')))
firstNid = 0
for i, coord in enumerate(coords):
if i == 0: continue # the first and last coordinate are the same
node = appendNewNode(coord, osmXml)
if i == 1: firstNid = node.get('id')
way.append(etree.Element('nd', ref=node.get('id')))
# Check each way segment for intersecting nodes
int_nodes = {}
try:
line = LineString([coord, coords[i+1]])
except IndexError:
line = LineString([coord, coords[1]])
for idx, c in enumerate(intersects):
if line.buffer(0.000001).contains(Point(c[0], c[1])) and c not in coords:
t_node = appendNewNode(c, osmXml)
for n in way.iter('nd'):
if n.get('ref') == t_node.get('id'):
break
else:
int_nodes[t_node.get('id')] = Point(c).distance(Point(coord))
for n in sorted(int_nodes, key=lambda key: int_nodes[key]): # add intersecting nodes in order
way.append(etree.Element('nd', ref=n))
way.append(etree.Element('nd', ref=firstNid)) # close way
osmXml.append(way)
return way
# Appends an address to a given node or way.
def appendAddress(address, element):
for k, v in convertAddress(address['properties']).iteritems():
element.append(etree.Element('tag', k=k, v=v))
# Appends a building to a given OSM xml document.
def appendBuilding(building, shape, address, osmXml):
# Check for intersecting buildings
intersects = []
for i in buildingIdx.intersection(shape.bounds):
try:
for c in buildingShapes[i].exterior.coords:
if Point(c[0], c[1]).buffer(0.000001).intersects(shape):
intersects.append(c)
except AttributeError:
for c in buildingShapes[i][0].exterior.coords:
if Point(c[0], c[1]).buffer(0.000001).intersects(shape):
intersects.append(c)
# Export building, create multipolygon if there are interior shapes.
interiors = []
try:
way = appendNewWay(list(shape.exterior.coords), intersects, osmXml)
for interior in shape.interiors:
interiors.append(appendNewWay(list(interior.coords), [], osmXml))
except AttributeError:
way = appendNewWay(list(shape[0].exterior.coords), intersects, osmXml)
for interior in shape[0].interiors:
interiors.append(appendNewWay(list(interior.coords), [], osmXml))
if len(interiors) > 0:
relation = etree.Element('relation', visible='true', id=str(newOsmId('way')))
relation.append(etree.Element('member', type='way', role='outer', ref=way.get('id')))
for interior in interiors:
relation.append(etree.Element('member', type='way', role='inner', ref=interior.get('id')))
relation.append(etree.Element('tag', k='type', v='multipolygon'))
osmXml.append(relation)
way = relation
way.append(etree.Element('tag', k='building', v='yes'))
if 'HEIGHT_ROO' in building['properties']:
height = round(((building['properties']['HEIGHT_ROO'] * 12) * 0.0254), 1)
if height > 0:
way.append(etree.Element('tag', k='height', v=str(height)))
if 'BIN' in building['properties']:
way.append(etree.Element('tag', k='nycdoitt:bin', v=str(building['properties']['BIN'])))
if address: appendAddress(address, way)
# Export buildings & addresses. Only export address with building if there is exactly
# one address per building. Export remaining addresses as individual nodes.
allAddresses = []
osmXml = etree.Element('osm', version='0.6', generator='[email protected]')
for i in range(0, len(buildings)):
# Filter out special addresses categories A and B
buildingAddresses = []
for address in buildings[i]['properties']['addresses']:
if address['properties']['SPECIAL_CO'] not in ['A', 'B']:
buildingAddresses.append(address)
address = None
if len(buildingAddresses) == 1:
address = buildingAddresses[0]
else:
allAddresses.extend(buildingAddresses)
if int(buildings[i]['properties']['HEIGHT_ROO']) == 0:
if shape.area > 1e-09:
appendBuilding(buildings[i], buildingShapes[i], address, osmXml)
else:
appendBuilding(buildings[i], buildingShapes[i], address, osmXml)
# Export any addresses that aren't the only address for a building.
if (len(allAddresses) > 0):
for address in allAddresses:
node = appendNewNode(address['geometry']['coordinates'], osmXml)
appendAddress(address, node)
with open(osmOut, 'w') as outFile:
outFile.writelines(tostring(osmXml, pretty_print=True, xml_declaration=True, encoding='UTF-8'))
print 'Exported ' + osmOut
0
Example 200
View licensedef ss_get_mysql_stats(options):
# Process connection options and connect to MySQL.
global cache_dir, poll_time, chk_options
# Connect to MySQL.
host = options.host
user = options.user
passwd = options.password
port = options.port
heartbeat = options.heartbeat
db = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, cursorclass=MySQLdb.cursors.DictCursor)
cursor = db.cursor()
sanitized_host = host.replace(':', '').replace('/', '_')
sanitized_host = sanitized_host + '_' + str(port)
cache_file = os.path.join(cache_dir, '%s-mysql_graphite_stats.txt' % (sanitized_host))
log_debug('Cache file is %s' % (cache_file))
# First, check the cache.
fp = None
if not options.nocache:
with open(cache_file, 'a+') as fp:
try:
fcntl.flock(fp, fcntl.LOCK_SH) # LOCK_SH
try:
lines = open(cache_file).readlines()
except Exception:
lines = []
if os.path.getsize(cache_file) > 0 and \
os.path.getctime(cache_file) + (poll_time/2) > int(time.time()) and \
len(lines) > 0:
# The cache file is good to use
log_debug('Using the cache file')
return lines[0]
else:
log_debug('The cache file seems too small or stale')
try:
# Escalate the lock to exclusive, so we can write to it.
fcntl.flock(fp, fcntl.LOCK_EX) # LOCK_EX
try:
lines = open(cache_file).readlines()
except Exception:
lines = []
if os.path.getsize(cache_file) > 0 and \
os.path.getctime(cache_file) + (poll_time/2) > int(time.time()) and \
len(lines) > 0:
log_debug("Using the cache file")
return lines[0]
f.truncate(0)
except Exception:
pass
except IOError:
log_debug("Couldn't lock the cache file, ignoring it.")
fp = None
else:
log_debug("Couldn't open cache file")
fp = None
# Set up variables
status = { # Holds the result of SHOW STATUS, SHOW INNODB STATUS, etc
# Define some indexes so they don't cause errors with += operations
'relay_log_space' : None,
'binary_log_space' : None,
'current_transactions' : 0,
'locked_transactions' : 0,
'active_transactions' : 0,
'innodb_locked_tables' : 0,
'innodb_tables_in_use' : 0,
'innodb_lock_structs' : 0,
'innodb_lock_wait_secs' : 0,
'innodb_sem_waits' : 0,
'innodb_set_wait_time_ms' : 0,
# Values for the 'state' column from SHOW PROCESSLIST (converted to
# lowercase, with spaces replaced by underscores)
'State_closing_tables' : None,
'State_copying_to_tmp_table': None,
'State_end' : None,
'State_freeing_items' : None,
'State_init' : None,
'State_login' : None,
'State_preparing' : None,
'State_reading_from_net' : None,
'State_sending_data' : None,
'State_sorting_result' : None,
'State_statistics' : None,
'State_updating' : None,
'State_writing_to_net' : None,
'State_none' : None,
'State_other' : None, # Everything not listed above
}
# Get SHOW STATUS
cursor.execute("SHOW /*!50002 GLOBAL */ STATUS")
result = cursor.fetchall()
for row in result:
row = dict_change_key_case(row, case='lower')
status[row.get('variable_name')] = row.get('value')
# Get SHOW VARIABLES
cursor.execute('SHOW VARIABLES')
result = cursor.fetchall()
for row in result:
row = dict_change_key_case(row, case='lower')
status[row.get('variable_name')] = row.get('value')
# Get SHOW SLAVE STATUS
if chk_options.get('slave') and not options.no_replication_client:
cursor.execute('SHOW SLAVE STATUS')
result = cursor.fetchall()
slave_status_row_gotten = 0
for row in result:
slave_status_row_gotten += 1
# Must lowercase keys because different MySQL versions have different
# lettercase.
row = dict_change_key_case(row, case='lower')
status['relay_log_space'] = row.get('relay_log_space')
status['slave_lag'] = row.get('seconds_behind_master')
if len(heartbeat) > 0:
cursor.execute(
'SELECT MAX(GREATEST(0, UNIX_TIMESTAMP() - UNIX_TIMESTAMP(ts) - 1)) AS delay FROM %s' % (heartbeat)
)
result2 = cursor.fetchall()
slave_delay_rows_gotten = 0
for row2 in result2:
slave_delay_rows_gotten += 1
if type(row2) == dict and 'delay' in row2.keys():
status['slave_lag'] = row2.get('delay')
else:
log_debug("Couldn't get slave lag from %s" % (heartbeat))
if slave_delay_rows_gotten == 0:
log_debug('Got nothing from heartbeat query')
# Scale slave_running and slave_stopped relative to the slave lag.
status['slave_running'] = status.get('slave_lag') if row.get('slave_sql_running') == 'Yes' else 0
status['slave_stopped'] = 0 if row.get('slave_sql_running') == 'Yes' else status.get('slave_lag')
if slave_status_row_gotten == 0:
log_debug('Got nothing from SHOW SLAVE STATUS')
# Get SHOW MASTER STATUS
if chk_options.get('master') and not options.no_super and status.get('log_bin') == 'ON':
binlogs = []
cursor.execute('SHOW MASTER LOGS')
result = cursor.fetchall()
for row in result:
row = dict_change_key_case(row, case='lower')
# Older versions of MySQL may not have the File_size column in the
# results of the command. Zero-size files indicate the user is
# deleting binlogs manually from disk (bad user! bad!).
if 'file_size' in row.keys() and row.get('file_size') > 0:
binlogs.append(row.get('file_size'))
if len(binlogs) > 0:
status['binary_log_space'] = sum(binlogs)
# Get SHOW PROCESSLIST and aggregate it by state
if chk_options.get('procs'):
cursor.execute('SHOW PROCESSLIST')
result = cursor.fetchall()
for row in result:
state = row.get('state')
if state is None:
state = 'NULL'
if state == '':
state = 'none'
# MySQL 5.5 replaces the 'Locked' state with a variety of "Waiting for
# X lock" types of statuses. Wrap these all back into "Locked" because
# we don't really care about the type of locking it is.
state = re.sub('^(Table lock|Waiting for .*lock)$', 'Locked', state)
state = state.replace(' ', '_')
if 'State_%s' % (state) in status.keys():
increment(status, 'State_%s' % (state), 1)
else:
increment(status, 'State_other', 1)
# Get SHOW INNODB STATUS and extract the desired metrics from it
if chk_options.get('innodb') and status.get('have_innodb') == 'YES':
cursor.execute('SHOW /*!50000 ENGINE*/ INNODB STATUS')
result = cursor.fetchall()
istatus_text = result[0].get('Status')
istatus_vals = get_innodb_array(istatus_text)
if chk_options.get('get_qrt') and status.get('have_response_time_distribution') == 'YES':
log_debug('Getting query time histogram')
i = 0
cursor.execute(
'''
SELECT `count`, total * 1000000 AS total
FROM INFORMATION_SCHEMA.QUERY_RESPONSE_TIME
WHERE `time` <> 'TOO LONG'
'''
)
result = cursor.fetchall()
for row in result:
if i > 13:
# It's possible that the number of rows returned isn't 14.
# Don't add extra status counters.
break
count_key = 'Query_time_count_%02d' % (i)
total_key = 'Query_time_total_%02d' % (i)
status[count_key] = row['count']
status[total_key] = row['total']
i += 1
# It's also possible that the number of rows returned is too few.
# Don't leave any status counters unassigned; it will break graphs.
while i <= 13:
count_key = 'Query_time_count_%02d' % (i)
total_key = 'Query_time_total_%02d' % (i)
status[count_key] = 0
status[total_key] = 0
i += 1
else:
log_debug('Not getting time histogram because it is not enabled')
# Override values from InnoDB parsing with values from SHOW STATUS,
# because InnoDB status might not have everything and the SHOW STATUS is
# to be preferred where possible.
overrides = {
'Innodb_buffer_pool_pages_data' : 'database_pages',
'Innodb_buffer_pool_pages_dirty' : 'modified_pages',
'Innodb_buffer_pool_pages_free' : 'free_pages',
'Innodb_buffer_pool_pages_total' : 'pool_size',
'Innodb_data_fsyncs' : 'file_fsyncs',
'Innodb_data_pending_reads' : 'pending_normal_aio_reads',
'Innodb_data_pending_writes' : 'pending_normal_aio_writes',
'Innodb_os_log_pending_fsyncs' : 'pending_log_flushes',
'Innodb_pages_created' : 'pages_created',
'Innodb_pages_read' : 'pages_read',
'Innodb_pages_written' : 'pages_written',
'Innodb_rows_deleted' : 'rows_deleted',
'Innodb_rows_inserted' : 'rows_inserted',
'Innodb_rows_read' : 'rows_read',
'Innodb_rows_updated' : 'rows_updated',
}
# If the SHOW STATUS value exists, override...
for k,v in overrides.items():
if k in status.keys():
log_debug('Override %s' % (k))
istatus_vals[v] = status[k]
# Now copy the values into $status.
for k in istatus_vals.keys():
status[k] = istatus_vals[k]
# Make table_open_cache backwards-compatible (issue 63).
if 'table_open_cache' in status.keys():
status['table_cache'] = status.get('table_open_cache')
# Compute how much of the key buffer is used and unflushed (issue 127).
status['Key_buf_bytes_used'] = big_sub(status.get('key_buffer_size'), big_multiply(status.get('Key_blocks_unused'), status.get('key_cache_block_size')))
status['Key_buf_bytes_unflushed'] = big_multiply(status.get('Key_blocks_not_flushed'), status.get('key_cache_block_size'))
if 'unflushed_log' in status.keys() and status.get('unflushed_log'):
# TODO: I'm not sure what the deal is here; need to debug this. But the
# unflushed log bytes spikes a lot sometimes and it's impossible for it to
# be more than the log buffer.
log_debug('Unflushed log: %s' % (status.get('unflushed_log')))
status['unflushed_log'] = max(status.get('unflushed_log'), status.get('innodb_log_buffer_size'))
keys = [
'Key_read_requests',
'Key_reads',
'Key_write_requests',
'Key_writes',
'history_list',
'innodb_transactions',
'read_views',
'current_transactions',
'locked_transactions',
'active_transactions',
'pool_size',
'free_pages',
'database_pages',
'modified_pages',
'pages_read',
'pages_created',
'pages_written',
'file_fsyncs',
'file_reads',
'file_writes',
'log_writes',
'pending_aio_log_ios',
'pending_aio_sync_ios',
'pending_buf_pool_flushes',
'pending_chkp_writes',
'pending_ibuf_aio_reads',
'pending_log_flushes',
'pending_log_writes',
'pending_normal_aio_reads',
'pending_normal_aio_writes',
'ibuf_inserts',
'ibuf_merged',
'ibuf_merges',
'spin_waits',
'spin_rounds',
'os_waits',
'rows_inserted',
'rows_updated',
'rows_deleted',
'rows_read',
'Table_locks_waited',
'Table_locks_immediate',
'Slow_queries',
'Open_files',
'Open_tables',
'Opened_tables',
'innodb_open_files',
'open_files_limit',
'table_cache',
'Aborted_clients',
'Aborted_connects',
'Max_used_connections',
'Slow_launch_threads',
'Threads_cached',
'Threads_connected',
'Threads_created',
'Threads_running',
'max_connections',
'thread_cache_size',
'Connections',
'slave_running',
'slave_stopped',
'Slave_retried_transactions',
'slave_lag',
'Slave_open_temp_tables',
'Qcache_free_blocks',
'Qcache_free_memory',
'Qcache_hits',
'Qcache_inserts',
'Qcache_lowmem_prunes',
'Qcache_not_cached',
'Qcache_queries_in_cache',
'Qcache_total_blocks',
'query_cache_size',
'Questions',
'Com_update',
'Com_insert',
'Com_select',
'Com_delete',
'Com_replace',
'Com_load',
'Com_update_multi',
'Com_insert_select',
'Com_delete_multi',
'Com_replace_select',
'Select_full_join',
'Select_full_range_join',
'Select_range',
'Select_range_check',
'Select_scan',
'Sort_merge_passes',
'Sort_range',
'Sort_rows',
'Sort_scan',
'Created_tmp_tables',
'Created_tmp_disk_tables',
'Created_tmp_files',
'Bytes_sent',
'Bytes_received',
'innodb_log_buffer_size',
'unflushed_log',
'log_bytes_flushed',
'log_bytes_written',
'relay_log_space',
'binlog_cache_size',
'Binlog_cache_disk_use',
'Binlog_cache_use',
'binary_log_space',
'innodb_locked_tables',
'innodb_lock_structs',
'State_closing_tables',
'State_copying_to_tmp_table',
'State_end',
'State_freeing_items',
'State_init',
'State_locked',
'State_login',
'State_preparing',
'State_reading_from_net',
'State_sending_data',
'State_sorting_result',
'State_statistics',
'State_updating',
'State_writing_to_net',
'State_none',
'State_other',
'Handler_commit',
'Handler_delete',
'Handler_discover',
'Handler_prepare',
'Handler_read_first',
'Handler_read_key',
'Handler_read_next',
'Handler_read_prev',
'Handler_read_rnd',
'Handler_read_rnd_next',
'Handler_rollback',
'Handler_savepoint',
'Handler_savepoint_rollback',
'Handler_update',
'Handler_write',
'innodb_tables_in_use',
'innodb_lock_wait_secs',
'hash_index_cells_total',
'hash_index_cells_used',
'total_mem_alloc',
'additional_pool_alloc',
'uncheckpointed_bytes',
'ibuf_used_cells',
'ibuf_free_cells',
'ibuf_cell_count',
'adaptive_hash_memory',
'page_hash_memory',
'dictionary_cache_memory',
'file_system_memory',
'lock_system_memory',
'recovery_system_memory',
'thread_hash_memory',
'innodb_sem_waits',
'innodb_sem_wait_time_ms',
'Key_buf_bytes_unflushed',
'Key_buf_bytes_used',
'key_buffer_size',
'Innodb_row_lock_time',
'Innodb_row_lock_waits',
'Query_time_count_00',
'Query_time_count_01',
'Query_time_count_02',
'Query_time_count_03',
'Query_time_count_04',
'Query_time_count_05',
'Query_time_count_06',
'Query_time_count_07',
'Query_time_count_08',
'Query_time_count_09',
'Query_time_count_10',
'Query_time_count_11',
'Query_time_count_12',
'Query_time_count_13',
'Query_time_total_00',
'Query_time_total_01',
'Query_time_total_02',
'Query_time_total_03',
'Query_time_total_04',
'Query_time_total_05',
'Query_time_total_06',
'Query_time_total_07',
'Query_time_total_08',
'Query_time_total_09',
'Query_time_total_10',
'Query_time_total_11',
'Query_time_total_12',
'Query_time_total_13',
]
# Return the output.
output = []
for k in keys:
# If the value isn't defined, return -1 which is lower than (most graphs')
# minimum value of 0, so it'll be regarded as a missing value.
val = status.get(k) if status.get(k) is not None else -1
output.append('%s:%s' % (k, str(val)))
result = ' '.join(output)
if fp is not None:
with open(cache_file, 'w+') as fp:
fp.write('%s\n' % result)
db.close()
return result
