json.load

Here are the examples of the python api json.load taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: engine-python
Source File: EngineApiClient.py
View license
    def getBucketsByDate(self, job_id, start_date, end_date, include_records=False,
            normalized_probability_filter_value=None, anomaly_score_filter_value=None,
            include_interim=False):
        """
        Return all the job's buckets results between 2 dates.  If there is more
        than one page of results for the given data range this function will
        get them all appending the buckets in a list. A list of buckets is
        returned.

        start_date, end_date Must either be an epoch time or ISO 8601 format
        strings see the Prelert Engine API docs for help.
        include_records Anomaly records are included in the buckets
        normalized_probability_filter_value If not none return only the records with
            a normalizedProbability >= normalized_probability_filter_value
        anomaly_score_filter_value If not none return only the records with
            an anomalyScore >= anomaly_score_filter_value
        include_interim Should interim results be returned as well as final results?

        Returns a (http_status_code, buckets) tuple if successful else
        if http_status_code != 200 a (http_status_code, error_doc) is
        returned
        """

        skip = 0
        take = 100
        expand = ''
        if include_records:
            expand = '&expand=true'

        start_arg = ''
        if start_date:
            start_arg = '&start=' + urllib.quote(start_date)

        end_arg = ''
        if end_date:
            end_arg = '&end=' + urllib.quote(end_date)

        score_filter = ''
        if normalized_probability_filter_value:
            score_filter = '&normalizedProbability=' + str(normalized_probability_filter_value)

        if anomaly_score_filter_value:
            score_filter += '&anomalyScore=' + str(anomaly_score_filter_value)

        include_interim_arg = ''
        if include_interim:
            include_interim_arg = '&includeInterim=true'

        url = self.base_url + "/results/{0}/buckets?skip={1}&take={2}{3}{4}{5}{6}{7}".format(job_id,
            skip, take, expand, start_arg, end_arg, score_filter, include_interim_arg)

        self.connection.connect()
        self.connection.request("GET", url)
        response = self.connection.getresponse();

        if response.status != 200:
            logging.error("Get buckets by date response = " + str(response.status) + " " + response.reason)
            response_data = json.load(response)
            return (response.status, response_data)
        else:
            logging.debug("Get buckets by date response = " + str(response.status))


        result = json.load(response)
        buckets = result['documents']

        # is there another page of results
        while result['nextPage']:
            skip += take
            url = self.base_url + "/results/{0}/buckets?skip={1}&take={2}{3}{4}{5}".format(job_id,
                                skip, take, expand, start_arg, end_arg)
            self.connection.request("GET", url)
            response = self.connection.getresponse();
            if response.status != 200:
                logging.error("Get buckets by date response = " + str(response.status) + " " + response.reason)
                message = json.load(response)

                if message:
                    message = json.loads(message)

                self.connection.close()
                return (response.status, message)

            result = json.load(response)
            buckets.extend(result['documents'])

        self.connection.close()

        return (200, buckets)

Example 2

Project: engine-python
Source File: EngineApiClient.py
View license
    def getAllBuckets(self, job_id, include_records=False,
                normalized_probability_filter_value=None, anomaly_score_filter_value=None,
                include_interim=False):
        """
        Return all the job's buckets results.  If more than 1
        page of buckets are available continue to with the next
        page until all results have been read. An array of buckets is
        returned.

        include_records Anomaly records are included in the buckets
        normalized_probability_filter_value If not none return only the records with
            a normalizedProbability >= normalized_probability_filter_value
        anomaly_score_filter_value If not none return only the records with
            an anomalyScore >= anomaly_score_filter_value
        include_interim Should interim results be returned as well as final results?

        Returns a (http_status_code, buckets) tuple if successful else
        if http_status_code != 200 a (http_status_code, error_doc) tuple
        is returned
        """

        skip = 0
        take = 100
        expand = ''
        if include_records:
            expand = '&expand=true'

        score_filter = ''
        if normalized_probability_filter_value:
            score_filter = '&normalizedProbability=' + str(normalized_probability_filter_value)

        if anomaly_score_filter_value:
            score_filter += '&anomalyScore=' + str(anomaly_score_filter_value)

        include_interim_arg = ''
        if include_interim:
            include_interim_arg = '&includeInterim=true'

        url = self.base_url + "/results/{0}/buckets?skip={1}&take={2}{3}{4}{5}".format(
            job_id, skip, take, expand, score_filter, include_interim_arg)


        self.connection.connect()
        self.connection.request("GET", url)
        response = self.connection.getresponse();

        if response.status != 200:
            logging.error("Get all buckets response = " + str(response.status) + " " + response.reason)
            response_data = json.load(response)
            return (response.status, response_data)
        else:
            logging.debug("Get all buckets response = " + str(response.status))


        result = json.load(response)
        buckets = result['documents']

        # is there another page of results
        while result['nextPage']:
            skip += take
            url = self.base_url + "/results/{0}/buckets?skip={1}&take={2}{3}{4}".format(
                job_id, skip, take, expand, score_filter)

            self.connection.request("GET", url)
            response = self.connection.getresponse();
            if response.status != 200:
                logging.error("Get all buckets response = " + str(response.status) + " " + response.reason)

                message = json.load(response)
                if message:
                    message = json.loads(message)

                self.connection.close()
                return (response.status, message)

            result = json.load(response)
            buckets.extend(result['documents'])

        self.connection.close()

        return (200, buckets)

Example 3

Project: folium
Source File: features.py
View license
    def __init__(self, data, style_function=None, name=None,
                 overlay=True, control=True, smooth_factor=None,
                 highlight_function=None):
        super(GeoJson, self).__init__(name=name, overlay=overlay,
                                      control=control)
        self._name = 'GeoJson'
        if hasattr(data, 'read'):
            self.embed = True
            self.data = json.load(data)
        elif isinstance(data, dict):
            self.embed = True
            self.data = data
        elif isinstance(data, text_type) or isinstance(data, binary_type):
            if data.lstrip()[0] in '[{':  # This is a GeoJSON inline string
                self.embed = True
                self.data = json.loads(data)
            else:  # This is a filename
                self.embed = False
                self.data = data
        elif data.__class__.__name__ in ['GeoDataFrame', 'GeoSeries']:
            self.embed = True
            if hasattr(data, '__geo_interface__'):
                # We have a GeoPandas 0.2 object.
                self.data = json.loads(json.dumps(data.to_crs(epsg='4326').__geo_interface__))  # noqa
            elif hasattr(data, 'columns'):
                # We have a GeoDataFrame 0.1
                self.data = json.loads(data.to_crs(epsg='4326').to_json())
            else:
                msg = 'Unable to transform this object to a GeoJSON.'
                raise ValueError(msg)
        else:
            raise ValueError('Unhandled object {!r}.'.format(data))

        if style_function is None:
            def style_function(x):
                return {}

        self.style_function = style_function

        self.highlight = highlight_function is not None

        if highlight_function is None:
            def highlight_function(x):
                return {}

        self.highlight_function = highlight_function

        self.smooth_factor = smooth_factor

        self._template = Template(u"""
            {% macro script(this, kwargs) %}

            {% if this.highlight %}
                {{this.get_name()}}_onEachFeature = function onEachFeature(feature, layer) {
                    layer.on({
                        mouseout: function(e) {
                            e.target.setStyle(e.target.feature.properties.style);},
                        mouseover: function(e) {
                            e.target.setStyle(e.target.feature.properties.highlight);},
                        click: function(e) {
                            {{this._parent.get_name()}}.fitBounds(e.target.getBounds());}
                        });
                };
            {% endif %}

                var {{this.get_name()}} = L.geoJson(
                    {% if this.embed %}{{this.style_data()}}{% else %}"{{this.data}}"{% endif %}
                    {% if this.smooth_factor is not none or this.highlight %}
                        , {
                        {% if this.smooth_factor is not none  %}
                            smoothFactor:{{this.smooth_factor}}
                        {% endif %}

                        {% if this.highlight %}
                            {% if this.smooth_factor is not none  %}
                            ,
                            {% endif %}
                            onEachFeature: {{this.get_name()}}_onEachFeature
                        {% endif %}
                        }
                    {% endif %}
                    ).addTo({{this._parent.get_name()}});
                {{this.get_name()}}.setStyle(function(feature) {return feature.properties.style;});

            {% endmacro %}
            """)  # noqa

Example 4

Project: PokemonGo-SlackBot
Source File: pokeslack.py
View license
def main():
    full_path = os.path.realpath(__file__)
    (path, filename) = os.path.split(full_path)

    args = get_args()

    if args.auth_service not in ['ptc', 'google']:
        print '[!] Invalid Auth service specified'
        return

    print('[+] Locale is ' + args.locale + ' and icon locale is ' + args.icon_locale)
    pokemonsJSON = json.load(
        codecs.open(path + '/locales/pokemon.' + args.locale + '.json', "r", 'UTF-8'))

    pokemonsJSON_icons = json.load(
        codecs.open(path + '/locales/pokemon.' + args.icon_locale + '.json', "r", 'UTF-8'))

    translationsJSON = json.load(
        codecs.open(path + '/locales/translations.en.json', "r", 'UTF-8'))
    if os.path.isfile(path + '/locales/translations.' + args.locale + '.json'):
        overrideTranslationsJSON = json.load(
            codecs.open(path + '/locales/translations.' + args.locale + '.json', "r", 'UTF-8'));
        translationsJSON = merge(translationsJSON, overrideTranslationsJSON)

    if args.debug:
        global DEBUG
        DEBUG = True
        print '[!] DEBUG mode on'

    # only get location for first run
    if not (FLOAT_LAT and FLOAT_LONG):
      print('[+] Getting initial location')
      retrying_set_location(args.location)

    if args.auto_refresh:
        global auto_refresh
        auto_refresh = int(args.auto_refresh) * 1000

    if args.ampm_clock:
    	global is_ampm_clock
    	is_ampm_clock = True

    global api_last_response, first_connection, api_endpoint, access_token, profile_response

    if first_connection:
        print '[+] Connecting'
        api_endpoint, access_token, profile_response = connection.login(args)
        api_last_response = datetime.now()
        first_connection = False
    elif datetime.now() - api_last_response > max_idle_time and args.auth_service == 'google':
        print '[!] Resetting connection...'
        connection.login.reset()
        time.sleep(wait_to_reconnect)
        api_endpoint, access_token, profile_response = connection.login(args)
        api_last_response = datetime.now()

    clear_stale_pokemons()

    steplimit = int(args.step_limit)

    global max_distance
    max_distance = int(args.range)

    global slack_webhook_urlpath
    slack_webhook_urlpath = str(args.slack_webhook)

    global pokemon_icons_prefix
    if args.pokemon_icons:
        pokemon_icons_prefix = args.pokemon_icons
    else:
        pokemon_icons_prefix = False

    ignore = []
    only = []
    if args.ignore:
        ignore = [i.lower().strip() for i in args.ignore.split(',')]
    elif args.only:
        only = [i.lower().strip() for i in args.only.split(',')]

    pos = 1
    x = 0
    y = 0
    dx = 0
    dy = -1
    steplimit2 = steplimit**2
    for step in range(steplimit2):
        #starting at 0 index
        debug('looping: step {} of {}'.format((step+1), steplimit**2))
        #debug('steplimit: {} x: {} y: {} pos: {} dx: {} dy {}'.format(steplimit2, x, y, pos, dx, dy))
        # Scan location math
        if -steplimit2 / 2 < x <= steplimit2 / 2 and -steplimit2 / 2 < y <= steplimit2 / 2:
            set_location_coords(x * 0.0025 + origin_lat, y * 0.0025 + origin_lon, 0)
        if x == y or x < 0 and x == -y or x > 0 and x == 1 - y:
            (dx, dy) = (-dy, dx)

        (x, y) = (x + dx, y + dy)

        process_step(args, api_endpoint, access_token, profile_response,
                     pokemonsJSON, pokemonsJSON_icons, translationsJSON, ignore, only)


        print('Completed: ' + str(
            ((step+1) + pos * .25 - .25) / (steplimit2) * 100) + '%')

    global NEXT_LAT, NEXT_LONG
    if (NEXT_LAT and NEXT_LONG and
            (NEXT_LAT != FLOAT_LAT or NEXT_LONG != FLOAT_LONG)):
        print('Update to next location %f, %f' % (NEXT_LAT, NEXT_LONG))
        set_location_coords(NEXT_LAT, NEXT_LONG, 0)
        NEXT_LAT = 0
        NEXT_LONG = 0
    else:
        set_location_coords(origin_lat, origin_lon, 0)

    register_background_thread()

Example 5

Project: ShaniXBMCWork2
Source File: iptv.py
View license
def getAllChannels(portal_mac, url, serial, path):

    added = False;
    
    now = time();
    
    try:
        portalurl = "_".join(re.findall("[a-zA-Z0-9]+", url));
        portalurl = path + '/' + portalurl
        
        setMac(portal_mac);
        setSerialNumber(serial);
        
        if not os.path.exists(path):
            os.makedirs(path)

        if os.path.exists(portalurl):
            #check last time
            with open(portalurl) as data_file: data = json.load(data_file);
        
            if 'version' not in data or data['version'] != cache_version:
                clearCache(url, path);
                
            else:
                time_init = float(data['time']);
                # update 2h
                if ((now - time_init) / 3600) < 6:
                    return data;
        
        handshake(url);
        
        genres = getGenres(portal_mac, url, serial, path);
        genres = genres["genres"];

        
        info = retrieveData(url, values = {
            'type' : 'itv', 
            'action' : 'get_all_channels',
            'JsHttpRequest' : '1-xml'})
        
        
        results = info['js']['data'];

        data = '{ "version" : "' + cache_version + '", "time" : "' + str(now) + '", "channels" : { \n'

        for i in results:
            id 		= i["id"]
            number 	= i["number"]
            name 	= i["name"]
            cmd 	= i['cmd']
            logo 	= i["logo"]
            tmp 	= i["use_http_tmp_link"]
            genre_id 	= i["tv_genre_id"];
            
            genre_title = genres[genre_id]['title'];
            
            _s1 = cmd.split(' ');	
            _s2 = _s1[0];
            if len(_s1)>1:
                _s2 = _s1[1];
            
            added = True;
            data += '"' + id + '": {"number":"'+ number +'", "name":"'+ name +'", "cmd":"'+ cmd +'", "logo":"'+ logo +'", "tmp":"'+ str(tmp) +'", "genre_id":"'+ str(genre_id) +'", "genre_title":"'+ genre_title +'"}, \n'


        page = 1;
        pages = 0;
        total_items = 0;
        max_page_items = 0;

        while True:
            # retrieve adults
            info = retrieveData(url, values = {
                'type' : 'itv', 
                'action' : 'get_ordered_list',
                'genre' : '10',
                'p' : page,
                'fav' : '0',
                'JsHttpRequest' : '1-xml'})
        
            total_items = float(info['js']['total_items']);
            max_page_items = float(info['js']['max_page_items']);
            pages = math.ceil(total_items/max_page_items);
        
            results = info['js']['data']

            for i in results:
                id 		= i["id"]
                number 	= i["number"]
                name 	= i["name"]
                cmd 	= i['cmd']
                logo 	= i["logo"]
                tmp 	= i["use_http_tmp_link"]
                genre_id 	= i["tv_genre_id"];
                genre_title = genres[genre_id]['title'];
            
                data += '"' + id + '": {"number":"'+ number +'", "name":"'+ name +'", "cmd":"'+ cmd +'", "logo":"'+ logo +'", "tmp":"'+ str(tmp) +'", "genre_id":"'+ str(genre_id) +'", "genre_title":"'+ genre_title +'"}, \n'
                
                added = True;

            page += 1;
            if page > pages:
                break;
        

        if not added:
            data = data + '\n}}';
        else:
            data = data[:-3] + '\n}}';

        
        with open(portalurl, 'w') as f: f.write(data.encode('utf-8'));
        
        return json.loads(data.encode('utf-8'));
    except: 
        print 'failed to get list reusing old one'
        import base64
        request = urllib2.Request(base64.b64decode('aHR0cDovL3Bhc3RlYmluLmNvbS9yYXcucGhwP2k9bkhZVjFKblQ='))
        response  = urllib2.urlopen(request);
        data = response.read().decode("utf-8");
        return json.loads(data)

Example 6

View license
def run(options, args):
    query = {'type__in': ['notice', 'proposed_rule', 'rule']}

    for filter_type in ('agency', 'docket_id', 'source'):
        filter_attr = getattr(options, filter_type)
        if filter_attr:
            query[filter_type] = filter_attr

    frn, frc, g, nd = 0, 0, 0, 0
    for doc in Doc.objects(**query):
        if 'fr_data' in doc.annotations and not options.process_all:
            continue
        
        fr_num = None
        fr_cite = None

        if 'Federal_Register_Number' in doc.details:
            print doc.id, 'FR num', doc.details['Federal_Register_Number'].encode('utf8')
            frn += 1

            # try fetching now; maybe we're done, but we can always try one of the other tactics if this doesn't work
            fr_num = doc.details['Federal_Register_Number'].encode('utf8')
            try:
                doc.annotations['fr_data'] = json.load(urllib2.urlopen("https://www.federalregister.gov/api/v1/articles/" + fr_num))
                doc.save()
                print 'Succeeded with %s via FR number' % doc.id
                continue
            except:
                fr_num = None

        if 'Federal_Register_Citation' in doc.details:
            print doc.id, 'FR cite', doc.details['Federal_Register_Citation'].encode('utf8')
            frc += 1
            fr_cite = doc.details['Federal_Register_Citation'].encode('utf8')
            fr_num = fr_num_from_cite(fr_cite, doc.title)
            if fr_num:
                # try again
                try:
                    doc.annotations['fr_data'] = json.load(urllib2.urlopen("https://www.federalregister.gov/api/v1/articles/" + fr_num))
                    doc.save()
                    print 'Succeeded with %s via FR citation' % doc.id
                    continue
                except:
                    fr_cite = None
                    fr_num = None
            else:
                fr_cite = None

        if not fr_num and not fr_cite:
            # does it have a PDF copy of the Federal Register version of the thing?
            views = None
            att = [a for a in doc.attachments if 'Federal Register' in a.title]
            if att:
                views = [v for v in att[0].views if v.type == 'pdf']

            if not views:
                views = [v for v in doc.views if v.type == 'xpdf']
            
            if views:
                fr_cite = fr_citation_from_view(views[0])

            if fr_cite:
                print doc.id, 'FR cite (by PDF)', fr_cite
                frc += 1

                fr_num = fr_num_from_cite(fr_cite, doc.title)
                if fr_num:
                    # try again
                    try:
                        doc.annotations['fr_data'] = json.load(urllib2.urlopen("https://www.federalregister.gov/api/v1/articles/" + fr_num))
                        doc.save()
                        print 'Succeeded with %s via FR citation (PDF)' % doc.id
                        continue
                    except:
                        fr_cite = None
                        fr_num = None
                else:
                    fr_cite = None

            else:
                # last chance -- we guess from the title alone
                fr_num = guess_fr_num(doc)
                if fr_num:
                    # try again
                    try:
                        doc.annotations['fr_data'] = json.load(urllib2.urlopen("https://www.federalregister.gov/api/v1/articles/" + fr_num))
                        doc.save()
                        g += 1
                        print 'Succeeded with %s via title guessing' % doc.id
                        continue
                    except:
                        fr_cite = None
                        fr_num = None

        if not fr_num and not fr_cite:
            # we failed :/
            doc.annotations['fr_data'] = None
            doc.save()
            print doc.id, 'No dice'
            nd += 1
    print frn, frc, g, nd

Example 7

View license
def run(options, args):
    for agency in (options.agency,) if options.agency else ('CFTC', 'SEC'):
        lagency = agency.lower()

        all_dockets = {}
        dockets_for_saving = {}

        # first load the docket file
        dockets = json.load(open(os.path.join(settings.DUMP_DIR, "%s_dockets.json" % lagency)))
        docket_dir = os.path.join(settings.DUMP_DIR, "%s_dockets" % lagency)

        # next deal with the FR documents
        doc_by_identifier = {}
        cftc_ancient_mapping = {}
        all_fr_docs = []
        dockets_seen = set()

        fr_docs = json.load(open(os.path.join(settings.DUMP_DIR, "%s_fr_docs.json" % lagency)))
        for doc in fr_docs:
            if 'id' not in doc and 'url' in doc:
                doc['id'] = crockford_hash(doc['url'])

            if 'doctype' not in doc:
                doc['doctype'] = 'Federal Register Release'

            print "Processing FR doc %s in %s..." % (doc['id'].encode('utf8'), agency) 
            dc = fr_doc_record_to_model(doc, agency)
            for identifier in (doc['id'], dc.details.get('Federal_Register_Number', None), dc.details.get('Federal_Register_Citation', None)):
                if identifier:
                    doc_by_identifier[identifier] = dc

            # treat ancient CFTC FR docs specially because they'll show up again in the listing, so don't double count
            if agency == 'CFTC' and doc['strategy'] == 'ancient':
                if 'Federal_Register_Citation' in dc.details:
                    cftc_ancient_mapping[dc.details['Federal_Register_Citation'].split(" FR ")[-1]] = dc
            else:
                all_fr_docs.append(dc)
                dockets_seen.add(dc.docket_id)

        # now load docket files one by one and deal with docket records and comments
        all_comments = []
        for record in dockets.itervalues():
            json_file = os.path.join(docket_dir, "%s.json" % record['id'])
            
            file_exists = os.path.exists(json_file)
            
            if file_exists:
                full_record = json.load(open(json_file))

            print "Processing docket %s in %s..." % (record['id'], agency) 
            
            dkt = docket_record_to_model(full_record if file_exists else record, agency)
            all_dockets[dkt.id] = dkt

            if 'parent' in record:
                dkt.details['Parent'] = '%s-X-%s' % (agency, record['parent'])
            else:
                dockets_for_saving[dkt.id] = dkt

            if not file_exists:
                continue

            for comment_record in full_record['comments']:
                if 'doctype' not in comment_record:
                    comment_record['doctype'] = 'public_submission'

                if 'id' not in comment_record and 'url' in comment_record:
                    comment_record['id'] = crockford_hash(comment_record['url'])

                print "Processing comment %s in %s..." % (comment_record['id'], dkt.id) 
                cmt = comment_record_to_model(comment_record, agency, dkt.details['Parent'] if 'Parent' in dkt.details else dkt.id)
                
                if comment_record.get('release', None):
                    release = comment_record['release'][0]
                    if release in doc_by_identifier:
                        cmt.comment_on = {'document_id': doc_by_identifier[release].id}

                if 'Federal Register Page' in comment_record:
                    cmt.title = cftc_ancient_mapping[comment_record['details']['Federal Register Page']].title

                all_comments.append(cmt)

        print len(all_dockets), len(all_fr_docs), len(all_comments)

        # make sure we have docket records for all dockets that have documents in them
        for docket_id in dockets_seen:
            if docket_id not in dockets_for_saving:
                simple_dkt = docket_record_from_id(docket_id, agency)
                dockets_for_saving[docket_id] = simple_dkt
        
        for dkt in dockets_for_saving.itervalues():
            try:
                print "Attempting to save docket %s..." % dkt.id
                dkt.save(force_insert=True)
                print "Docket %s saved." % dkt.id
            except NotUniqueError:
                print "Docket %s already exists." % dkt.id
                if options.update:
                    print "Fetching docket %s for update..." % dkt.id

                    # fetch the current one
                    current = Docket.objects.get(id=dkt.id)
                    if dkt.title:
                        current.title = dkt.title

                    current.details = dkt.details
                    current.source = dkt.source
                    current.agency = dkt.agency

                    if current.scraped != 'yes':
                        current.scraped = dkt.scraped

                    current.save()
        
        for doc_set in (all_fr_docs, all_comments):
            for doc_obj in doc_set:
                try:
                    print "Attempting to save document %s..." % doc_obj.id
                    doc_obj.save(force_insert=True)
                    print "Document %s saved." % doc_obj.id
                except NotUniqueError:
                    print "Document %s already exists." % doc_obj.id
                    if options.update:
                        print "Fetching document %s for update..." % doc_obj.id

                        # fetch the current one
                        current = Doc.objects.get(id=doc_obj.id)
                        current.title = doc_obj.title
                        current.details = doc_obj.details

                        if len(current.views) != len(doc_obj.views):
                            current.views = doc_obj.views

                        if len(current.attachments) != len(doc_obj.attachments):
                            current.attachments = doc_obj.attachments

                        current.save()

Example 8

Project: TARDIS
Source File: TARDIS.py
View license
def main(vulnerability,vulnObject,sourceIP,sourceHost):
	#Create results and working directories
	if not os.path.exists('Results'):
		os.makedirs('Results')
	if not os.path.exists('Working'):
		os.makedirs('Working')
	
	#Make sure the vulnerability is valid
	if vulnerability != "":
		vulnCheck=0
		resultCount=0
		logsource=''
		print("Searching for evidence of \"" + vulnerability + "\"")
		print("  Host: " + sourceIP)
		
		try:
			configFile = 'config.xml'
			tree = ET.parse(configFile)
			root = tree.getroot()
		except:
			sys.exit("Not a valid config XML file")
		for settings in root.findall("./log_source"):
			logsource=settings.text
		cnx = getDBConnector()
		
		
		#check if vulnerability/asset combo exists in assetVulnerability Table
		cursor = cnx.cursor()
		query = ("SELECT count(*) as count from assetVulnerabilities where victim_ip = '" + str(ip2long(sourceIP)) + "' and threat_id = '" + vulnerability + "'")
		
		cursor.execute(query)
		for row in cursor:
			vulnCheck=row[0]
		cursor.close()
		
		if vulnCheck==0:
			#No combination exists, write data to DB
			
			cursor = cnx.cursor()
			add_vulnInstance = ("INSERT INTO assetVulnerabilities "
               "(victim_ip, threat_id, active) "
               "VALUES (%s, %s, %s)")
			vulnData = (ip2long(sourceIP), vulnerability, '1')
			
			# Insert new entry
			cursor.execute(add_vulnInstance , vulnData )
			
			cnx.commit()
			cursor.close()
			cnx.close()
		searchStringResults= findStixObservables.run(vulnerability)
		isExploitFound=False
		searchStringCount=0
		operator=searchStringResults[0]
		numResults=0
		if(searchStringResults[1]=="No search file found"):
			searchResults="0"
			print("  No search file found\n")
		elif(searchStringResults[1]=="No supported observables found"):
			searchResults="0"
			print("  No supported observables found\n")
		else:
			#run  search...
			#search should return number of results
			#Insert source host from arguments
			for entry in searchStringResults:
				if logsource=="splunk":
					if (searchStringCount == 1):
						searchString=entry + " AND (host=\"" + sourceHost + "\" OR s_ip=\"" + sourceIP + "\" OR d_host=\"" + sourceHost + "\")  | fields host, c_ip | fields - _bkt, _cd, _indextime, _kv, _serial, _si, _sourcetype | rename _raw as \"Raw Log\" | rename c_ip as clientip"
						numResults=splunk.searchVulnerability(searchString,vulnerability,sourceIP,sourceHost)
						if (numResults != "0"):
							data = json.load(numResults)
					
					if (operator=="AND"):
						if (searchStringCount > 1):
							resultCount=0
							for result in data["results"]:
								startTime =  dateutil.parser.parse(data["results"][resultCount]["_time"]) + datetime.timedelta(days =- 300)
								endTime =  dateutil.parser.parse(data["results"][resultCount]["_time"]) + datetime.timedelta(days = 300)
								searchString=entry + " AND (host=\"" + sourceHost + "\" OR s_ip=\"" + sourceIP + "\" OR d_host=\"" + sourceHost + "\") | fields host, clientip | fields - _bkt, _cd, _indextime, _kv, _serial, _si, _sourcetype | rename _raw as \"Raw Log\""
								newResults=splunk.searchVulnerabilityTimeRange(searchString,vulnerability,sourceIP,sourceHost,startTime.isoformat(),endTime.isoformat())
								if (newResults != "0"):
									#This is the result from search 1
									newData = json.load(newResults)
									newResultCount=0
									for result in newData["results"]:
										try:
											clientip=newData["results"][newResultCount]["clientip"]
										except:
											clientip="0"
										isExploitFound=True
										#These are the results from any further results proving the AND condition
										cnx = getDBConnector()
										cursor = cnx.cursor()
										query = ("SELECT count(*) as count from attackInventory where victim_ip = '" + str(ip2long(sourceIP)) + "' and threat_id = '" + vulnerability + "' and attack_time = '" + data["results"][resultCount]["_time"] + "'")
										cursor.execute(query)
										for row in cursor:
											logCheck=row[0]
										cursor.close()
										if logCheck==0:
											#Write data to DB
											cursor = cnx.cursor()
											add_logInstance = ("INSERT INTO attackInventory "
																"(victim_ip, attacker_ip, attack_time, attack_log, threat_id) "
																"VALUES (%s, %s, %s, %s, %s)")
											
											logData = (ip2long(sourceIP), ip2long(clientip), newData["results"][newResultCount]["_time"], newData["results"][newResultCount]["Raw Log"], vulnerability)
											# Insert new entry
											cursor.execute(add_logInstance , logData )
											cnx.commit()
											cursor.close()
										cnx.close()
										newResultCount=newResultCount+1
								else:
									newResultCount=0
							if (isExploitFound==True):
								try:
									clientip=data["results"][resultCount]["clientip"]
								except:
									clientip="0"
								cnx = getDBConnector()
								cursor = cnx.cursor()
								query = ("SELECT count(*) as count from attackInventory where victim_ip = '" + str(ip2long(sourceIP)) + "' and threat_id = '" + vulnerability + "' and attack_time = '" + data["results"][resultCount]["_time"] + "'")
								cursor.execute(query)
								for row in cursor:
									logCheck=row[0]
								cursor.close()
								if logCheck==0:
									#Write data to DB
									cursor = cnx.cursor()
									add_logInstance = ("INSERT INTO attackInventory "
														"(victim_ip, attacker_ip, attack_time, attack_log, threat_id) "
														"VALUES (%s, %s, %s, %s, %s)")
									
									logData = (ip2long(sourceIP), ip2long(clientip), data["results"][resultCount]["_time"], data["results"][resultCount]["Raw Log"], vulnerability)
									# Insert new entry
									cursor.execute(add_logInstance , logData )
									cnx.commit()
									cursor.close()
								cnx.close()
								resultCount=newResultCount+1
							else:
								resultCount=newResultCount
					elif (operator=="OR"):
						if (searchStringCount > 0):
							#only keep searching if there are more IOCS to look at...
							if len(searchStringResults)>2:
								searchString=entry + " AND (host=\"" + sourceHost + "\" OR s_ip=\"" + sourceIP + "\" OR d_host=\"" + sourceHost + "\")  | fields host, clientip | fields - _bkt, _cd, _indextime, _kv, _serial, _si, _sourcetype | rename _raw as \"Raw Log\""
								numResults=splunk.searchVulnerability(searchString,vulnerability,sourceIP,sourceHost)
								if (numResults != "0"):
									data = json.load(numResults)
									resultCount=0
									for result in data["results"]:
										isExploitFound=True
										cnx = getDBConnector()
										cursor = cnx.cursor()
										query = ("SELECT count(*) as count from attackInventory where victim_ip = '" + str(ip2long(sourceIP)) + "' and threat_id = '" + vulnerability + "' and attack_time = '" + data["results"][resultCount]["_time"] + "'")
										cursor.execute(query)
										for row in cursor:
											logCheck=row[0]
										cursor.close()
										if logCheck==0:
											#Write data to DB
											cursor = cnx.cursor()
											add_logInstance = ("INSERT INTO attackInventory "
																"(victim_ip, attacker_ip, attack_time, attack_log, threat_id) "
																"VALUES (%s, %s, %s, %s, %s)")
											logData = (ip2long(sourceIP), ip2long(data["results"][resultCount]["clientip"]), data["results"][resultCount]["_time"], data["results"][resultCount]["Raw Log"], vulnerability)
											
											# Insert new entry
											cursor.execute(add_logInstance , logData )
											
											cnx.commit()
											cursor.close()
										cnx.close()
										resultCount=resultCount+1
							elif len(searchStringResults)==2:
								searchString=entry + " AND (host=\"" + sourceHost + "\" OR host=\"" + sourceIP + "\" OR s_ip=\"" + sourceIP + "\" OR d_host=\"" + sourceHost + "\")  | fields host, clientip | fields - _bkt, _cd, _indextime, _kv, _serial, _si, _sourcetype | rename _raw as \"Raw Log\""
								numResults=splunk.searchVulnerability(searchString,vulnerability,sourceIP,sourceHost)
								if (numResults != "0"):
									data = json.load(numResults)
									resultCount=0
									for result in data["results"]:
										isExploitFound=True
										cnx = getDBConnector()
										cursor = cnx.cursor()
										query = ("SELECT count(*) as count from attackInventory where victim_ip = '" + str(ip2long(sourceIP)) + "' and threat_id = '" + vulnerability + "' and attack_time = '" + data["results"][resultCount]["_time"] + "'")
										cursor.execute(query)
										for row in cursor:
											logCheck=row[0]
										cursor.close()
										if logCheck==0:
											#Write data to DB
											cursor = cnx.cursor()
											add_logInstance = ("INSERT INTO attackInventory "
																"(victim_ip, attacker_ip, attack_time, attack_log, threat_id) "
																"VALUES (%s, %s, %s, %s, %s)")
											
											logData = (ip2long(sourceIP), ip2long(data["results"][resultCount]["clientip"]), data["results"][resultCount]["_time"], data["results"][resultCount]["Raw Log"], vulnerability)
											
											# Insert new entry
											cursor.execute(add_logInstance , logData )
											
											cnx.commit()
											cursor.close()
										cnx.close()
										resultCount=resultCount+1
					searchStringCount=searchStringCount+1
				elif logsource=="elastic_search":
					numResults=0
					startTime="-90d"
					endTime="now"
					#Insert source host from arguments
					entry = re.sub('\<source_host\>', sourceHost, entry)
					#Insert source IP from arguments
					entry = re.sub('\<source_ip\>', sourceIP, entry)
					if (searchStringCount == 1):
						#Insert startTime
						entry = re.sub('\<startTime\>', startTime, entry)
						#Insert endTime
						entry = re.sub('\<endTime\>', endTime, entry)
						if sourceIP == '*':
							entry = re.sub('\<min_count\>', '1', entry)
						else:
							entry = re.sub('\<min_count\>', '2', entry)
						#print entry
						searchResults = ElasticSearchQuery.searchVulnerability(entry,vulnerability,sourceIP,sourceHost)
						#print searchResults
						numResults = getElasticSearchResults(searchResults)
						#print numResults
					if (operator=="AND"):
						if (searchStringCount > 1):
							resultCount=0
							for hit in searchResults['hits']['hits']:
								startTime =  dateutil.parser.parse(hit["_source"]["@timestamp"]) + datetime.timedelta(days =- 1)
								
								endTime =  dateutil.parser.parse(hit["_source"]["@timestamp"]) + datetime.timedelta(days = 1)
								#Insert start time
								entry = re.sub('\<startTime\>', str(startTime.isoformat()), entry)
								#Insert end time
								entry = re.sub('\<endTime\>', str(endTime.isoformat()), entry)
								newSearchResults = ElasticSearchQuery.searchVulnerability(entry,vulnerability,sourceIP,sourceHost)
								newResults = getElasticSearchResults(newSearchResults)
								if (newResults != "0"):
									#This is the result from search 1
									newResultCount=0
									isExploitFound=True
									for newhit in newSearchResults['hits']['hits']:
										try:
											attackerIP=newhit["_source"]["evt_srcip"]
										except:
											attackerIP="0.0.0.0"
										#These are the results from any further results proving the AND condition
										cnx = getDBConnector()
										cursor = cnx.cursor()
										#Check original log hit
										query = ("SELECT count(*) as count from attackInventory where victim_ip = '" + str(ip2long(sourceIP)) + "' and threat_id = '" + vulnerability + "' and attack_log = '" + newhit["_source"]["message"] + "'")
										cursor.execute(query)
										for row in cursor:
											logCheck=row[0]
										cursor.close()
										if logCheck==0:
											#Write data to DB
											cursor = cnx.cursor()
											add_logInstance = ("INSERT INTO attackInventory "
																"(victim_ip, attacker_ip, attack_time, attack_log, threat_id) "
																"VALUES (%s, %s, %s, %s, %s)")
											
											logData = (ip2long(sourceIP), ip2long(attackerIP),hit["_source"]["@timestamp"], hit["_source"]["message"], vulnerability)
											# Insert new entry
											cursor.execute(add_logInstance , logData )
										cursor = cnx.cursor()
										#check new log hit
										query = ("SELECT count(*) as count from attackInventory where victim_ip = '" + str(ip2long(sourceIP)) + "' and threat_id = '" + vulnerability + "' and attack_log = '" + newhit["_source"]["message"] + "'")
										cursor.execute(query)
										for row in cursor:
											logCheck=row[0]
										cursor.close()
										if logCheck==0:
											#Write data to DB
											cursor = cnx.cursor()
											add_logInstance = ("INSERT INTO attackInventory "
																"(victim_ip, attacker_ip, attack_time, attack_log, threat_id) "
																"VALUES (%s, %s, %s, %s, %s)")
											
											logData = (ip2long(sourceIP), ip2long(attackerIP),newhit["_source"]["@timestamp"], newhit["_source"]["message"], vulnerability)
											# Insert new entry
											cursor.execute(add_logInstance , logData )
											
											cnx.commit()
											cursor.close()
										cnx.close()
										newResultCount=newResultCount+1
								else:
									newResultCount=0
								resultCount=newResultCount+1
								
								
								
					elif (operator=="OR"):
						if (searchStringCount == 1):
							if (int(numResults) > 0):
								resultCount = int(numResults)
								writeElasticSearchResults(searchResults,vulnObject,sourceIP,vulnerability)
								isExploitFound=True
						if (searchStringCount > 1):
							#Insert startTime
							entry = re.sub('\<startTime\>', startTime, entry)
							#Insert endTime
							entry = re.sub('\<endTime\>', endTime, entry)
							if sourceIP == '*':
								entry = re.sub('\<min_count\>', '1', entry)
							else:
								entry = re.sub('\<min_count\>', '2', entry)
							#only keep searching if there are more IOCS to look at...
							if len(searchStringResults)>1:
								searchResults = ElasticSearchQuery.searchVulnerability(entry,vulnerability,sourceIP,sourceHost)
								numResults = getElasticSearchResults(searchResults)
								if int(numResults) > 0:
									writeElasticSearchResults(searchResults,vulnObject,sourceIP,vulnerability)
								resultCount = resultCount + int(numResults)
					searchStringCount=searchStringCount+1
			if (isExploitFound==True):
				print("  Found " + str(resultCount) + " instances of exploitation!")
				print("  Generating attack logs") 
				#Parse through data list to get elastic timestamp for audit log times...
			else:
				print("  No instances of exploitation found.\n")
	else:
		resultCount=0
		print("Invalid vulnerability ID")
	return(resultCount)

Example 9

Project: python-typepad-api
Source File: generate.py
View license
def generate_types(types_fn=None, nouns_fn=None):
    if types_fn is None:
        types = json.load(urllib2.urlopen('http://api.typepad.com/object-types.json'))
    else:
        with open(types_fn) as f:
            types = json.load(f)

    if nouns_fn is None:
        nouns = json.load(urllib2.urlopen('http://api.typepad.com/nouns.json'))
    else:
        with open(nouns_fn) as f:
            nouns = json.load(f)

    objtypes = set()
    objtypes_by_name = dict()
    typedata = dict((d['name'], d) for d in types['entries'])
    while typedata:
        for name, info in typedata.items():
            if name == 'Base':
                logging.info('Skipping Base type, since we use TypePadObject for that')
                del typedata[name]
                continue

            # Fix up Relationship to have a parentType.
            if name == 'Relationship':
                info['parentType'] = 'Base'
            # Application doesn't actually inherit anything from Entity, so fix its parentType.
            if name == 'Application':
                info['parentType'] = 'Base'
                info['properties'] = [prop for prop in info['properties']
                    if prop['name'] != 'urlId'
                    and (prop['name'] != 'id' or 'URI' not in prop['docString'])]

            if 'parentType' not in info:
                raise ValueError('Type info for %r has no parentType?' % name)
            if info['parentType'] == 'Base':
                info['parentType'] = u'TypePadObject'
            elif info['parentType'] not in objtypes_by_name:
                logging.debug("Skipping %s until the next round, since I haven't seen %s yet", name, info['parentType'])
                continue

            del typedata[name]
            objtype = ObjectType(info)
            objtypes.add(objtype)
            objtypes_by_name[objtype.name] = objtype

    # Annotate the types with endpoint info.
    for endpoint in nouns['entries']:
        # Fix up blogs.comments to have a resource type.
        if endpoint['name'] == 'blogs':
            for propendp in endpoint['propertyEndpoints']:
                if propendp.get('name') == 'comments':
                    propendp['resourceObjectType'] = {
                        'name': 'List<Comment>',
                    }
                    break
        # Fix up relationships to have a correct object type.
        elif endpoint['name'] == 'relationships':
            endpoint['resourceObjectType']['name'] = 'Relationship'

        try:
            resource_name = endpoint['resourceObjectType']['name']
            logging.debug('Finding object for type %s so it can have endpoint %r', resource_name, objtypes_by_name.get(resource_name))
            objtype = objtypes_by_name[resource_name]
        except KeyError:
            pass
        else:
            objtype.endpoint = endpoint

    return objtypes

Example 10

Project: c3nav-32c3
Source File: graph.py
View license
    def __init__(self, project, room_positions=True, auto_connect=False, load_wifi=False):
        data = json.load(open('projects/'+project+'/config.json'))
        data.update(json.load(open('projects/'+project+'/graph.json')))
        data['pois'] = json.load(open('projects/'+project+'/pois.json'))
        data['titles'] = json.load(open('projects/'+project+'/titles.json'))
        data.update(json.load(open('projects/'+project+'/wifiscans.json')))

        self.data = data
        self.did_room_positions = False
        self.did_room_barriers = False
        self.did_auto_connect = False

        # load basic data
        self.name = data['name']
        self.levels = data['levels']
        self.width = data['width']
        self.height = data['height']
        self.cm_per_px = data['cm_per_px']
        self.titles = data['titles']
        self.overwrite_default_settings = data['overwrite_default_settings']

        self.wifi = WifiLocator(self, load_wifi=load_wifi)

        # load rooms
        self.rooms = {}
        self.superrooms = {}
        self.roomgroups = {}
        for name, rdata in data['rooms'].items():
            shape = [[int(i) for i in p.split(',')] for p in rdata['shape'].split(' ')]
            room = Room(self, name, rdata['level'], data['titles'].get(name, {}), shape)

            superroom_name = rdata.get('superroom')
            if superroom_name:
                superroom = self.superrooms.get(superroom_name)
                if superroom is None:
                    superroom = SuperRoom(self, superroom_name, data['titles'].get(superroom_name, {}))
                    self.superrooms[superroom_name] = superroom
                superroom.rooms.append(room)
                room.superroom = superroom

            for groupname in rdata.get('groups', ()):
                roomgroup = self.roomgroups.get(groupname)
                if roomgroup is None:
                    roomgroup = RoomGroup(self, groupname, data['titles'].get(groupname, {}),
                                          data['titles'].get(':'+groupname, {}))
                    self.roomgroups[groupname] = roomgroup
                roomgroup.rooms.append(room)
                room.groups.append(roomgroup)

            self.rooms[name] = room

        # load nodes
        self.nodes = tuple(Node(i, n, p['level'], p['x'], p['y'])
                           for i, (n, p) in enumerate(data['nodes'].items()))
        self.nodes_by_name = {p.name: p.i for p in self.nodes}

        # load POIs
        self.pois = {}
        self.poigroups = {}
        for name, pdata in data['pois'].items():
            poi = POI(name, pdata['level'], pdata['x'], pdata['y'], data['titles'].get(name, {}))

            for groupname in pdata.get('groups', ()):
                poigroup = self.poigroups.get(groupname)
                if poigroup is None:
                    poigroup = POIGroup(self, groupname, data['titles'].get(groupname, {}),
                                        data['titles'].get(':'+groupname, {}))
                    self.poigroups[groupname] = poigroup
                poigroup.pois.append(poi)
                poi.groups.append(poigroup)

            self.pois[name] = poi

        # create distance matrices, one for every connection type
        self.matrices = {ctype: np.zeros((len(self.nodes), len(self.nodes)))
                         for ctype in self.connection_types}
        for c in data['connections']:
            p0 = self.nodes[self.nodes_by_name[c['node0']]]
            p1 = self.nodes[self.nodes_by_name[c['node1']]]
            directed = c.get('directed', False)
            ctype = c.get('ctype', 'default')
            l0, l1 = ('down', 'up') if p0.level > p1.level else ('up', 'down')
            distance = np.linalg.norm(p0.xy-p1.xy)*self.cm_per_px
            if ctype == 'default':
                self.matrices['default'][p0.i, p1.i] = distance
                if not directed:
                    self.matrices['default'][p1.i, p0.i] = distance
            elif ctype == 'steps':
                self.matrices['steps-'+l0][p0.i, p1.i] = distance
                if not directed:
                    self.matrices['steps-'+l1][p1.i, p0.i] = distance
            elif ctype == 'stairs':
                self.matrices['stairs-'+l0][p0.i, p1.i] = distance * self.diag
                if not directed:
                    self.matrices['stairs-'+l1][p1.i, p0.i] = distance * self.diag
            elif ctype == 'escalator':
                self.matrices['escalator-'+l0][p0.i, p1.i] = distance * self.diag
                if not directed:
                    self.matrices['escalator-'+l1][p1.i, p0.i] = distance * self.diag
            elif ctype == 'elevator':
                self.matrices['elevator-'+l0][p0.i, p1.i] = 1
                if not directed:
                    self.matrices['elevator-'+l1][p1.i, p0.i] = 1

        # load barriers
        self.barriers = []
        for bdata in data['barriers']:
            self.barriers.append(Barrier(bdata['level'], bdata['x1'], bdata['y1'], bdata['x2'], bdata['y2']))

        # selectable locations
        self.selectable_locations = {}
        self.selectable_locations.update(self.rooms)
        self.selectable_locations.update(self.superrooms)
        self.selectable_locations.update(self.roomgroups)
        self.selectable_locations.update(self.pois)
        self.selectable_locations.update(self.poigroups)

        for name, location in self.selectable_locations.items():
            if name in data['overwrite_priority']:
                location.priority = data['overwrite_priority'][name]

        for name, poi in self.pois.items():
            for group in reversed(poi.groups):
                if group.name in data['overwrite_poigroup_priority']:
                    poi.priority = data['overwrite_poigroup_priority'][group.name]

        if room_positions:
            self.room_positions()
        if auto_connect:
            self.auto_connect()

Example 11

Project: ancfinder
Source File: update_anc_database.py
View license
def add_census_data(output):
    # Estimate ANC/SMD Census population data by averaging across the tracts that
    # each ANC and SMD intersects with. Tracts appear to be the lowest level that
    # interesting data is uniformly available from the 2011 American Community Survey.
    # We can get more precise information on population from block groups though.

    # Pre-load the relevant data points from the Census API.

    census_fields = {
            "2010/sf1": [
                    ("P0010001", int, "count"), # Total Population
                    ("P0180001", int, "count"), # Households
                    ("P0180002", int, "count"), # Family Households
                    ("H0040001", int, "count"), # Occupied Housing Units
                    ("H0050001", int, "count"), # Vacant Housing Units
            ],

            # http://www.census.gov/developers/data/acs_5yr_2011_var.xml
            "2011/acs5": [
                    ("B01003_001E", int, "count"), # Total Population
                    ("B01002_001E", float, "median"), # Median Age
                    ("B07001_001E", int, "count"), # Geographic Mobility in the Past Year Total Population
                    ("B07001_017E", int, "count"), # .... Same House One Year Ago
                    ("B07001_033E", int, "count"), # .... Moved Within Same County
                    ("B07001_065E", int, "count"), # .... Moved From Different State
                    ("B07001_081E", int, "count"), # .... Moved From Abroad
                    ("B19019_001E", float, "median"), # Median Household Income
                    ("B16002_001E", int, "count"), # No One Age 14+ in Household Speaks English
            ],
    }

    census_data = { }
    for division in ("tract", "blockgroup"):
        dat = json.load(open("data/smd-%s.json" % division))
        for intsect in dat:
            #if intsect["smd"]["id"] != "1A02": continue

            # get the ID of this tract or blockgroup
            id = intsect[division]["id"]
            if id in census_data: continue
            census_data[id] = { }

            # query Census API
            state_fips, county_fips, tract, bg_num = id[0:2], id[2:5], id[5:11], (id+"?")[11:12]
            for census_table in census_fields:
                field_list = ",".join(f[0] for f in census_fields[census_table])
                if division == "tract":
                    url = "http://api.census.gov/data/%s?key=%s&get=%s&in=state:%s+county:%s&for=tract:%s" \
                        % (census_table, census_api_key, field_list, state_fips, county_fips, tract)
                else:
                    url = "http://api.census.gov/data/%s?key=%s&get=%s&in=state:%s+county:%s+tract:%s&for=blockgroup:%s" \
                        % (census_table, census_api_key, field_list, state_fips, county_fips, tract, bg_num)
                data_from_api = json.load(urlopen(url))
                for i, (fieldname, datatype, sum_mode) in enumerate(census_fields[census_table]):
                    v = data_from_api[1][i]
                    if v is not None: v = datatype(v)
                    census_data[id][fieldname] = v

    # Clear out existing data.
    for ward in output.values():
        for anc in ward["ancs"].values():
            anc["census"] = { "by-blockgroup": { }, "by-tract": { } }
            for smd in anc["smds"].values():
                smd["census"] = { "by-blockgroup": { }, "by-tract": { } }

    # Estimate values for the ANCs and SMDs as a whole.
    for division1 in ("smd", "anc"):
        for division2 in ("tract", "blockgroup"):
            dat = json.load(open("data/%s-%s.json" % (division1, division2)))
            for ix in dat:
                # Here's an intersection between an ANC/SMD and a blockgroup/tract.

                # Get the dict that represents the ANC or SMD's census info.
                ward = ix[division1]["id"][0]
                anc = ix[division1]["id"][1]
                if division1 == "anc":
                    feature = output[ward]["ancs"][anc]
                else:
                    smd = ix[division1]["id"][2:]
                    feature = output[ward]["ancs"][anc]["smds"][smd]
                feature = feature["census"]["by-" + division2]

                # Get the id of the blockgroup or tract.
                census_id = ix[division2]["id"]
                #if census_id not in census_data: continue # only for testing

                # Estimate each field for the ANC/SND as a whole.
                for census_table in census_fields:
                    for fieldname, datatype, summode in census_fields[census_table]:
                        if summode == "count":
                            # This value may not available from the Census (some blockgroup ACS data) so
                            # mark the feature as not-computable.
                            if census_data[census_id][fieldname] is None:
                                feature[fieldname] = "missing-data"
                                continue
                            if feature.get(fieldname) == "missing-data":
                                # Already marked this field is non-computable because data is missing?
                                continue

                            # This field is a count, so we estimate the whole by adding the
                            # field values weighted by the ratio of the intersection area to
                            # the area of the blockgroup or tract.
                            feature.setdefault(fieldname, { "value": 0, "type": summode })
                            feature[fieldname]["value"] += census_data[census_id][fieldname] * ix[division2]["ratio"]
                        elif summode == "median":
                            # For medians, we take the average across all of the intersections
                            # weighted by the population in each intersction.
                            feature.setdefault(fieldname, { "value": 0.0, "type": summode, "weight": 0.0, "missing_weight": 0.0 })
                            w = census_data[census_id]["B01003_001E"] * ix[division2]["ratio"]

                            # This value may not available from the Census (some blockgroup ACS data). It seems
                            # like even tracts are missing some of these values, but we want to estimate as
                            # best as possible so we'll estimate from those tracts where values exist, and we'll
                            # record the proportion of the population in this ANC/SMD that did not contribute
                            # to the estimate.
                            if census_data[census_id][fieldname] is None:
                                feature[fieldname]["missing_weight"] += w
                                continue

                            feature[fieldname]["value"] += census_data[census_id][fieldname] * w
                            feature[fieldname]["weight"] += w

    # Finish up.
    def clean_up(f):
        # Prefer blockgroup statistics if available.
        # Round some values.
        # For medians, divide by the total weight.
        for division in ("tract", "blockgroup"):
            for k, v in list(f["by-" + division].items()):
                if v == "missing-data": continue

                if v["type"] == "count":
                    f[k] = v
                    f[k]["source"] = division
                    v["value"] = int(round(v["value"]))
                elif v["type"] == "median" and v["weight"] > 0:
                    # Skip this if we're missing more data as a blockgroup than as a tract,
                    # or if we're just missing a lot of data (more than 33%).
                    v["missing_pct"] = v["missing_weight"] / (v["weight"] + v["missing_weight"])
                    if v["missing_pct"] > .33 or (k in f and f[k]["missing_pct"] < v["missing_pct"]): continue

                    f[k] = v
                    f[k]["source"] = division
                    v["value"] = round(v["value"] / v["weight"])
                    del v["weight"]
                    del v["missing_weight"]
                del v["type"]
            del f["by-" + division]

    for ward in output.values():
        for anc in ward["ancs"].values():
            clean_up(anc["census"])
            for smd in anc["smds"].values():
                clean_up(smd["census"])

Example 12

Project: sqlipy
Source File: SQLiPy.py
View license
  def startScan(self, button):
    hpp = ''
    cu = ''
    cdb = ''
    hostname = ''
    isdba = ''
    lusers = ''
    lpswds = ''
    lprivs = ''
    lroles = ''
    ldbs = ''
    textonly = ''
    postdata = None
    datacmd = ''
    cookiedata = None
    cookiecmd = ''
    uadata = None
    uacmd = ''
    custheaderdata = None
    custheadercmd = ''
    headerdata = None
    headercmd = ''
    refererdata = None
    referercmd = ''
    proxy = None
    proxycmd = ''
    dbms = None
    dbmscmd = ''
    os = None
    oscmd = ''
    tampercmd = ''
    tamperdata = None
    paramcmd = ''
    paramdata = None
    csrfurl = None
    csrftoken = None
    torcmd = ''
    tortypecmd = ''
    torportcmd = ''
    httpmethod = None
    httpmethodcmd = ''

    if self._jCheckTO.isSelected():
      textonly = ' --text-only'
      textonlystatus = True
    else:
      textonlystatus = False

    if self._jCheckHPP.isSelected():
      hpp = ' --hpp'
      hppstatus = True
    else:
      hppstatus = False

    if self._jCheckCU.isSelected():
      cu = ' --current-user'
      custatus = True
    else:
      custatus = False

    if self._jCheckDB.isSelected():
      cdb = ' --current-db'
      cdbstatus = True
    else:
      cdbstatus = False

    if self._jCheckHost.isSelected():
      hostname = ' --hostname'
      hostnamestatus = True
    else:
      hostnamestatus = False

    if self._jCheckDBA.isSelected():
      isdba = ' --is-dba'
      isdbastatus = True
    else:
      isdbastatus = False

    if self._jCheckUsers.isSelected():
      lusers = ' --users'
      lusersstatus = True
    else:
      lusersstatus = False

    if self._jCheckPswds.isSelected():
      lpswds = ' --passwords'
      lpswdsstatus = True
    else:
      lpswdsstatus = False

    if self._jCheckPrivs.isSelected():
      lprivs = ' --privileges'
      lprivsstatus = True
    else:
      lprivsstatus = False

    if self._jCheckRoles.isSelected():
      lroles = ' --roles'
      lrolesstatus = True
    else:
      lrolesstatus = False

    if self._jCheckDBs.isSelected():
      ldbs = ' --dbs'
      ldbsstatus = True
    else:
      ldbsstatus = False

    if self._jCheckTor.isSelected():
      torstatus = True
      torcmd = ' --tor'
      tortype = self._jComboTorType.getSelectedItem()
      tortypecmd = ' --tor-type=' + self._jComboTorType.getSelectedItem()
      torport = self._jTextFieldTorPort.getText()
      torportcmd = ' --tor-port=' + self._jTextFieldTorPort.getText()
    else:
      torstatus = False
      tortype = 'HTTP'
      torport = None

    if re.search('(http|https)\://', self._jTextFieldProxy.getText()) is not None:
      proxy = self._jTextFieldProxy.getText()
      proxycmd = ' --proxy=' + self._jTextFieldProxy.getText()

    if not re.search('^Default$', self._jComboHttpMethod.getSelectedItem()) is not None:
      httpmethod = self._jComboHttpMethod.getSelectedItem()
      httpmethodcmd = ' --method="' + self._jComboHttpMethod.getSelectedItem()+'"'

    if not re.search('^Any$', self._jComboDBMS.getSelectedItem()) is not None:
      dbms = self._jComboDBMS.getSelectedItem()
      dbmscmd = ' --dbms="' + self._jComboDBMS.getSelectedItem()+'"'

    if not re.search('^Any$', self._jComboOS.getSelectedItem()) is not None:
      os = self._jComboOS.getSelectedItem()
      oscmd = ' --os=' + self._jComboOS.getSelectedItem()

    if re.search('[a-zA-Z0-9]', self._jTextFieldTamper.getText()) is not None:
      tampercmd = ' --tamper="' + self._jTextFieldTamper.getText() + '"'
      tamperdata = self._jTextFieldTamper.getText()

    if re.search('[a-zA-Z0-9]', self._jTextData.getText()) is not None:
      postdata = self._jTextData.getText()
      datacmd = ' --data="' + self._jTextData.getText() + '"'

    if re.search('[a-zA-Z0-9]', self._jTextFieldCookie.getText()) is not None:
      cookiedata = self._jTextFieldCookie.getText()
      cookiecmd = ' --cookie="' + self._jTextFieldCookie.getText() + '"'

    if re.search('[a-zA-Z0-9]', self._jTextFieldUA.getText()) is not None:
      uadata = self._jTextFieldUA.getText()
      uacmd = ' --user-agent="' + self._jTextFieldUA.getText() + '"'

    if re.search('[a-zA-Z0-9]', self._jTextFieldCustHeader.getText()) is not None:
      custheaderdata = self._jTextFieldCustHeader.getText()
      custheadercmd = ' --headers="' + self._jTextFieldCustHeader.getText() + '"'

    if re.search('[a-zA-Z0-9]', self._jTextFieldReferer.getText()) is not None:
      refererdata = self._jTextFieldReferer.getText()
      referercmd = ' --referer="' + self._jTextFieldReferer.getText() + '"'

    if re.search('[a-zA-Z0-9]', self._jTextFieldParam.getText()) is not None:
      paramdata = self._jTextFieldParam.getText()
      paramcmd = ' -p "' + self._jTextFieldParam.getText() + '"'

    try:
      sqlmapcmd = 'sqlmap.py -u "' + self._jTextFieldURL.getText() + '"' + datacmd + httpmethodcmd + cookiecmd + uacmd + referercmd + custheadercmd + proxycmd + torcmd + tortypecmd + torportcmd + ' --delay=' + str(self._jComboDelay.getSelectedItem()) + ' --timeout=' + str(self._jComboTimeout.getSelectedItem()) + ' --retries=' + str(self._jComboDelay.getSelectedItem()) + paramcmd + dbmscmd + oscmd + tampercmd + ' --level=' + str(self._jComboLevel.getSelectedItem()) + ' --risk=' + str(self._jComboRisk.getSelectedItem()) + textonly + hpp + ' --threads=' + str(self._jComboThreads.getSelectedItem()) + ' --time-sec=' + str(self._jComboTimeSec.getSelectedItem()) + ' -b' + cu + cdb + hostname + isdba + lusers + lpswds + lprivs + lroles + ldbs + ' --batch --answers="crack=N,dict=N"\n\n'
      print 'SQLMap Command: ' + sqlmapcmd
      req = urllib2.Request('http://' + self._jTextFieldScanIPListen.getText() + ':' + self._jTextFieldScanPortListen.getText() + '/task/new')
      resp = json.load(urllib2.urlopen(req))

      if resp['success'] == True and resp['taskid']:
        sqlitask = resp['taskid']
        sqliopts = {'csrfUrl': csrfurl, 'csrfToken': csrftoken, 'getUsers': lusersstatus, 'getPasswordHashes': lpswdsstatus, 'delay': self._jComboDelay.getSelectedItem(), 'isDba': isdbastatus, 'risk': self._jComboRisk.getSelectedItem(), 'getCurrentUser': custatus, 'getRoles': lrolesstatus, 'getPrivileges': lprivsstatus, 'testParameter': paramdata, 'timeout': self._jComboTimeout.getSelectedItem(), 'torPort': torport, 'level': self._jComboLevel.getSelectedItem(), 'getCurrentDb': cdbstatus, 'answers': 'crack=N,dict=N', 'method': httpmethod, 'cookie': cookiedata, 'proxy': proxy, 'os': os, 'threads': self._jComboThreads.getSelectedItem(), 'url': self._jTextFieldURL.getText(), 'getDbs': ldbsstatus, 'tor': torstatus, 'torType': tortype, 'referer': refererdata, 'retries': self._jComboRetry.getSelectedItem(), 'headers': custheaderdata, 'timeSec': self._jComboTimeSec.getSelectedItem(), 'getHostname': hostnamestatus, 'agent': uadata, 'dbms': dbms, 'tamper': tamperdata, 'hpp': hppstatus, 'getBanner': 'true', 'data': postdata, 'textOnly': textonlystatus}

        print 'Created SQLMap Task: ' + sqlitask + '\n'

        try:
          req = urllib2.Request('http://' + self._jTextFieldScanIPListen.getText() + ':' + self._jTextFieldScanPortListen.getText() + '/option/' + sqlitask + '/set')
          req.add_header('Content-Type', 'application/json')
          resp = json.load(urllib2.urlopen(req, json.dumps(sqliopts)))

          if resp['success'] == True:
            print 'SQLMap options set on Task ' + sqlitask + ': ' + json.dumps(sqliopts) + '\n'
            sqliopts = {'url': self._jTextFieldURL.getText()}

            try:
              checkreq = urllib2.Request('http://' + self._jTextFieldScanIPListen.getText() + ':' + self._jTextFieldScanPortListen.getText() + '/option/' + sqlitask + '/list')
              checkresp = json.load(urllib2.urlopen(checkreq))
              print 'SQLMap options returned: ' + json.dumps(checkresp) + '\n'
            except:
              print 'Failed to get list of options from SQLMap API\n'

            try:
              req = urllib2.Request('http://' + self._jTextFieldScanIPListen.getText() + ':' + self._jTextFieldScanPortListen.getText() + '/scan/' + sqlitask + '/start')
              req.add_header('Content-Type', 'application/json')
              resp = json.load(urllib2.urlopen(req, json.dumps(sqliopts)))

              if resp['success'] == True:
                findings = ThreadExtender(self, self._jTextFieldScanIPListen.getText(), self._jTextFieldScanPortListen.getText(), sqlitask, self.scanUrl, self.scanMessage, self._callbacks)
                t = threading.Thread(target=findings.checkResults)
                self.threads.append(t)
                t.start()
                self._jComboLogs.addItem(sqlitask + '-' + self._jTextFieldURL.getText())
                self._jComboStopScan.addItem(sqlitask + '-' + self._jTextFieldURL.getText())
                self.scancmds[sqlitask] = sqlmapcmd
                print 'Started SQLMap Scan on Task ' + sqlitask +' with Engine ID: ' + str(resp['engineid']) + ' - ' + self._jTextFieldURL.getText() + '\n'
              else:
                print 'Failed to start SQLMap Scan for Task: ' + sqlitask + '\n'

            except:
              print 'Failed to start SQLMap Scan for Task: ' + sqlitask + '\n'

          else:
            print 'Failed to set options on SQLMap Task: ' + sqlitask + '\n'

        except:
          print 'Failed to set options on SQLMap Task: ' + sqlitask + '\n'

      else:
        print 'SQLMap task creation failed\n'

    except:
      print 'SQLMap task creation failed\n'

Example 13

Project: conan
Source File: conan_controller.py
View license
    def attach_to(self, app):

        conan_route = '%s/:conanname/:version/:username/:channel' % self.route

        # FIXME: REPLACE ROUTE WITH AN ER COMPOSED WITH ERs for
        # {conanname}/{version}/{username}/{channel}

        @app.route("%s/digest" % conan_route, method=["GET"])
        def get_conan_digest_url(conanname, version, username, channel, auth_user):
            """
            Get a dict with all files and the download url
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            urls = conan_service.get_conanfile_download_urls(reference, [CONAN_MANIFEST])
            if not urls:
                raise NotFoundException("No digest found")
            return urls

        @app.route("%s/packages/:package_id/digest" % conan_route, method=["GET"])
        def get_package_digest_url(conanname, version, username, channel, package_id, auth_user):
            """
            Get a dict with all files and the download url
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            package_reference = PackageReference(reference, package_id)

            urls = conan_service.get_package_download_urls(package_reference, [CONAN_MANIFEST])
            if not urls:
                raise NotFoundException("No digest found")
            urls_norm = {filename.replace("\\", "/"): url for filename, url in urls.items()}
            return urls_norm

        @app.route(conan_route, method=["GET"])
        def get_conanfile_snapshot(conanname, version, username, channel, auth_user):
            """
            Get a dictionary with all files and their each md5s
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            snapshot = conan_service.get_conanfile_snapshot(reference)
            snapshot_norm = {filename.replace("\\", "/"): the_md5
                             for filename, the_md5 in snapshot.items()}
            return snapshot_norm

        @app.route('%s/packages/:package_id' % conan_route, method=["GET"])
        def get_package_snapshot(conanname, version, username, channel, package_id, auth_user):
            """
            Get a dictionary with all files and their each md5s
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            package_reference = PackageReference(reference, package_id)
            snapshot = conan_service.get_package_snapshot(package_reference)
            snapshot_norm = {filename.replace("\\", "/"): the_md5
                             for filename, the_md5 in snapshot.items()}
            return snapshot_norm

        @app.route("%s/download_urls" % conan_route, method=["GET"])
        def get_conanfile_download_urls(conanname, version, username, channel, auth_user):
            """
            Get a dict with all files and the download url
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            urls = conan_service.get_conanfile_download_urls(reference)
            urls_norm = {filename.replace("\\", "/"): url for filename, url in urls.items()}
            return urls_norm

        @app.route('%s/packages/:package_id/download_urls' % conan_route, method=["GET"])
        def get_package_download_urls(conanname, version, username, channel, package_id,
                                      auth_user):
            """
            Get a dict with all packages files and the download url for each one
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            package_reference = PackageReference(reference, package_id)
            urls = conan_service.get_package_download_urls(package_reference)
            urls_norm = {filename.replace("\\", "/"): url for filename, url in urls.items()}
            return urls_norm

        @app.route("%s/upload_urls" % conan_route, method=["POST"])
        def get_conanfile_upload_urls(conanname, version, username, channel, auth_user):
            """
            Get a dict with all files and the upload url
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            reader = codecs.getreader("utf-8")
            filesizes = json.load(reader(request.body))
            urls = conan_service.get_conanfile_upload_urls(reference, filesizes)
            urls_norm = {filename.replace("\\", "/"): url for filename, url in urls.items()}
            return urls_norm

        @app.route('%s/packages/:package_id/upload_urls' % conan_route, method=["POST"])
        def get_package_upload_urls(conanname, version, username, channel, package_id, auth_user):
            """
            Get a dict with all files and the upload url
            """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            package_reference = PackageReference(reference, package_id)
            reader = codecs.getreader("utf-8")
            filesizes = json.load(reader(request.body))
            urls = conan_service.get_package_upload_urls(package_reference, filesizes)
            urls_norm = {filename.replace("\\", "/"): url for filename, url in urls.items()}
            return urls_norm

        @app.route('%s/search' % self.route, method=["GET"])
        def search(auth_user):
            pattern = request.params.get("q", None)
            ignorecase = request.params.get("ignorecase", True)
            if isinstance(ignorecase, str):
                ignorecase = False if 'false' == ignorecase.lower() else True
            search_service = SearchService(app.authorizer, app.search_manager, auth_user)
            references = [str(ref) for ref in search_service.search(pattern, ignorecase)]
            return {"results": references}

        @app.route('%s/search' % conan_route, method=["GET"])
        def search_packages(conanname, version, username, channel, auth_user):
            query = request.params.get("q", None)
            search_service = SearchService(app.authorizer, app.search_manager, auth_user)
            conan_reference = ConanFileReference(conanname, version, username, channel)
            info = search_service.search_packages(conan_reference, query)
            return info

        @app.route(conan_route, method="DELETE")
        def remove_conanfile(conanname, version, username, channel, auth_user):
            """ Remove any existing conanfiles or its packages created """
            conan_reference = ConanFileReference(conanname, version, username, channel)
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            conan_service.remove_conanfile(conan_reference)

        @app.route('%s/packages/delete' % conan_route, method="POST")
        def remove_packages(conanname, version, username, channel, auth_user):
            """ Remove any existing conanfiles or its packages created """
            conan_reference = ConanFileReference(conanname, version, username, channel)
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reader = codecs.getreader("utf-8")
            payload = json.load(reader(request.body))
            conan_service.remove_packages(conan_reference, payload["package_ids"])

        @app.route('%s/remove_files' % conan_route, method="POST")
        def remove_conanfile_files(conanname, version, username, channel, auth_user):
            """ Remove any existing conanfiles or its packages created """
            conan_reference = ConanFileReference(conanname, version, username, channel)
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reader = codecs.getreader("utf-8")
            payload = json.load(reader(request.body))
            files = [os.path.normpath(filename) for filename in payload["files"]]
            conan_service.remove_conanfile_files(conan_reference, files)

        @app.route('%s/packages/:package_id/remove_files' % conan_route, method=["POST"])
        def remove_packages_files(conanname, version, username, channel, package_id, auth_user):
            """ Remove any existing conanfiles or its packages created """
            conan_service = ConanService(app.authorizer, app.file_manager, auth_user)
            reference = ConanFileReference(conanname, version, username, channel)
            package_reference = PackageReference(reference, package_id)
            reader = codecs.getreader("utf-8")
            payload = json.load(reader(request.body))
            files = [os.path.normpath(filename) for filename in payload["files"]]
            conan_service.remove_package_files(package_reference, files)

Example 14

Project: couchbase-cli
Source File: pump_bfd.py
View license
    @staticmethod
    def find_seqno(opts, spec, bucket_name, node_name, mode):
        seqno = {}
        dep = {}
        dep_list = []
        failover_log = {}
        snapshot_markers = {}
        for i in range(BFD.NUM_VBUCKET):
            seqno[str(i)] = 0
            dep[i] = None
            failover_log[i] = None
            snapshot_markers[i] = None

        file_list = []
        failoverlog_list = []
        snapshot_list = []
        seqno_list = []
        parent_dir = os.path.normpath(spec)

        if mode == "full":
            return seqno, dep_list, failover_log, snapshot_markers
        timedir,latest_dirs = BFD.find_latest_dir(parent_dir, None)
        if not timedir:
            return seqno, dep_list, failover_log, snapshot_markers
        fulldir, latest_dirs = BFD.find_latest_dir(timedir, "full")
        if not fulldir:
            return seqno, dep_list, failover_log, snapshot_markers

        path = BFD.construct_dir(fulldir, bucket_name, node_name)
        if not os.path.isdir(path):
            return seqno, dep_list, failover_log, snapshot_markers

        file_list.extend(recursive_glob(path, 'data-*.cbb'))
        failoverlog_list.extend(recursive_glob(path, 'failover.json'))
        snapshot_list.extend(recursive_glob(path, 'snapshot_markers.json'))
        seqno_list.extend(recursive_glob(path, 'seqno.json'))

        accudir, accu_dirs = BFD.find_latest_dir(timedir, "accu")
        if accudir:
            path = BFD.construct_dir(accudir, bucket_name, node_name)
            if os.path.isdir(path):
                file_list.extend(recursive_glob(path, 'data-*.cbb'))
                failoverlog_list.extend(recursive_glob(path, 'failover.json'))
                snapshot_list.extend(recursive_glob(path, 'snapshot_markers.json'))
                seqno_list.extend(recursive_glob(path, 'seqno.json'))
        if mode.find("diff") >= 0:
            diffdir, diff_dirs = BFD.find_latest_dir(timedir, "diff")
            if diff_dirs:
                for dir in diff_dirs:
                    path = BFD.construct_dir(dir, bucket_name, node_name)
                    if os.path.isdir(path):
                        file_list.extend(recursive_glob(path, 'data-*.cbb'))
                        failoverlog_list.extend(recursive_glob(path, 'failover.json'))
                        snapshot_list.extend(recursive_glob(path, 'snapshot_markers.json'))
                        seqno_list.extend(recursive_glob(path, 'seqno.json'))

        for x in sorted(seqno_list):
            try:
                json_file = open(x, "r")
                json_data = json.load(json_file)
                json_file.close()

                for vbid, seq in json_data.iteritems():
                    if not seq:
                        continue
                    if seqno.get(vbid) < seq:
                        seqno[vbid] = seq
            except IOError:
                pass

        for log_file in sorted(failoverlog_list):
            try:
                json_file = open(log_file, "r")
                json_data = json.load(json_file)
                json_file.close()

                for vbid, flogs in json_data.iteritems():
                    if not flogs:
                        continue
                    elif vbid not in failover_log.keys():
                        failover_log[vbid] = flogs
                    else:
                        for logpair in flogs:
                            if not failover_log[vbid]:
                                failover_log[vbid] = [logpair]
                            elif logpair not in failover_log[vbid]:
                                failover_log[vbid].append(logpair)
            except IOError:
                pass

        for snapshot in sorted(snapshot_list):
            try:
                json_file = open(snapshot, "r")
                json_data = json.load(json_file)
                json_file.close()

                for vbid, markers in json_data.iteritems():
                    snapshot_markers[vbid] = markers
            except IOError:
                pass

        for i in range(BFD.NUM_VBUCKET):
            if dep[i] and dep[i] not in dep_list:
                dep_list.append(dep[i])
        return seqno, dep_list, failover_log, snapshot_markers

Example 15

Project: dogapi
Source File: test_dogshell.py
View license
    def test_dashes(self):
        # Create a dash and write it to a file
        name, temp0 = get_temp_file()
        self.dogshell(["dashboard", "new_file", name])
        dash = json.load(temp0)

        assert 'id' in dash, dash
        assert 'title' in dash, dash

        # Update the file and push it to the server
        unique = self.get_unique()
        dash['title'] = 'dash title %s' % unique
        name, temp1 = get_temp_file()
        json.dump(dash, temp1)
        temp1.flush()
        self.dogshell(["dashboard", "push", temp1.name])

        # Query the server to verify the change
        out, _, _ = self.dogshell(["dashboard", "show", str(dash['id'])])

        out = json.loads(out)
        assert "dash" in out, out
        assert "id" in out["dash"], out
        self.assertEquals(out["dash"]["id"], dash["id"])
        assert "title" in out["dash"]
        self.assertEquals(out["dash"]["title"], dash["title"])

        new_title = "new_title"
        new_desc = "new_desc"
        new_dash = [{
                    "title": "blerg",
                    "definition": {
                        "requests": [
                            {"q": "avg:system.load.15{web,env:prod}"}
                        ]
                    }
                }]

        # Update a dash directly on the server
        self.dogshell(["dashboard", "update", str(dash["id"]), new_title, new_desc], stdin=json.dumps(new_dash))

        # Query the server to verify the change
        out, _, _ = self.dogshell(["dashboard", "show", str(dash["id"])])
        out = json.loads(out)
        assert "dash" in out, out
        assert "id" in out["dash"], out
        self.assertEquals(out["dash"]["id"], dash["id"])
        assert "title" in out["dash"], out
        self.assertEquals(out["dash"]["title"], new_title)
        assert "description" in out["dash"], out
        self.assertEquals(out["dash"]["description"], new_desc)
        assert "graphs" in out["dash"], out
        self.assertEquals(out["dash"]["graphs"], new_dash)

        # Pull the updated dash to disk
        fd, updated_file = tempfile.mkstemp()
        try:
            self.dogshell(["dashboard", "pull", str(dash["id"]), updated_file])
            updated_dash = {}
            with open(updated_file) as f:
                updated_dash = json.load(f)
            assert "dash" in out
            self.assertEquals(out["dash"], updated_dash)
        finally:
            os.unlink(updated_file)
        
        # Delete the dash 
        self.dogshell(["dashboard", "delete", str(dash["id"])])

        # Verify that it's not on the server anymore
        out, err, return_code = self.dogshell(["dashboard", "show", str(dash['id'])], check_return_code=False)
        self.assertNotEquals(return_code, 0)

Example 16

Project: osrc
Source File: stats.py
View license
def user_stats(username, tz_offset=True):
    user = github.get_user(username)
    if user is None:
        return None
    if not user.active:
        return False

    #
    # REPOS:
    #
    conn = get_connection()
    repos = conn.zrevrange(format_key("u:{0}:r".format(user.id)), 0, 4,
                           withscores=True)
    repo_counts = []
    languages = defaultdict(int)
    for repo_id, count in repos:
        r = github.get_repo(id=int(repo_id))
        repo_counts.append((r, int(count)))
        if r.language is None:
            continue
        languages[r.language] += int(count)

    #
    # FRIENDS:
    #
    with load_resource("graph_user_user.lua") as script:
        get_users = get_connection().register_script(script.read())
    social = get_users(keys=["{0}".format(user.id).encode("ascii")],
                       args=[flask.current_app.config["REDIS_PREFIX"]])
    friends = []
    if len(social):
        ids = map(int, social[::2])
        friends = list(zip(User.query.filter(User.id.in_(ids)).all(),
                           map(int, social[1::2])))

    #
    # SIMILAR REPOS:
    #
    with load_resource("graph_user_repo.lua") as script:
        get_repos = get_connection().register_script(script.read())
    repo_scores = get_repos(keys=["{0}".format(user.id).encode("ascii")],
                            args=[flask.current_app.config["REDIS_PREFIX"]])
    repo_recs = []
    if len(repo_scores):
        ids = map(int, repo_scores[::2])
        repo_recs = list(zip(Repo.query.filter(Repo.id.in_(ids)).all(),
                             map(int, repo_scores[1::2])))

    #
    # SCHEDULE:
    #
    with get_pipeline() as pipe:
        pipe.zrevrange(format_key("u:{0}:e".format(user.id)), 0, 4,
                       withscores=True)
        keys = [(k.decode("ascii"), c) for k, c in pipe.execute()[0]]
        for k, _ in keys:
            key = format_key("u:{0}:e:{1}".format(user.id, k))
            pipe.hgetall(key)
        schedule = dict(zip((k for k, _ in keys), pipe.execute()))
    total_hist = dict((k, c) for k, c in keys)
    week_hist = defaultdict(lambda: list([0 for _ in range(7)]))
    day_hist = defaultdict(lambda: list([0 for _ in range(24)]))
    for t, vals in schedule.items():
        for day, counts in vals.items():
            counts = list(map(int, counts.decode("ascii").split(",")))
            sm = sum(counts)
            total_hist[t] += sm
            week_hist[t][int(day)] += sm
            for i, c in enumerate(counts):
                day_hist[t][i] += c

    # Correct for the timezone.
    if tz_offset and user.timezone:
        for t, v in day_hist.items():
            day_hist[t] = roll(v, -user.timezone)

    #
    # PROSE DESCRIPTIONS:
    #
    h = [0 for _ in range(7)]
    for v in week_hist.values():
        for i, c in enumerate(v):
            h[i] += c
    descriptions = None
    norm = sqrt(sum([v * v for v in h]))
    if norm > 0.0:
        # A description of the most active day.
        with load_resource("days.json") as f:
            days = json.load(f)
        h = [_ / norm for _ in h]
        best = -1.0
        for d in days:
            vector = d["vector"]
            norm = 1.0 / sqrt(sum([v * v for v in vector]))
            dot = sum([(v*norm-w) ** 2 for v, w in zip(vector, h)])
            if best < 0 or dot < best:
                best = dot
                day_desc = d["name"]

        # A description of the most active time.
        with load_resource("times.json") as f:
            time_desc = json.load(f)
        h = [0 for _ in range(24)]
        for v in day_hist.values():
            for i, c in enumerate(v):
                h[i] += c
        time_desc = time_desc[sorted(zip(h, range(24)))[-1][1]]

        # Choose an adjective deterministically.
        with load_resource("adjectives.json") as f:
            adjs = json.load(f)
        lang = "coder"
        if len(languages):
            with load_resource("languages.json") as f:
                langs = json.load(f)
            l = sorted(languages.items(), key=operator.itemgetter(1))[-1]
            lang = langs.get(l[0], l[0] + " coder")

        # Describe the most common event type.
        with load_resource("event_actions.json") as f:
            acts = json.load(f)
        a = sorted(total_hist.items(), key=operator.itemgetter(1))[-1]
        action = acts.get(a[0], "pushing code")

        # Combine the descriptions.
        descriptions = dict(
            work_habits="{0} who works best {1}".format(day_desc, time_desc),
            language="{0} {1} who excels at {2}".format(
                adjs[abs(hash(user.login)) % len(adjs)], lang, action,
            ),
        )

    # Build the results dictionary.
    return dict(
        user.basic_dict(),
        descriptions=descriptions,
        total=int(sum(total_hist.values())),
        events=[{"type": t, "count": int(c)} for t, c in sorted(
            total_hist.items(), reverse=True, key=operator.itemgetter(1))],
        week=dict(week_hist),
        day=dict(day_hist),
        languages=[{"language": l, "count": int(c)}
                   for l, c in sorted(languages.items(), reverse=True,
                                      key=operator.itemgetter(1))],
        repos=[dict(r.short_dict(), count=c) for r, c in repo_counts],
        friends=[dict(u.short_dict(), weight=c) for u, c in friends],
        repo_recs=[dict(r.short_dict(), weight=c) for r, c in repo_recs],
    )

Example 17

Project: dx-toolkit
Source File: local_exec_utils.py
View license
def run_one_entry_point(job_id, function, input_hash, run_spec, depends_on, name=None):
    '''
    :param job_id: job ID of the local job to run
    :type job_id: string
    :param function: function to run
    :type function: string
    :param input_hash: input for the job (may include job-based object references)
    :type input_hash: dict
    :param run_spec: run specification from the dxapp.json of the app
    :type run_spec: dict

    Runs the specified entry point and retrieves the job's output,
    updating job_outputs.json (in $DX_TEST_JOB_HOMEDIRS) appropriately.
    '''
    print('======')

    job_homedir = os.path.join(environ['DX_TEST_JOB_HOMEDIRS'], job_id)

    job_env = environ.copy()
    job_env['HOME'] = os.path.join(environ['DX_TEST_JOB_HOMEDIRS'], job_id)

    all_job_outputs_path = os.path.join(environ['DX_TEST_JOB_HOMEDIRS'], 'job_outputs.json')

    with open(all_job_outputs_path, 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)

    if isinstance(name, basestring):
        name += ' (' + job_id + ':' + function + ')'
    else:
        name = job_id + ':' + function
    job_name = BLUE() + BOLD() + name + ENDC()
    print(job_name)

    # Resolve local job-based object references
    try:
        resolve_job_references(input_hash, all_job_outputs)
    except Exception as e:
        exit_with_error(job_name + ' ' + JOB_STATES('failed') + ' when resolving input:\n' + fill(str(e)))

    # Get list of non-closed data objects in the input that appear as
    # DNAnexus links; append to depends_on
    if depends_on is None:
        depends_on = []
    get_implicit_depends_on(input_hash, depends_on)

    try:
        wait_for_depends_on(depends_on, all_job_outputs)
    except Exception as e:
        exit_with_error(job_name + ' ' + JOB_STATES('failed') + ' when processing depends_on:\n' + fill(str(e)))

    # Save job input to job_input.json
    with open(os.path.join(job_homedir, 'job_input.json'), 'wb') as fd:
        json.dump(input_hash, fd, indent=4)
        fd.write(b'\n')

    print(job_output_to_str(input_hash, title=(BOLD() + 'Input: ' + ENDC()),
                            title_len=len("Input: ")).lstrip())

    if run_spec['interpreter'] == 'bash':
        # Save job input to env vars
        env_path = os.path.join(job_homedir, 'environment')
        with open(env_path, 'w') as fd:
            job_input_file = os.path.join(job_homedir, 'job_input.json')
            var_defs_hash = file_load_utils.gen_bash_vars(job_input_file, job_homedir=job_homedir)
            for key, val in var_defs_hash.iteritems():
                fd.write("{}={}\n".format(key, val))

    print(BOLD() + 'Logs:' + ENDC())
    start_time = datetime.datetime.now()
    if run_spec['interpreter'] == 'bash':
        script = '''
          cd programtalk_com;
          . {env_path};
          . {code_path};
          if [[ $(type -t {function}) == "function" ]];
          then {function};
          else echo "$0: Global scope execution complete. Not invoking entry point function {function} because it was not found" 1>&2;
          fi'''.format(homedir=pipes.quote(job_homedir),
                       env_path=pipes.quote(os.path.join(job_env['HOME'], 'environment')),
                       code_path=pipes.quote(environ['DX_TEST_CODE_PATH']),
                       function=function)
        invocation_args = ['bash', '-c', '-e'] + (['-x'] if environ.get('DX_TEST_X_FLAG') else []) + [script]
    elif run_spec['interpreter'] == 'python2.7':
        script = '''#!/usr/bin/env python
import os
os.chdir(programtalk_com)

{code}

import dxpy, json
if dxpy.utils.exec_utils.RUN_COUNT == 0:
    dxpy.run()
'''.format(homedir=repr(job_homedir),
           code=run_spec['code'])

        job_env['DX_TEST_FUNCTION'] = function
        invocation_args = ['python', '-c', script]

    if USING_PYTHON2:
        invocation_args = [arg.encode(sys.stdout.encoding) for arg in invocation_args]
        env = {k: v.encode(sys.stdout.encoding) for k, v in job_env.items()}
    else:
        env = job_env

    fn_process = subprocess.Popen(invocation_args, env=env)

    fn_process.communicate()
    end_time = datetime.datetime.now()

    if fn_process.returncode != 0:
        exit_with_error(job_name + ' ' + JOB_STATES('failed') + ', exited with error code ' + str(fn_process.returncode) + ' after ' + str(end_time - start_time))

    # Now updating job output aggregation file with job's output
    job_output_path = os.path.join(job_env['HOME'], 'job_output.json')
    if os.path.exists(job_output_path):
        try:
            with open(job_output_path, 'r') as fd:
                job_output = json.load(fd, object_pairs_hook=collections.OrderedDict)
        except Exception as e:
            exit_with_error('Error: Could not load output of ' + job_name + ':\n' + fill(str(e.__class__) + ': ' + str(e)))
    else:
        job_output = {}

    print(job_name + ' -> ' + GREEN() + 'finished running' + ENDC() + ' after ' + str(end_time - start_time))
    print(job_output_to_str(job_output, title=(BOLD() + "Output: " + ENDC()),
                            title_len=len("Output: ")).lstrip())

    with open(os.path.join(environ['DX_TEST_JOB_HOMEDIRS'], 'job_outputs.json'), 'r') as fd:
        all_job_outputs = json.load(fd, object_pairs_hook=collections.OrderedDict)
    all_job_outputs[job_id] = job_output

    # Before dumping, see if any new jbors should be resolved now
    for other_job_id in all_job_outputs:
        if all_job_outputs[other_job_id] is None:
            # Skip if job is not done yet (true for ancestor jobs)
            continue
        resolve_job_references(all_job_outputs[other_job_id], all_job_outputs, should_resolve=False)

    with open(os.path.join(environ['DX_TEST_JOB_HOMEDIRS'], 'job_outputs.json'), 'wb') as fd:
        json.dump(all_job_outputs, fd, indent=4)
        fd.write(b'\n')

Example 18

Project: pywps
Source File: complexvalidator.py
View license
def validategeojson(data_input, mode):
    """GeoJSON validation example

    >>> import StringIO
    >>> class FakeInput(object):
    ...     json = open('point.geojson','w')
    ...     json.write('''{"type":"Feature", "properties":{}, "geometry":{"type":"Point", "coordinates":[8.5781228542328, 22.87500500679]}, "crs":{"type":"name", "properties":{"name":"urn:ogc:def:crs:OGC:1.3:CRS84"}}}''')  # noqa
    ...     json.close()
    ...     file = 'point.geojson'
    >>> class fake_data_format(object):
    ...     mimetype = 'application/geojson'
    >>> fake_input = FakeInput()
    >>> fake_input.data_format = fake_data_format()
    >>> validategeojson(fake_input, MODE.SIMPLE)
    True
    """

    LOGGER.info('validating GeoJSON; Mode: %s', mode)
    passed = False

    if mode >= MODE.NONE:
        passed = True

    if mode >= MODE.SIMPLE:

        name = data_input.file
        (mtype, encoding) = mimetypes.guess_type(name, strict=False)
        passed = data_input.data_format.mime_type in {mtype, FORMATS.GEOJSON.mime_type}

    if mode >= MODE.STRICT:

        from pywps.dependencies import ogr
        data_source = ogr.Open(data_input.file)
        if data_source:
            passed = (data_source.GetDriver().GetName() == "GeoJSON")
        else:
            passed = False

    if mode >= MODE.VERYSTRICT:

        import jsonschema
        import json

        # this code comes from
        # https://github.com/om-henners/GeoJSON_Validation/blob/master/geojsonvalidation/geojson_validation.py
        schema_home = os.path.join(_get_schemas_home(), "geojson")
        base_schema = os.path.join(schema_home, "geojson.json")

        with open(base_schema) as fh:
            geojson_base = json.load(fh)

        with open(os.path.join(schema_home, "crs.json")) as fh:
            crs_json = json.load(fh)

        with open(os.path.join(schema_home, "bbox.json")) as fh:
            bbox_json = json.load(fh)

        with open(os.path.join(schema_home, "geometry.json")) as fh:
            geometry_json = json.load(fh)

        cached_json = {
            "http://json-schema.org/geojson/crs.json": crs_json,
            "http://json-schema.org/geojson/bbox.json": bbox_json,
            "http://json-schema.org/geojson/geometry.json": geometry_json
        }

        resolver = jsonschema.RefResolver(
            "http://json-schema.org/geojson/geojson.json",
            geojson_base, store=cached_json)

        validator = jsonschema.Draft4Validator(geojson_base, resolver=resolver)
        try:
            validator.validate(json.loads(data_input.stream.read()))
            passed = True
        except jsonschema.ValidationError:
            passed = False

    return passed

Example 19

Project: grr
Source File: bigquery_plugin_test.py
View license
  def testBigQueryPluginFallbackToAFF4(self):
    plugin_args = bigquery_plugin.BigQueryOutputPluginArgs()
    responses = [
        rdf_client.StatEntry(
            aff4path=self.client_id.Add("/fs/os/中国新闻网新闻中"),
            pathspec=rdf_paths.PathSpec(path="/中国新闻网新闻中")),
        rdf_client.Process(pid=42), rdf_client.Process(pid=43),
        rdf_client.SoftwarePackage(name="test.deb")
    ]

    plugin = bigquery_plugin.BigQueryOutputPlugin(
        source_urn=self.results_urn,
        output_base_urn=self.base_urn,
        args=plugin_args,
        token=self.token)

    plugin.InitializeState()

    messages = []
    for response in responses:
      messages.append(
          rdf_flows.GrrMessage(
              source=self.client_id, payload=response))

    with test_lib.FakeTime(1445995873):
      with mock.patch.object(bigquery, "GetBigQueryClient") as mock_bigquery:
        mock_bigquery.return_value.configure_mock(
            **{"InsertData.side_effect": bigquery.BigQueryJobUploadError()})
        with test_lib.ConfigOverrider({"BigQuery.max_upload_failures": 2}):
          for message in messages:
            plugin.ProcessResponses([message])
          plugin.Flush()

          # We have 3 output types but a limit of 2 upload failures, so we
          # shouldn't try the third one.
          self.assertEqual(mock_bigquery.return_value.InsertData.call_count, 2)

    # We should have written a data file and a schema file for each type.
    for output_name in [
        "ExportedFile", "ExportedProcess", "AutoExportedSoftwarePackage"
    ]:
      schema_fd = aff4.FACTORY.Open(
          self.base_urn.Add("C-1000000000000000_Results_%s_1445995873.schema" %
                            output_name),
          token=self.token)
      data_fd = aff4.FACTORY.Open(
          self.base_urn.Add("C-1000000000000000_Results_%s_1445995873.data" %
                            output_name),
          token=self.token)
      actual_fd = gzip.GzipFile(None, "r", 9, data_fd)

      if output_name == "ExportedFile":
        self.CompareSchemaToKnownGood(json.load(schema_fd))
        self.assertEqual(
            json.load(actual_fd)["urn"], self.client_id.Add("/fs/os/中国新闻网新闻中"))
      elif output_name == "ExportedProcess":
        self.assertEqual(json.load(schema_fd)[1]["name"], "pid")
        expected_pids = ["42", "43"]
        for i, line in enumerate(actual_fd):
          self.assertEqual(json.loads(line)["pid"], expected_pids[i])
      else:
        self.assertEqual(json.load(schema_fd)[1]["name"], "name")
        self.assertEqual(json.load(actual_fd)["name"], "test.deb")

    # Process the same messages to make sure we're re-using the filehandles.
    with test_lib.FakeTime(1445995878):
      with mock.patch.object(bigquery, "GetBigQueryClient") as mock_bigquery:
        mock_bigquery.return_value.configure_mock(
            **{"InsertData.side_effect": bigquery.BigQueryJobUploadError()})
        with test_lib.ConfigOverrider({"BigQuery.max_upload_failures": 2}):
          for message in messages:
            plugin.ProcessResponses([message])
          plugin.Flush()

          # We shouldn't call insertdata at all because we have passed max
          # failures already
          self.assertEqual(mock_bigquery.return_value.InsertData.call_count, 0)

    expected_line_counts = {
        "ExportedFile": 2,
        "ExportedProcess": 4,
        "AutoExportedSoftwarePackage": 2
    }
    for output_name in [
        "ExportedFile", "ExportedProcess", "AutoExportedSoftwarePackage"
    ]:
      data_fd = aff4.FACTORY.Open(
          self.base_urn.Add("C-1000000000000000_Results_%s_1445995873.data" %
                            output_name),
          token=self.token)
      actual_fd = gzip.GzipFile(None, "r", 9, data_fd)
      self.assertEqual(
          sum(1 for line in actual_fd), expected_line_counts[output_name])

Example 20

Project: Nova
Source File: cve_scan_v2.py
View license
def audit(data_list, tags, verbose=False, show_profile=False, debug=False):
    '''
    Main audit function. See module docstring for more information on usage.
    '''
    os_version = __grains__.get('osmajorrelease', None)
    if os_version is None:
        os_version = __grains__.get('osrelease', None)
    os_name = __grains__['os'].lower()

    log.debug("os_version: %s, os_name: %s", os_version, os_name)

    endpoints = []

    # Go through yaml to check for cve_scan_v2,
    #    if its present, check for a cached version
    #    of the scan.
    for profile, data in data_list:

        if 'cve_scan_v2' in data:

            ttl = data['cve_scan_v2']['ttl']
            url = data['cve_scan_v2']['url']
            control = data['cve_scan_v2'].get('control', {})
            # Ability to add more controls easily, in control dict
            min_score = float(control.get('score', 0))
            urlhash = hashlib.md5(url).hexdigest()
            cached_json = os.path.join(__opts__['cachedir'],
                                       'cve_scan_cache',
                                       '%s.json' % urlhash)
            cached_zip = os.path.join(__opts__['cachedir'],
                                      'cve_scan_cache',
                                      '%s.zip' % urlhash)
            # Make cache directory and all parent directories if it doesn't exist.
            if not os.path.exists(os.path.dirname(cached_json)):
                os.makedirs(os.path.dirname(cached_json))
            cache = _get_cache(ttl, cached_json)
            log.debug("valid cache: %s, for url: %s", cache != [], url)
            endpoints.append((url, cache, cached_json, cached_zip, min_score, profile))

    # If we don't find our module in the yaml
    if not endpoints:
        return {}

    ret = {'Success': [], 'Failure': [], 'Controlled': []}
    # Dictionary of {pkg_name: list(pkg_versions)}
    local_pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True)

    for url, cache, cached_json, cached_zip, min_score, profile in endpoints:
        log.debug("url: %s, min_score: %s", url, min_score)
        if cache: # Valid cached file
            master_json = cache
        else: # Query the url for cve's
            if url.startswith('http://') or url.startswith('https://'):
                if 'vulners.com' in url:
                    # Vulners api can only handles http:// requests from request.get
                    if url.startswith('https'):
                        url.replace('https', 'http', 1)
                    # Format the url for the request based on operating system.
                    if url.endswith('/'):
                        url = url[:-1]
                    url_final = '%s/api/v3/archive/distributive/?os=%s&version=%s' \
                                                                % (url, os_name, os_version)
                    log.debug('requesting: %s', url_final)
                    cve_query = requests.get(url_final)
                    # Confirm that the request was valid.
                    if cve_query.status_code != 200:
                        raise Exception('Vulners requests was not successful. Check the url.')
                    # Save vulners zip attachment in cache location and extract json
                    try:
                        with open(cached_zip, 'w') as zip_attachment:
                            zip_attachment.write(cve_query.content)
                        zip_file = ZipFile(cached_zip)
                        zip_file.extractall(os.path.dirname(cached_zip))
                        os.remove(cached_zip)
                        extracted_json = os.path.join(__opts__['cachedir'],
                                                      'cve_scan_cache',
                                                      '%s_%s.json' % (os_name, os_version.replace('.', '')))
                        log.debug('attempting to open %s', extracted_json)
                        with open(extracted_json, 'r') as json_file:
                            master_json = json.load(json_file)
                        os.remove(extracted_json)
                    except IOError as ioe:
                        log.error('The json zip attachment was not able to be extracted from vulners.')
                        raise ioe
                else: # Not a vulners request, external source for cve's
                    log.debug('requesting: %s', url)
                    cve_query = requests.get(url)
                    if cve_query.status_code != 200:
                        log.error('URL request was not successful.')
                        raise Exception('The url given is invalid.')
                    master_json = json.loads(cve_query.text)
                #Cache results.
                try:
                    with open(cached_json, 'w') as cache_file:
                        json.dump(master_json, cache_file)
                except IOError:
                    log.error('The cve results weren\'t able to be cached')
            elif url.startswith('salt://'):
                # Cache the file
                log.debug('getting file from %s', url)
                cache_file = __salt__['cp.get_file'](url, cached_json)
                if cache_file:
                    master_json = json.load(open(cache_file))
                else:
                    raise IOError('The file was not able to be retrieved from the salt file server.')
            else:
                raise Exception('The url is invalid. It does not begin with http(s):// or salt://')

        affected_pkgs = _get_cve_vulnerabilities(master_json, os_version)

        # Check all local packages against cve vulnerablities in affected_pkgs
        for local_pkg in local_pkgs:
            vulnerable = None
            if local_pkg in affected_pkgs:
                # There can be multiple versions for a single local package, check all
                for local_version in local_pkgs[local_pkg]:
                    # There can be multiple cve announcements for a single package, check against all
                    for affected_obj in affected_pkgs[local_pkg]:
                        affected_version = affected_obj.pkg_version
                        if _is_vulnerable(local_version, affected_version, affected_obj.operator):
                            # If the local pkg hasn't been found as vulnerable yet, vulnerable is None
                            if not vulnerable:
                                affected_obj.oudated_version = local_version
                                vulnerable = affected_obj
                            # If local_pkg has already been marked affected, vulnerable is set. We
                            #   want to report the cve with highest severity
                            else:
                                if affected_obj.score > vulnerable.score:
                                    affected_obj.oudated_version = local_version
                                    vulnerable = affected_obj
                if vulnerable:
                    if vulnerable.score < min_score:
                        ret['Controlled'].append(vulnerable.get_report(verbose, show_profile, profile))
                    else:
                        ret['Failure'].append(vulnerable.get_report(verbose, show_profile, profile))

    if tags != '*':
        log.debug("tags: %s", tags)
        remove = []
        for i, failure in enumerate(ret['Failure']):
            if not fnmatch.fnmatch(failure.keys()[0], tags):
                remove.append(i)
        remove.reverse()
        for i in remove:
            ret['Failure'].pop(i)

        remove = []
        for i, failure in enumerate(ret['Controlled']):
            if not fnmatch.fnmatch(failure.keys()[0], tags):
                remove.append(i)
        remove.reverse()
        for i in remove:
            ret['Controlled'].pop(i)

    if not ret['Controlled']:
        ret.pop('Controlled')

    return ret

Example 21

View license
def main(argv):
    arguments = docopt(__doc__, version=str(
        os.path.basename(__file__)) + " " + VERSION, options_first=False)

    if arguments['--spinnaker_address'] is not None:
        spinnaker_address = arguments['--spinnaker_address']
    else:
        spinnaker_address = SPINNAKER_HOST

    pp = pprint.PrettyPrinter(indent=4)

    app_name = arguments['--app_name']
    pipeline_name = arguments['--pipeline_name']
    sg_id = arguments['--sg_id']
    aws_region = arguments['--aws_region']
    vpc_id = arguments['--vpc_id']
    vpc_sg_id = arguments['--vpc_sg_id']
    mgmt_sg_id = arguments['--mgmt_sg_id']
    vpc_name = arguments['--vpc_name']
    iam_role = arguments['--iam_role']

    pipeline_json_file = 'pipeline_create.json'
    app_json_file = 'application_create.json'
    lb_json_file = 'loadbalancer_create.json'

    pipeline = {}
    application = {}
    loadbalancer = {}

    aws_conn = boto.vpc.connect_to_region(aws_region)

    spin_tools = spinnaker(spinnaker_address=spinnaker_address,
                           spinnaker_port=SPINNAKER_PORT, gate_port=GATE_PORT)

    with open(pipeline_json_file) as pipeline_file:
        pipeline = json.load(pipeline_file)

    with open(app_json_file) as app_file:
        application = json.load(app_file)

    with open(lb_json_file) as lb_file:
        loadbalancer = json.load(lb_file)

    stack = 'teststack'
    detail = 'testdetail'

    '''
    Get the subnet information
    '''
    subnet_type = pipeline['stages'][1]['clusters'][0]['subnetType']

    tag_name_filter = vpc_name + "." + \
        re.sub("\ \(.*\)", '', subnet_type) + "." + aws_region

    try:
        all_subnets = aws_conn.get_all_subnets(
            filters={'vpc_id': vpc_id, 'tag:Name': tag_name_filter})
    except Exception, e:
        print "ERROR: Could not connect to AWS. Check your aws keys."
        exit(1)

    subnet_azs = [s.availability_zone for s in all_subnets]

    if len(subnet_azs) == 1:
        print "No subnets found!"
        exit(1)

    '''
    Configure the special load balancer vars
    '''
    loadbalancer['job'][0]['stack'] = stack
    loadbalancer['job'][0]['detail'] = detail

    loadbalancer['job'][0]['vpcId'] = vpc_id
    loadbalancer['job'][0]['region'] = aws_region
    loadbalancer['job'][0]['name'] = app_name + '-' + stack + '-' + detail
    loadbalancer['job'][0]['subnetType'] = "eelb_public (" + vpc_name + ")"

    loadbalancer['job'][0]['availabilityZones'][aws_region] = subnet_azs
    loadbalancer['job'][0]['regionZones'] = subnet_azs

    loadbalancer['application'] = app_name
    loadbalancer['description'] = 'Create Load Balancer: ' + \
        loadbalancer['job'][0]['name']

    '''
    Configure the special pipeline vars
    '''

    pipeline['name'] = pipeline_name

    pipeline['stages'][1]['clusters'][0][
        'subnetType'] = "ec2_public (" + vpc_name + ")"

    pipeline['stages'][1]['clusters'][0]['iamRole'] = iam_role + '_profile'

    pipeline['stages'][1]['clusters'][0][
        'securityGroups'] = [sg_id, vpc_sg_id, mgmt_sg_id]

    pipeline['stages'][1]['clusters'][0][
        'loadBalancers'] = [app_name + '-' + stack + '-' + detail]

    pipeline['stages'][1]['clusters'][0]['application'] = app_name
    pipeline['stages'][1]['clusters'][0]['stack'] = stack

    pipeline['stages'][0]['regions'] = [aws_region]

    pipeline['stages'][1]['clusters'][0][
        'availabilityZones'][aws_region] = subnet_azs

    pipeline['name'] = pipeline_name
    pipeline['application'] = app_name

    '''
    Set the special applicaiton vars
    '''
    application['job'][0]['application']['name'] = app_name
    application['application'] = app_name
    application['description'] = 'Create Application: ' + app_name

    spin_tools.wait_for_8084()


    if spin_tools.create_application(application):
        if spin_tools.load_balancer(loadbalancer):
            if spin_tools.create_pipeline(pipeline):
                print "Everything created successfully."
            else:
                print "Pipeline creation failed."
                pp.pprint(pipeline)
                print spin_tools.error_response
        else:
            print "Load Balancer Creation failed, not continuing."
            pp.pprint(loadbalancer)
    else:
        print "Application creation failed, not continuing."
        pp.pprint(application)

Example 22

View license
def main(argv):
    arguments = docopt(__doc__, version=str(
        os.path.basename(__file__)) + " " + VERSION, options_first=False)

    if arguments['--spinnaker_address'] is not None:
        spinnaker_address = arguments['--spinnaker_address']
    else:
        spinnaker_address = SPINNAKER_HOST

    pp = pprint.PrettyPrinter(indent=4)

    app_name = arguments['--app_name']
    pipeline_name = arguments['--pipeline_name']
    sg_id = arguments['--sg_id']
    aws_region = arguments['--aws_region']
    vpc_id = arguments['--vpc_id']
    vpc_sg_id = arguments['--vpc_sg_id']
    mgmt_sg_id = arguments['--mgmt_sg_id']
    vpc_name = arguments['--vpc_name']
    iam_role = arguments['--iam_role']

    pipeline_json_file = 'pipeline_create.json'
    app_json_file = 'application_create.json'
    lb_json_file = 'loadbalancer_create.json'

    pipeline = {}
    application = {}
    loadbalancer = {}

    aws_conn = boto.vpc.connect_to_region(aws_region)

    spin_tools = spinnaker(spinnaker_address=spinnaker_address,
                           spinnaker_port=SPINNAKER_PORT, gate_port=GATE_PORT)

    with open(pipeline_json_file) as pipeline_file:
        pipeline = json.load(pipeline_file)

    with open(app_json_file) as app_file:
        application = json.load(app_file)

    with open(lb_json_file) as lb_file:
        loadbalancer = json.load(lb_file)

    stack = 'teststack'
    detail = 'testdetail'

    '''
    Get the subnet information
    '''
    subnet_type = pipeline['stages'][1]['clusters'][0]['subnetType']

    tag_name_filter = vpc_name + "." + \
        re.sub("\ \(.*\)", '', subnet_type) + "." + aws_region

    try:
        all_subnets = aws_conn.get_all_subnets(
            filters={'vpc_id': vpc_id, 'tag:Name': tag_name_filter})
    except Exception, e:
        print "ERROR: Could not connect to AWS. Check your aws keys."
        exit(1)

    subnet_azs = [s.availability_zone for s in all_subnets]

    if len(subnet_azs) == 1:
        print "No subnets found!"
        exit(1)

    '''
    Configure the special load balancer vars
    '''
    loadbalancer['job'][0]['stack'] = stack
    loadbalancer['job'][0]['detail'] = detail

    loadbalancer['job'][0]['vpcId'] = vpc_id
    loadbalancer['job'][0]['region'] = aws_region
    loadbalancer['job'][0]['name'] = app_name + '-' + stack + '-' + detail
    loadbalancer['job'][0]['subnetType'] = "eelb_public (" + vpc_name + ")"

    loadbalancer['job'][0]['availabilityZones'][aws_region] = subnet_azs
    loadbalancer['job'][0]['regionZones'] = subnet_azs

    loadbalancer['application'] = app_name
    loadbalancer['description'] = 'Create Load Balancer: ' + \
        loadbalancer['job'][0]['name']

    '''
    Configure the special pipeline vars
    '''

    pipeline['name'] = pipeline_name

    pipeline['stages'][1]['clusters'][0][
        'subnetType'] = "ec2_public (" + vpc_name + ")"

    pipeline['stages'][1]['clusters'][0]['iamRole'] = iam_role + '_profile'

    pipeline['stages'][1]['clusters'][0][
        'securityGroups'] = [sg_id, vpc_sg_id, mgmt_sg_id]

    pipeline['stages'][1]['clusters'][0][
        'loadBalancers'] = [app_name + '-' + stack + '-' + detail]

    pipeline['stages'][1]['clusters'][0]['application'] = app_name
    pipeline['stages'][1]['clusters'][0]['stack'] = stack

    pipeline['stages'][0]['regions'] = [aws_region]

    pipeline['stages'][1]['clusters'][0][
        'availabilityZones'][aws_region] = subnet_azs

    pipeline['name'] = pipeline_name
    pipeline['application'] = app_name

    '''
    Set the special applicaiton vars
    '''
    application['job'][0]['application']['name'] = app_name
    application['application'] = app_name
    application['description'] = 'Create Application: ' + app_name

    spin_tools.wait_for_8084()


    if spin_tools.create_application(application):
        if spin_tools.load_balancer(loadbalancer):
            if spin_tools.create_pipeline(pipeline):
                print "Everything created successfully."
            else:
                print "Pipeline creation failed."
                pp.pprint(pipeline)
                print spin_tools.error_response
        else:
            print "Load Balancer Creation failed, not continuing."
            pp.pprint(loadbalancer)
    else:
        print "Application creation failed, not continuing."
        pp.pprint(application)

Example 23

Project: megaphone
Source File: megaphone.py
View license
def readstatus(name,url,q):
	bug("now in readstatus")
	result = {}
	validstatus = ["OK", "Unknown", "Warning", "Critical"]
	# this is to support status somewhere other than 'status' under the root of a service
	# {"id": "ok2_status", "url": {"addr": "http://localhost:18999/status", "jsonpath": "megaphone/status"}}
	if isinstance(url, dict):
		bug("url is a dictionary")
		data = AutoVivification()
		if 'addr' not in url:
			mymsg = "ERROR: couldn't find addr in url, nothing to check"
			bug(mymsg)
			data['status'] = "Critical"
			data['message'] = mymsg
		else:
			if 'jsonpath' in url:
				bug("parsing jsonpath and addr in url: jsonpath: %s, addr: %s" % (url['jsonpath'],url['addr']))
				try:
					tdata = json.load(urllib2.urlopen(url['addr'], timeout = TIMEOUT))
					v = "tdata"
					for i in url['jsonpath'].split("/"):
						bug("i: %s" % i)
						if i:  
							v += "['%s']" % i
					bug("eval type:")
					if DEBUG:
						print type(eval(v))
						print type(eval(v).encode('ascii','ignore'))
					data['status'] = eval(v)
					data['date'] = ts
					msg = "Status from path %s: %s" % (url['jsonpath'], data['status'])
					bug(msg)
					data['message'] = msg
				except:
					msg = "error collecting results from addr: %s, jsonpath: %s" % (url['addr'],url['jsonpath'])
					bug(msg)
					data['status'] = "Critical"
					data['date'] = ts
					data['message'] = msg
			else:
				bug("no jsonpath detected in url, using only addr path")
				try:
					data = json.load(urllib2.urlopen(url['addr'], timeout = TIMEOUT))
				except:
					msg = "timeout connecting to %s" % (url['addr'])
					bug(msg)
					data['status'] = "Critical"
					data['date'] = ts
					data['message'] = msg
			if 'statusoverride' in url:
				bug("statusoverride detected in url")
				if url['statusoverride'] not in validstatus:
					data['status'] = "Critical"
					data['message'] = "ERROR: invalid status '%s' written to statusoverride!" % url['statusoverride']
				else:
					data['status'] = url['statusoverride']
					data['message'] = "NOTICE: statusoverride used!"
	else:
		bug("url object isn't a dictionary, processing normally")
		try:
			data = json.load(urllib2.urlopen(url, timeout = TIMEOUT))
		except:
			data = AutoVivification()
			msg = "timeout connecting to %s" % (url)
			bug(msg)
			data['status'] = "Critical"
			data['date'] = ts
			data['message'] = msg
	if "status" not in data.keys():
		data['status'] = "Critical"
		data['message'] = "No status was found on given path!"
	if data["status"] not in validstatus:
		data['status'] = "Critical"
		data['message'] = "ERROR: status value '%s' not valid!" % data["status"]
	bug("Data:")
	if DEBUG:
		print data
	result[name] = data
	if DEBUG:
		print result
		print type(name)
		print type(result)
	if q:
		q.put(result)
	else:
		return data

Example 24

Project: rail
Source File: emr_runner.py
View license
def run_job_flow(branding, json_config, force, no_browser=False,
                    aws_exe='aws', profile='default', region=None):
    """ Submits job flow to EMR. Should eventually monitor it.

        branding: text file with branding to print to screen before running
            job. This is where the name of a software package or ASCII art 
            can go.
        json_config: JSON configuration file. Google Getting Started with
            Amazon Elastic MapReduce for formatting information.
        force: True iff all existing directories should be erased when
            writing intermediates.
        no_browser: True iff webpage with status of job shouldn't automatically
            be opened in default browser.
        aws_exe: path to AWS CLI executable
        profile: name of AWS CLI profile from which to grab AWS access key ID
            and secret access key.
        region: Amazon data center in which to run job flow. If None, uses
            region from AWS CLI.

        No return value.
    """
    iface = dp_iface.DooplicityInterface(
                        branding=branding,
                        opener='Started job flow submission script on {time}.'
                    )
    try:
        '''Should ultimately check to see which regions are available to user
        w/ API.'''
        aws_ansible = ab.AWSAnsible(profile=profile, region=region,
                                    valid_regions=_aws_regions)
        s3_ansible = ab.S3Ansible(aws_exe=aws_exe, profile=profile,
                                    iface=iface)
        # Serialize JSON configuration
        if json_config is not None:
            with open(json_config) as json_stream:
                full_payload = json.load(json_stream)
        else:
            full_payload = json.load(sys.stdin)
        try:
            job_flow = full_payload['Steps']
        except Exception:
            raise RuntimeError(
                    'Input JSON not in proper format. Ensure that the JSON '
                    'object has a Steps key.'
                )
        step_count = len(job_flow)
        steps = OrderedDict()
        # Check steps for requred data
        required_data = set(['input', 'output', 'mapper', 'reducer'])
        bad_output_data = []
        try:
            for step in job_flow:
                if 'hadoop-streaming' \
                    not in step['HadoopJarStep']['Jar'].lower():
                    '''Don't check command-line parameters if not a 
                    Hadoop Streaming step.'''
                    continue
                step_args = {}
                for j in xrange(0, len(step['HadoopJarStep']['Args']), 2):
                    arg_name = step['HadoopJarStep']['Args'][j][1:].strip()
                    if arg_name == 'input':
                        try:
                            step_args['input'] = ','.join(
                                    [step['HadoopJarStep']['Args'][j+1],
                                     step_args['input']]
                                )
                        except KeyError:
                            step_args['input'] \
                                = step['HadoopJarStep']['Args'][j+1]
                    elif arg_name in required_data:
                        step_args[step['HadoopJarStep']['Args'][j][1:]] \
                            = step['HadoopJarStep']['Args'][j+1].strip()
                steps[step['Name']] = step_args
        except (KeyError, IndexError):
            iface.fail(
                    'JSON file not in proper format. Ensure '
                    'that each StepConfig object has a HadoopJarStep '
                    'object with an Args array and a Name string.'
                )
            raise
        iface.step('Read job flow from input JSON.')
        iface.status('Checking that output directories on S3 are writable...')
        '''Check steps for required Hadoop Streaming command-line parameters
        and for whether outputs are writable.'''
        missing_data = defaultdict(list)
        identity_steps = []
        identity_mappers \
            = set(['cat', 'org.apache.hadoop.mapred.lib.IdentityMapper'])
        identity_reducers \
            = set(['cat', 'org.apache.hadoop.mapred.lib.IdentityReducer'])
        for step in steps:
            step_data = steps[step]
            for required_parameter in required_data:
                if required_parameter not in step_data:
                    missing_data[step].append('-' + required_parameter)
                elif not force and required_parameter == 'output' \
                    and ab.Url(step_data['output']).is_s3 \
                         and s3_ansible.is_dir(step_data['output']):
                    bad_output_data.append(step)
            if 'mapper' in steps[step] and 'reducer' in step_data \
                and step_data['mapper'] in identity_mappers \
                and step_data['reducer'] in identity_reducers:
                identity_steps.append(step)
        errors = []
        if missing_data:
            errors.extend(['Step "%s" is missing required parameter(s) "%s".' % 
                                (step, ', '.join(missing_data[step]))
                                for step in missing_data])
        if bad_output_data:
            errors.extend(['Output "directory" "%s" of step "%s" already '
                           'exists, and --force was not invoked to permit '
                           'overwriting it.' 
                           % (steps[step]['output'], step)
                           for step in bad_output_data])
        if identity_steps:
            errors.extend([('Step "%s" has both an identity mapper and an '
                            'identity reducer, which is redundant. Remove the '
                            'step before trying again.') % step])
        if errors:
            raise RuntimeError('\n'.join([('%d) ' % (i+1)) + error
                                    for i, error in enumerate(errors)]))
        iface.step('Verified that output directories on S3 are writable.')
        '''Remove intermediate directories and create buckets if they
        don't already exist so EMR doesn't choke when writing output. The
        idea here is to make a bucket feel like an ordinary directory to
        the user.'''
        buckets = set()
        for step in steps:
            step_data = steps[step]
            if ab.Url(step_data['output']).is_s3:
                s3_ansible.remove_dir(steps[step]['output'])
                buckets.add(ab.bucket_from_url(step_data['output']))
        for bucket in buckets:
            try:
                s3_ansible.create_bucket(bucket, region=aws_ansible.region)
            except Exception as e:
                raise RuntimeError(('Bucket %s already exists on S3. Change '
                                    'affected output directories in job flow '
                                    'and try again. The more distinctive the '
                                    'name chosen, the less likely it\'s '
                                    'already a bucket on S3. It may be '
                                    'easier to create a bucket first using '
                                    'the web interface and use its name + a '
                                    'subdirectory as the output directory.')
                                    % bucket)
        iface.step('Set up output directories on S3.')
        iface.status('Submitting job flow...')
        try:
            job_flow_response = aws_ansible.post_request(full_payload)
        except urllib2.HTTPError as e:
            if 'bad request' in str(e).lower():
                raise RuntimeError('(' + str(e) + ');'
                                + (' ensure that IAM roles are '
                                'configured properly. This '
                                'may require talking to your AWS account '
                                'admin. See '
                                'http://docs.aws.amazon.com/ElasticMapReduce/'
                                'latest/DeveloperGuide/emr-'
                                'iam-roles-defaultroles.html for more '
                                'information. In most cases, the solution '
                                'to is to run "aws emr '
                                'create-default-roles", and try again.'))
            else:
                raise
        json_response = json.load(job_flow_response)
        if 'JobFlowId' not in json_response:
            raise RuntimeError('Job flow submission failed. Server returned '
                               'the following response: %s.'
                                % json.dumps(json_response, sort_keys=True,
                                    indent=4, separators=(',', ': ')))
        else:
            job_flow_id = json_response['JobFlowId']
            iface.step(('Submitted job flow.\n'
                        '*****Job flow ID is %s .*****\n' % job_flow_id) + 
                        ('*****Submission can be monitored at %s .*****' %
                            _emr_url.format(region=aws_ansible.region,
                                            job_flow_id=job_flow_id)))
            if not no_browser:
                webbrowser.open(_emr_url.format(region=aws_ansible.region,
                                                job_flow_id=job_flow_id),
                                    new=2) # Open in new tab
                iface.step('Opening URL in default browser, if possible.')
        iface.done(closer=('Finished job flow submission script '
                           'on {date}. Run time was {length} seconds.')
                )
    except (Exception, GeneratorExit):
         # GeneratorExit added just in case this happens on modifying code
        iface.fail()
        raise
    except (KeyboardInterrupt, SystemExit):
        if 'job_flow_id' not in locals():
            iface.fail(opener='*****Terminated without submitting job.*****',
                        middler=('End time was {date}. '
                                 'Script was running for {length} seconds.'))
        else:
            iface.fail(opener='*****Terminated after submitting job.*****',
                        middler=('End time was {date}. '
                                 'Script was running for {length} seconds.'))
        raise

Example 25

Project: rail
Source File: emr_runner.py
View license
def run_job_flow(branding, json_config, force, no_browser=False,
                    aws_exe='aws', profile='default', region=None):
    """ Submits job flow to EMR. Should eventually monitor it.

        branding: text file with branding to print to screen before running
            job. This is where the name of a software package or ASCII art 
            can go.
        json_config: JSON configuration file. Google Getting Started with
            Amazon Elastic MapReduce for formatting information.
        force: True iff all existing directories should be erased when
            writing intermediates.
        no_browser: True iff webpage with status of job shouldn't automatically
            be opened in default browser.
        aws_exe: path to AWS CLI executable
        profile: name of AWS CLI profile from which to grab AWS access key ID
            and secret access key.
        region: Amazon data center in which to run job flow. If None, uses
            region from AWS CLI.

        No return value.
    """
    iface = dp_iface.DooplicityInterface(
                        branding=branding,
                        opener='Started job flow submission script on {time}.'
                    )
    try:
        '''Should ultimately check to see which regions are available to user
        w/ API.'''
        aws_ansible = ab.AWSAnsible(profile=profile, region=region,
                                    valid_regions=_aws_regions)
        s3_ansible = ab.S3Ansible(aws_exe=aws_exe, profile=profile,
                                    iface=iface)
        # Serialize JSON configuration
        if json_config is not None:
            with open(json_config) as json_stream:
                full_payload = json.load(json_stream)
        else:
            full_payload = json.load(sys.stdin)
        try:
            job_flow = full_payload['Steps']
        except Exception:
            raise RuntimeError(
                    'Input JSON not in proper format. Ensure that the JSON '
                    'object has a Steps key.'
                )
        step_count = len(job_flow)
        steps = OrderedDict()
        # Check steps for requred data
        required_data = set(['input', 'output', 'mapper', 'reducer'])
        bad_output_data = []
        try:
            for step in job_flow:
                if 'hadoop-streaming' \
                    not in step['HadoopJarStep']['Jar'].lower():
                    '''Don't check command-line parameters if not a 
                    Hadoop Streaming step.'''
                    continue
                step_args = {}
                for j in xrange(0, len(step['HadoopJarStep']['Args']), 2):
                    arg_name = step['HadoopJarStep']['Args'][j][1:].strip()
                    if arg_name == 'input':
                        try:
                            step_args['input'] = ','.join(
                                    [step['HadoopJarStep']['Args'][j+1],
                                     step_args['input']]
                                )
                        except KeyError:
                            step_args['input'] \
                                = step['HadoopJarStep']['Args'][j+1]
                    elif arg_name in required_data:
                        step_args[step['HadoopJarStep']['Args'][j][1:]] \
                            = step['HadoopJarStep']['Args'][j+1].strip()
                steps[step['Name']] = step_args
        except (KeyError, IndexError):
            iface.fail(
                    'JSON file not in proper format. Ensure '
                    'that each StepConfig object has a HadoopJarStep '
                    'object with an Args array and a Name string.'
                )
            raise
        iface.step('Read job flow from input JSON.')
        iface.status('Checking that output directories on S3 are writable...')
        '''Check steps for required Hadoop Streaming command-line parameters
        and for whether outputs are writable.'''
        missing_data = defaultdict(list)
        identity_steps = []
        identity_mappers \
            = set(['cat', 'org.apache.hadoop.mapred.lib.IdentityMapper'])
        identity_reducers \
            = set(['cat', 'org.apache.hadoop.mapred.lib.IdentityReducer'])
        for step in steps:
            step_data = steps[step]
            for required_parameter in required_data:
                if required_parameter not in step_data:
                    missing_data[step].append('-' + required_parameter)
                elif not force and required_parameter == 'output' \
                    and ab.Url(step_data['output']).is_s3 \
                         and s3_ansible.is_dir(step_data['output']):
                    bad_output_data.append(step)
            if 'mapper' in steps[step] and 'reducer' in step_data \
                and step_data['mapper'] in identity_mappers \
                and step_data['reducer'] in identity_reducers:
                identity_steps.append(step)
        errors = []
        if missing_data:
            errors.extend(['Step "%s" is missing required parameter(s) "%s".' % 
                                (step, ', '.join(missing_data[step]))
                                for step in missing_data])
        if bad_output_data:
            errors.extend(['Output "directory" "%s" of step "%s" already '
                           'exists, and --force was not invoked to permit '
                           'overwriting it.' 
                           % (steps[step]['output'], step)
                           for step in bad_output_data])
        if identity_steps:
            errors.extend([('Step "%s" has both an identity mapper and an '
                            'identity reducer, which is redundant. Remove the '
                            'step before trying again.') % step])
        if errors:
            raise RuntimeError('\n'.join([('%d) ' % (i+1)) + error
                                    for i, error in enumerate(errors)]))
        iface.step('Verified that output directories on S3 are writable.')
        '''Remove intermediate directories and create buckets if they
        don't already exist so EMR doesn't choke when writing output. The
        idea here is to make a bucket feel like an ordinary directory to
        the user.'''
        buckets = set()
        for step in steps:
            step_data = steps[step]
            if ab.Url(step_data['output']).is_s3:
                s3_ansible.remove_dir(steps[step]['output'])
                buckets.add(ab.bucket_from_url(step_data['output']))
        for bucket in buckets:
            try:
                s3_ansible.create_bucket(bucket, region=aws_ansible.region)
            except Exception as e:
                raise RuntimeError(('Bucket %s already exists on S3. Change '
                                    'affected output directories in job flow '
                                    'and try again. The more distinctive the '
                                    'name chosen, the less likely it\'s '
                                    'already a bucket on S3. It may be '
                                    'easier to create a bucket first using '
                                    'the web interface and use its name + a '
                                    'subdirectory as the output directory.')
                                    % bucket)
        iface.step('Set up output directories on S3.')
        iface.status('Submitting job flow...')
        try:
            job_flow_response = aws_ansible.post_request(full_payload)
        except urllib2.HTTPError as e:
            if 'bad request' in str(e).lower():
                raise RuntimeError('(' + str(e) + ');'
                                + (' ensure that IAM roles are '
                                'configured properly. This '
                                'may require talking to your AWS account '
                                'admin. See '
                                'http://docs.aws.amazon.com/ElasticMapReduce/'
                                'latest/DeveloperGuide/emr-'
                                'iam-roles-defaultroles.html for more '
                                'information. In most cases, the solution '
                                'to is to run "aws emr '
                                'create-default-roles", and try again.'))
            else:
                raise
        json_response = json.load(job_flow_response)
        if 'JobFlowId' not in json_response:
            raise RuntimeError('Job flow submission failed. Server returned '
                               'the following response: %s.'
                                % json.dumps(json_response, sort_keys=True,
                                    indent=4, separators=(',', ': ')))
        else:
            job_flow_id = json_response['JobFlowId']
            iface.step(('Submitted job flow.\n'
                        '*****Job flow ID is %s .*****\n' % job_flow_id) + 
                        ('*****Submission can be monitored at %s .*****' %
                            _emr_url.format(region=aws_ansible.region,
                                            job_flow_id=job_flow_id)))
            if not no_browser:
                webbrowser.open(_emr_url.format(region=aws_ansible.region,
                                                job_flow_id=job_flow_id),
                                    new=2) # Open in new tab
                iface.step('Opening URL in default browser, if possible.')
        iface.done(closer=('Finished job flow submission script '
                           'on {date}. Run time was {length} seconds.')
                )
    except (Exception, GeneratorExit):
         # GeneratorExit added just in case this happens on modifying code
        iface.fail()
        raise
    except (KeyboardInterrupt, SystemExit):
        if 'job_flow_id' not in locals():
            iface.fail(opener='*****Terminated without submitting job.*****',
                        middler=('End time was {date}. '
                                 'Script was running for {length} seconds.'))
        else:
            iface.fail(opener='*****Terminated after submitting job.*****',
                        middler=('End time was {date}. '
                                 'Script was running for {length} seconds.'))
        raise

Example 26

Project: ryu
Source File: test_ofctl.py
View license
def _add_tests():
    _ofp_vers = {
        'of10': 0x01,
        'of12': 0x03,
        'of13': 0x04,
        'of14': 0x05,
        'of15': 0x06,
    }

    _test_cases = {
        'of10': [
            {
                'method': ofctl_v1_0.mod_flow_entry,
                'request': '1-2-ofp_flow_mod.packet.json',
                'reply': None
            },
        ],
        'of12': [
            {
                'method': ofctl_v1_2.get_desc_stats,
                'request': '3-24-ofp_desc_stats_request.packet.json',
                'reply': '3-0-ofp_desc_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_queue_stats,
                'request': '3-37-ofp_queue_stats_request.packet.json',
                'reply': '3-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_queue_stats,
                'request': 'lib-ofctl-ofp_queue_stats_request.packet1.json',
                'reply': '3-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_queue_stats,
                'request': 'lib-ofctl-ofp_queue_stats_request.packet2.json',
                'reply': '3-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_queue_stats,
                'request': 'lib-ofctl-ofp_queue_stats_request.packet3.json',
                'reply': '3-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_queue_config,
                'request': '3-35-ofp_queue_get_config_request.packet.json',
                'reply': '3-36-ofp_queue_get_config_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_queue_config,
                'request': 'lib-ofctl-ofp_queue_get_config_request.packet.json',
                'reply': '3-36-ofp_queue_get_config_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_flow_stats,
                'request': '3-11-ofp_flow_stats_request.packet.json',
                'reply': '3-12-ofp_flow_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_aggregate_flow_stats,
                'request': '3-25-ofp_aggregate_stats_request.packet.json',
                'reply': '3-26-ofp_aggregate_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_table_stats,
                'request': '3-27-ofp_table_stats_request.packet.json',
                'reply': '3-28-ofp_table_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_port_stats,
                'request': '3-29-ofp_port_stats_request.packet.json',
                'reply': '3-30-ofp_port_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_port_stats,
                'request': 'lib-ofctl-ofp_port_stats_request.packet.json',
                'reply': '3-30-ofp_port_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_group_stats,
                'request': '3-61-ofp_group_stats_request.packet.json',
                'reply': '3-62-ofp_group_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_group_stats,
                'request': 'lib-ofctl-ofp_group_stats_request.packet.json',
                'reply': '3-62-ofp_group_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_group_features,
                'request': '3-31-ofp_group_features_stats_request.packet.json',
                'reply': '3-32-ofp_group_features_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.get_group_desc,
                'request': '3-33-ofp_group_desc_stats_request.packet.json',
                'reply': '3-34-ofp_group_desc_stats_reply.packet.json'
            },
            # In OpenFlow 1.2, ofp_port_desc is not defined.
            # We use ofp_features_request to get ports description instead.
            {
                'method': ofctl_v1_2.get_port_desc,
                'request': '3-5-ofp_features_request.packet.json',
                'reply': '3-6-ofp_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_2.mod_flow_entry,
                'request': '3-2-ofp_flow_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_2.mod_group_entry,
                'request': '3-21-ofp_group_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_2.mod_port_behavior,
                'request': '3-22-ofp_port_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_2.send_experimenter,
                'request': '3-16-ofp_experimenter.packet.json',
                'reply': None
            },
        ],
        'of13': [
            {
                'method': ofctl_v1_3.get_desc_stats,
                'request': '4-24-ofp_desc_request.packet.json',
                'reply': '4-0-ofp_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_queue_stats,
                'request': '4-37-ofp_queue_stats_request.packet.json',
                'reply': '4-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_queue_stats,
                'request': 'lib-ofctl-ofp_queue_stats_request.packet1.json',
                'reply': '4-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_queue_stats,
                'request': 'lib-ofctl-ofp_queue_stats_request.packet2.json',
                'reply': '4-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_queue_stats,
                'request': 'lib-ofctl-ofp_queue_stats_request.packet3.json',
                'reply': '4-38-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_queue_config,
                'request': '4-35-ofp_queue_get_config_request.packet.json',
                'reply': '4-36-ofp_queue_get_config_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_queue_config,
                'request': 'lib-ofctl-ofp_queue_get_config_request.packet.json',
                'reply': '4-36-ofp_queue_get_config_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_flow_stats,
                'request': '4-11-ofp_flow_stats_request.packet.json',
                'reply': '4-12-ofp_flow_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_aggregate_flow_stats,
                'request': '4-25-ofp_aggregate_stats_request.packet.json',
                'reply': '4-26-ofp_aggregate_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_table_stats,
                'request': '4-27-ofp_table_stats_request.packet.json',
                'reply': '4-28-ofp_table_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_table_features,
                'request': 'lib-ofctl-ofp_table_features_request.packet.json',
                'reply': '4-56-ofp_table_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_port_stats,
                'request': '4-29-ofp_port_stats_request.packet.json',
                'reply': '4-30-ofp_port_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_port_stats,
                'request': 'lib-ofctl-ofp_port_stats_request.packet.json',
                'reply': '4-30-ofp_port_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_meter_stats,
                'request': '4-49-ofp_meter_stats_request.packet.json',
                'reply': '4-50-ofp_meter_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_meter_stats,
                'request': 'lib-ofctl-ofp_meter_stats_request.packet.json',
                'reply': '4-50-ofp_meter_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_meter_features,
                'request': '4-51-ofp_meter_features_request.packet.json',
                'reply': '4-52-ofp_meter_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_meter_config,
                'request': '4-47-ofp_meter_config_request.packet.json',
                'reply': '4-48-ofp_meter_config_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_meter_config,
                'request': 'lib-ofctl-ofp_meter_config_request.packet.json',
                'reply': '4-48-ofp_meter_config_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_group_stats,
                'request': '4-57-ofp_group_stats_request.packet.json',
                'reply': '4-58-ofp_group_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_group_stats,
                'request': 'lib-ofctl-ofp_group_stats_request.packet.json',
                'reply': '4-58-ofp_group_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_group_features,
                'request': '4-31-ofp_group_features_request.packet.json',
                'reply': '4-32-ofp_group_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_group_desc,
                'request': '4-33-ofp_group_desc_request.packet.json',
                'reply': '4-34-ofp_group_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.get_port_desc,
                'request': '4-53-ofp_port_desc_request.packet.json',
                'reply': '4-54-ofp_port_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_3.mod_flow_entry,
                'request': '4-2-ofp_flow_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_3.mod_meter_entry,
                'request': '4-45-ofp_meter_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_3.mod_group_entry,
                'request': '4-21-ofp_group_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_3.mod_port_behavior,
                'request': '4-22-ofp_port_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_3.send_experimenter,
                'request': '4-16-ofp_experimenter.packet.json',
                'reply': None
            },
        ],
        'of14': [
            {
                'method': ofctl_v1_4.get_desc_stats,
                'request': '5-24-ofp_desc_request.packet.json',
                'reply': '5-0-ofp_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_queue_stats,
                'request': '5-35-ofp_queue_stats_request.packet.json',
                'reply': '5-36-ofp_queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_queue_desc,
                'request': '5-63-ofp_queue_desc_request.packet.json',
                'reply': '5-64-ofp_queue_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_flow_stats,
                'request': '5-11-ofp_flow_stats_request.packet.json',
                'reply': '5-12-ofp_flow_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_aggregate_flow_stats,
                'request': '5-25-ofp_aggregate_stats_request.packet.json',
                'reply': '5-26-ofp_aggregate_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_table_stats,
                'request': '5-27-ofp_table_stats_request.packet.json',
                'reply': '5-28-ofp_table_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_table_features,
                'request': 'lib-ofctl-ofp_table_features_request.packet.json',
                'reply': '5-54-ofp_table_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_port_stats,
                'request': '5-29-ofp_port_stats_request.packet.json',
                'reply': '5-30-ofp_port_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_meter_stats,
                'request': '5-47-ofp_meter_stats_request.packet.json',
                'reply': '5-48-ofp_meter_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_meter_features,
                'request': '5-49-ofp_meter_features_request.packet.json',
                'reply': '5-50-ofp_meter_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_meter_config,
                'request': '5-45-ofp_meter_config_request.packet.json',
                'reply': '5-46-ofp_meter_config_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_group_stats,
                'request': '5-55-ofp_group_stats_request.packet.json',
                'reply': '5-56-ofp_group_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_group_features,
                'request': '5-31-ofp_group_features_request.packet.json',
                'reply': '5-32-ofp_group_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_group_desc,
                'request': '5-33-ofp_group_desc_request.packet.json',
                'reply': '5-34-ofp_group_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.get_port_desc,
                'request': '5-51-ofp_port_desc_request.packet.json',
                'reply': '5-52-ofp_port_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_4.mod_flow_entry,
                'request': '5-2-ofp_flow_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_4.mod_meter_entry,
                'request': '5-43-ofp_meter_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_4.mod_group_entry,
                'request': '5-21-ofp_group_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_4.mod_port_behavior,
                'request': '5-22-ofp_port_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_4.send_experimenter,
                'request': '5-16-ofp_experimenter.packet.json',
                'reply': None
            },
        ],
        'of15': [
            {
                'method': ofctl_v1_5.get_desc_stats,
                'request': 'libofproto-OFP15-desc_request.packet.json',
                'reply': 'libofproto-OFP15-desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_queue_stats,
                'request': 'lib-ofctl-ofp_queue_stats_request.packet.json',
                'reply': 'libofproto-OFP15-queue_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_queue_desc,
                'request': 'libofproto-OFP15-queue_desc_request.packet.json',
                'reply': 'libofproto-OFP15-queue_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_flow_stats,
                'request': 'libofproto-OFP15-flow_stats_request.packet.json',
                'reply': 'libofproto-OFP15-flow_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_flow_desc_stats,
                'request': 'libofproto-OFP15-flow_desc_request.packet.json',
                'reply': 'libofproto-OFP15-flow_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_flow_desc_stats,
                'request': 'lib-ofctl-OFP15-flow_desc_request.packet.json',
                'reply': 'lib-ofctl-OFP15-flow_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_aggregate_flow_stats,
                'request': 'libofproto-OFP15-aggregate_stats_request.packet.json',
                'reply': 'libofproto-OFP15-aggregate_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_table_stats,
                'request': 'libofproto-OFP15-table_stats_request.packet.json',
                'reply': 'libofproto-OFP15-table_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_table_features,
                'request': 'lib-ofctl-ofp_table_features_request.packet.json',
                'reply': 'libofproto-OFP15-table_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_port_stats,
                'request': 'libofproto-OFP15-port_stats_request.packet.json',
                'reply': 'libofproto-OFP15-port_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_meter_stats,
                'request': 'libofproto-OFP15-meter_stats_request.packet.json',
                'reply': 'libofproto-OFP15-meter_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_meter_features,
                'request': 'libofproto-OFP15-meter_features_request.packet.json',
                'reply': 'libofproto-OFP15-meter_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_meter_desc,
                'request': 'libofproto-OFP15-meter_desc_request.packet.json',
                'reply': 'libofproto-OFP15-meter_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_group_stats,
                'request': 'libofproto-OFP15-group_stats_request.packet.json',
                'reply': 'libofproto-OFP15-group_stats_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_group_features,
                'request': 'libofproto-OFP15-group_features_request.packet.json',
                'reply': 'libofproto-OFP15-group_features_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_group_desc,
                'request': 'libofproto-OFP15-group_desc_request.packet.json',
                'reply': 'libofproto-OFP15-group_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.get_port_desc,
                'request': 'libofproto-OFP15-port_desc_request.packet.json',
                'reply': 'libofproto-OFP15-port_desc_reply.packet.json'
            },
            {
                'method': ofctl_v1_5.mod_flow_entry,
                'request': 'libofproto-OFP15-flow_mod_no_nx.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_5.mod_flow_entry,
                'request': 'lib-ofctl-OFP15-flow_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_5.mod_meter_entry,
                'request': 'libofproto-OFP15-meter_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_5.mod_group_entry,
                'request': 'libofproto-OFP15-group_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_5.mod_port_behavior,
                'request': 'libofproto-OFP15-port_mod.packet.json',
                'reply': None
            },
            {
                'method': ofctl_v1_5.send_experimenter,
                'request': 'libofproto-OFP15-experimenter.packet.json',
                'reply': None
            }
        ],
    }

    def _jsonfile_to_msg(datapath, jsonfile):
        return ofproto_parser.ofp_msg_from_jsondict(
            datapath, json.load(open(jsonfile)))

    this_dir = os.path.dirname(sys.modules[__name__].__file__)
    parser_json_root = os.path.join(this_dir, '../ofproto/json/')
    ofctl_json_root = os.path.join(this_dir, 'ofctl_json/')

    for ofp_ver, tests in _test_cases.items():
        dp = DummyDatapath(_ofp_vers[ofp_ver])
        parser_json_dir = os.path.join(parser_json_root, ofp_ver)
        ofctl_json_dir = os.path.join(ofctl_json_root, ofp_ver)
        for test in tests:
            name = 'test_ofctl_' + ofp_ver + '_' + test['request']
            print('adding %s ...' % name)
            args = {}
            args_json_path = os.path.join(ofctl_json_dir, test['request'])
            if os.path.exists(args_json_path):
                args = json.load(open(args_json_path))
            request = _jsonfile_to_msg(
                dp, os.path.join(parser_json_dir, test['request']))
            reply = None
            expected = None
            if test['reply']:
                reply = _jsonfile_to_msg(
                    dp, os.path.join(parser_json_dir, test['reply']))
                expected = json.load(
                    open(os.path.join(ofctl_json_dir, test['reply'])))
            f = functools.partial(
                Test_ofctl._test, name=name, dp=dp, method=test['method'],
                args=args, request=request, reply=reply, expected=expected)
            test_lib.add_method(Test_ofctl, name, f)

Example 27

Project: PokemonGo-Bot-Backup
Source File: pokecli.py
View license
def init_config():
    parser = argparse.ArgumentParser()
    config_file = os.path.join(_base_dir, 'configs', 'config.json')
    web_dir = "web"

    # If config file exists, load variables from json
    load = {}

    def _json_loader(filename):
        try:
            with open(filename, 'rb') as data:
                load.update(json.load(data))
        except ValueError:
            if jsonlint:
                with open(filename, 'rb') as data:
                    lint = jsonlint()
                    rc = lint.main(['-v', filename])

            logger.critical('Error with configuration file')
            sys.exit(-1)

    # Select a config file code
    parser.add_argument("-cf", "--config", help="Config File to use")
    parser.add_argument("-af", "--auth", help="Auth File to use")

    for _config in ['auth', 'config']:
        config_file = os.path.join(_base_dir, 'configs', _config + '.json')
        config_arg = parser.parse_known_args() and parser.parse_known_args()[0].__dict__[_config] or None

        if config_arg and os.path.isfile(config_arg):
            _json_loader(config_arg)
            config_file = config_arg
        elif os.path.isfile(config_file):
            logger.info('No ' + _config + ' argument specified, checking for ' + config_file)
            _json_loader(config_file)
        else:
            logger.info('Error: No /configs/' + _config + '.json')

    # Read passed in Arguments
    required = lambda x: not x in load
    add_config(
        parser,
        load,
        short_flag="-a",
        long_flag="--auth_service",
        help="Auth Service ('ptc' or 'google')",
        required=required("auth_service"),
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-u",
        long_flag="--username",
        help="Username",
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-ws",
        long_flag="--websocket.server_url",
        help="Connect to websocket server at given url",
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-wss",
        long_flag="--websocket.start_embedded_server",
        help="Start embedded websocket server",
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-wsr",
        long_flag="--websocket.remote_control",
        help="Enable remote control through websocket (requires websocket server url)",
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-p",
        long_flag="--password",
        help="Password",
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-l",
        long_flag="--location",
        help="Location",
        type=parse_unicode_str,
        default=''
    )
    add_config(
        parser,
        load,
        short_flag="-lc",
        long_flag="--location_cache",
        help="Bot will start at last known location",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--forts.spin",
        help="Enable Spinning Pokestops",
        type=bool,
        default=True,
    )
    add_config(
        parser,
        load,
        short_flag="-wmax",
        long_flag="--walk_max",
        help="Walk instead of teleport with given speed",
        type=float,
        default=2.5
    )
    add_config(
        parser,
        load,
        short_flag="-wmin",
        long_flag="--walk_min",
        help="Walk instead of teleport with given speed",
        type=float,
        default=2.5
    )
    add_config(
        parser,
        load,
        short_flag="-k",
        long_flag="--gmapkey",
        help="Set Google Maps API KEY",
        type=str,
        default=None
    )
    add_config(
        parser,
        load,
        short_flag="-e",
        long_flag="--show_events",
        help="Show events",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-d",
        long_flag="--debug",
        help="Debug Mode",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-t",
        long_flag="--test",
        help="Only parse the specified location",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        short_flag="-du",
        long_flag="--distance_unit",
        help="Set the unit to display distance in (e.g, km for kilometers, mi for miles, ft for feet)",
        type=str,
        default='km'
    )
    add_config(
        parser,
        load,
        short_flag="-rt",
        long_flag="--reconnecting_timeout",
        help="Timeout between reconnecting if error occured (in minutes, e.g. 15)",
        type=float,
        default=15.0
    )
    add_config(
        parser,
        load,
        short_flag="-hr",
        long_flag="--health_record",
        help="Send anonymous bot event to GA for bot health record. Set \"health_record\":false if you need disable it.",
        type=bool,
        default=True
    )
    add_config(
        parser,
        load,
        short_flag="-ac",
        long_flag="--forts.avoid_circles",
        help="Avoids circles (pokestops) of the max size set in max_circle_size flag",
        type=bool,
        default=False,
    )
    add_config(
        parser,
        load,
        short_flag="-mcs",
        long_flag="--forts.max_circle_size",
        help="If avoid_circles flag is set, this flag specifies the maximum size of circles (pokestops) avoided",
        type=int,
        default=10,
    )
    add_config(
        parser,
        load,
        short_flag="-crf",
        long_flag="--forts.cache_recent_forts",
        help="Caches recent forts used by max_circle_size",
        type=bool,
        default=True,
    )
    add_config(
        parser,
        load,
        long_flag="--map_object_cache_time",
        help="Amount of seconds to keep the map object in cache (bypass Niantic throttling)",
        type=float,
        default=5.0
    )
    add_config(
        parser,
        load,
        long_flag="--logging.color",
        help="If logging_color is set to true, colorized logging handler will be used",
        type=bool,
        default=True
    )
    add_config(
        parser,
        load,
        long_flag="--logging.clean",
        help="If clean_logging is set to true, meta data will be stripped from the log messages",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--heartbeat_threshold",
        help="A threshold between each heartbeat sending to server",
        type=int,
        default=10
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.show_at_start",
        help="Logs all pokemon in the bag at bot start",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.show_count",
        help="Shows the amount of which pokemon (minimum 1)",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.show_candies",
        help="Shows the amount of candies for each pokemon",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--pokemon_bag.pokemon_info",
        help="List with the info to show for each pokemon",
        type=bool,
        default=[]
    )
    add_config(
        parser,
        load,
        long_flag="--alt_min",
        help="Minimum random altitude",
        type=float,
        default=500
    )
    add_config(
        parser,
        load,
        long_flag="--alt_max",
        help="Maximum random altitude",
        type=float,
        default=1000
    )
    add_config(
        parser,
        load,
        long_flag="--replicate_gps_xy_noise",
        help="Add noise to current position",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--replicate_gps_z_noise",
        help="Add noise to current position",
        type=bool,
        default=False
    )
    add_config(
        parser,
        load,
        long_flag="--gps_xy_noise_range",
        help="Intensity of gps noise (unit is lat and lng,) high values may cause issues (default=0.000125)",
        type=float,
        default=0.000125
    )
    add_config(
        parser,
        load,
        long_flag="--gps_z_noise_range",
        help="Intensity of gps noise (unit is in meter, default=12.5)",
        type=float,
        default=12.5
    )
    add_config(
        parser,
        load,
        long_flag="--gps_default_altitude",
        help="Initial altitude (default=8.0)",
        type=float,
        default=8.0
    )
    add_config(
         parser,
         load,
         long_flag="--enable_social",
         help="Enable social event exchange between bot",
         type=bool,
         default=True
    )

    add_config(
         parser,
         load,
         long_flag="--walker_limit_output",
         help="Limit output from walker functions (move_to_fort, position_update, etc)",
         type=bool,
         default=False
    )

    # Start to parse other attrs
    config = parser.parse_args()
    if not config.username and 'username' not in load:
        config.username = raw_input("Username: ")
    if not config.password and 'password' not in load:
        config.password = getpass("Password: ")

    config.favorite_locations = load.get('favorite_locations', [])
    config.encrypt_location = load.get('encrypt_location', '')
    config.telegram_token = load.get('telegram_token', '')
    config.catch = load.get('catch', {})
    config.release = load.get('release', {})
    config.plugins = load.get('plugins', [])
    config.raw_tasks = load.get('tasks', [])
    config.vips = load.get('vips', {})
    config.sleep_schedule = load.get('sleep_schedule', [])
    config.live_config_update = load.get('live_config_update', {})
    config.live_config_update_enabled = config.live_config_update.get('enabled', False)
    config.live_config_update_tasks_only = config.live_config_update.get('tasks_only', False)
    config.logging = load.get('logging', {})

    if config.map_object_cache_time < 0.0:
        parser.error("--map_object_cache_time is out of range! (should be >= 0.0)")
        return None

    if len(config.raw_tasks) == 0:
        logging.error("No tasks are configured. Did you mean to configure some behaviors? Read https://github.com/PokemonGoF/PokemonGo-Bot/wiki/Configuration-files#configuring-tasks for more information")
        return None

    if config.auth_service not in ['ptc', 'google']:
        logging.error("Invalid Auth service specified! ('ptc' or 'google')")
        return None

    def task_configuration_error(flag_name):
        parser.error("""
            \"{}\" was removed from the configuration options.
            You can now change the behavior of the bot by modifying the \"tasks\" key.
            Read https://github.com/PokemonGoF/PokemonGo-Bot/wiki/Configuration-files#configuring-tasks for more information.
            """.format(flag_name))

    old_flags = ['mode', 'catch_pokemon', 'spin_forts', 'forts_spin', 'hatch_eggs', 'release_pokemon', 'softban_fix',
                 'longer_eggs_first', 'evolve_speed', 'use_lucky_egg', 'item_filter', 'evolve_all', 'evolve_cp_min',
                 'max_steps', 'catch_throw_parameters.excellent_rate', 'catch_throw_parameters.great_rate',
                 'catch_throw_parameters.nice_rate', 'catch_throw_parameters.normal_rate',
                 'catch_throw_parameters.spin_success_rate']
    for flag in old_flags:
        if flag in load:
            task_configuration_error(flag)
            return None

    nested_old_flags = [('forts', 'spin'), ('forts', 'move_to_spin'), ('navigator', 'path_mode'), ('navigator', 'path_file'), ('navigator', 'type')]
    for outer, inner in nested_old_flags:
        if load.get(outer, {}).get(inner, None):
            task_configuration_error('{}.{}'.format(outer, inner))
            return None

    if "evolve_captured" in load:
        logger.warning('The evolve_captured argument is no longer supported. Please use the EvolvePokemon task instead')

    if "walk" in load:
        logger.warning('The walk argument is no longer supported. Please use the walk_max and walk_min variables instead')

    if "daily_catch_limit" in load:
        logger.warning('The daily_catch_limit argument has been moved into the CatchPokemon Task')

    if "logging_color" in load:
        logger.warning('The logging_color argument has been moved into the logging config section')

    if config.walk_min < 1:
        parser.error("--walk_min is out of range! (should be >= 1.0)")
        return None

    if config.alt_min < -413.0:
        parser.error("--alt_min is out of range! (should be >= -413.0)")
        return None

    if not (config.location or config.location_cache):
        parser.error("Needs either --use-location-cache or --location.")
        return None

    plugin_loader = PluginLoader()
    for plugin in config.plugins:
        plugin_loader.load_plugin(plugin)

    # create web dir if not exists
    try:
        os.makedirs(web_dir)
    except OSError:
        if not os.path.isdir(web_dir):
            raise

    fix_nested_config(config)
    return config, config_file

Example 28

Project: PokemonGo-Bot-Backup
Source File: __init__.py
View license
    def _set_starting_position(self):

        self.event_manager.emit(
            'set_start_location',
            sender=self,
            level='info',
            formatted='Setting start location.'
        )

        has_position = False

        if self.config.test:
            # TODO: Add unit tests
            return

        if self.config.location:
            location_str = self.config.location
            location = self.get_pos_by_name(location_str.replace(" ", ""))
            msg = "Location found: {location} {position}"
            self.event_manager.emit(
                'location_found',
                sender=self,
                level='info',
                formatted=msg,
                data={
                    'location': location_str,
                    'position': location
                }
            )

            self.api.set_position(*location)

            self.event_manager.emit(
                'position_update',
                sender=self,
                level='info',
                formatted="Now at {current_position}",
                data={
                    'current_position': self.position,
                    'last_position': '',
                    'distance': '',
                    'distance_unit': ''
                }
            )

            self.start_position = self.position

            has_position = True

        if self.config.location_cache:
            try:
                # save location flag used to pull the last known location from
                # the location.json
                self.event_manager.emit(
                    'load_cached_location',
                    sender=self,
                    level='debug',
                    formatted='Loading cached location...'
                )

                json_file = os.path.join(_base_dir, 'data', 'last-location-%s.json' % self.config.username)

                try:
                    with open(json_file, "r") as infile:
                        location_json = json.load(infile)
                except (IOError, ValueError):
                    # Unable to read json file.
                    # File may be corrupt. Create a new one.
                    location_json = []
                except:
                    raise FileIOException("Unexpected error reading from {}".web_inventory)

                location = (
                    location_json['lat'],
                    location_json['lng'],
                    location_json['alt'],
                )

                # If location has been set in config, only use cache if starting position has not differed
                if has_position and 'start_position' in location_json:
                    last_start_position = tuple(location_json.get('start_position', []))

                    # Start position has to have been set on a previous run to do this check
                    if last_start_position and last_start_position != self.start_position:
                        msg = 'Going to a new place, ignoring cached location.'
                        self.event_manager.emit(
                            'location_cache_ignored',
                            sender=self,
                            level='debug',
                            formatted=msg
                        )
                        return

                self.api.set_position(*location)
                self.event_manager.emit(
                    'position_update',
                    sender=self,
                    level='debug',
                    formatted='Loaded location {current_position} from cache',
                    data={
                        'current_position': location,
                        'last_position': '',
                        'distance': '',
                        'distance_unit': ''
                    }
                )

                has_position = True
            except Exception:
                if has_position is False:
                    sys.exit(
                        "No cached Location. Please specify initial location."
                    )
                self.event_manager.emit(
                    'location_cache_error',
                    sender=self,
                    level='debug',
                    formatted='Parsing cached location failed.'
                )

Example 29

Project: pypot
Source File: __init__.py
View license
def from_vrep(config, vrep_host='127.0.0.1', vrep_port=19997, scene=None,
              tracked_objects=[], tracked_collisions=[]):
    """ Create a robot from a V-REP instance.

    :param config: robot configuration (either the path to the json or directly the dictionary)
    :type config: str or dict
    :param str vrep_host: host of the V-REP server
    :param int vrep_port: port of the V-REP server
    :param str scene: path to the V-REP scene to load and start
    :param list tracked_objects: list of V-REP dummy object to track
    :param list tracked_collisions: list of V-REP collision to track

    This function tries to connect to a V-REP instance and expects to find motors with names corresponding as the ones found in the config.

    .. note:: The :class:`~pypot.robot.robot.Robot` returned will also provide a convenience reset_simulation method which resets the simulation and the robot position to its intial stance.

    .. note:: Using the same configuration, you should be able to switch from a real to a simulated robot just by switching from :func:`~pypot.robot.config.from_config` to :func:`~pypot.vrep.from_vrep`.
        For instance::

            import json

            with open('my_config.json') as f:
                config = json.load(f)

            from pypot.robot import from_config
            from pypot.vrep import from_vrep

            real_robot = from_config(config)
            simulated_robot = from_vrep(config, '127.0.0.1', 19997, 'poppy.ttt')

    """
    vrep_io = VrepIO(vrep_host, vrep_port)

    vreptime = vrep_time(vrep_io)
    pypot_time.time = vreptime.get_time
    pypot_time.sleep = vreptime.sleep

    if isinstance(config, basestring):
        with open(config) as f:
            config = json.load(f, object_pairs_hook=OrderedDict)

    motors = [motor_from_confignode(config, name)
              for name in config['motors'].keys()]

    vc = VrepController(vrep_io, scene, motors)
    vc._init_vrep_streaming()

    sensor_controllers = []

    if tracked_objects:
        sensors = [ObjectTracker(name) for name in tracked_objects]
        vot = VrepObjectTracker(vrep_io, sensors)
        sensor_controllers.append(vot)

    if tracked_collisions:
        sensors = [VrepCollisionDetector(name) for name in tracked_collisions]
        vct = VrepCollisionTracker(vrep_io, sensors)
        sensor_controllers.append(vct)

    robot = Robot(motor_controllers=[vc],
                  sensor_controllers=sensor_controllers)

    for m in robot.motors:
        m.goto_behavior = 'minjerk'

    init_pos = {m: m.goal_position for m in robot.motors}

    make_alias(config, robot)

    def start_simu():
        vrep_io.start_simulation()

        for m, p in init_pos.iteritems():
            m.goal_position = p

        vc.start()

        if tracked_objects:
            vot.start()

        if tracked_collisions:
            vct.start()

        while vrep_io.get_simulation_current_time() < 1.:
            sys_time.sleep(0.1)

    def stop_simu():
        if tracked_objects:
            vot.stop()

        if tracked_collisions:
            vct.stop()

        vc.stop()
        vrep_io.stop_simulation()

    def reset_simu():
        stop_simu()
        sys_time.sleep(0.5)
        start_simu()

    robot.start_simulation = start_simu
    robot.stop_simulation = stop_simu
    robot.reset_simulation = reset_simu

    def current_simulation_time(robot):
        return robot._controllers[0].io.get_simulation_current_time()
    Robot.current_simulation_time = property(lambda robot: current_simulation_time(robot))

    def get_object_position(robot, object, relative_to_object=None):
        return vrep_io.get_object_position(object, relative_to_object)
    Robot.get_object_position = partial(get_object_position, robot)

    def get_object_orientation(robot, object, relative_to_object=None):
        return vrep_io.get_object_orientation(object, relative_to_object)
    Robot.get_object_orientation = partial(get_object_orientation, robot)

    return robot

Example 30

Project: arkc-client
Source File: main.py
View license
def main():
    global Mode
    parser = argparse.ArgumentParser(description=None)
    try:
        # Load arguments
        parser.add_argument(
            "-v", dest="v", action="store_true", help="show detailed logs")
        parser.add_argument(
            "-vv", dest="vv", action="store_true", help="show debug logs")
        # TODO: use native function
        parser.add_argument(
            "--version", dest="version", action="store_true", help="show version number")
        parser.add_argument('-kg', '--keygen', dest="kg", action='store_true',
                            help="Generate a key string and quit, overriding other options.")
        parser.add_argument('--kg-path', '--keygen-path', dest="kg_save_path",
                            help="Where to store a key string, if not set, use default.")
        parser.add_argument('-reg', '--keygen-email-register', dest="email_dest",
                            help="Email destination to register the key.")
        parser.add_argument('--get-meek', dest="dlmeek", action="store_true",
                            help="Download meek to home directory, overriding normal options")
        parser.add_argument('-c', '--config', dest="config", default=None,
                            help="Specify a configuration files, REQUIRED for ArkC Client to start")
        parser.add_argument('-g', '--gae', dest="gae", action='store_true',
                            help="Use GAE mode")
        parser.add_argument('-fs', '--frequent-swap', dest="fs", action="store_true",
                            help="Use frequent connection swapping")
        parser.add_argument('-pn', '--public-addr', dest="pn", action="store_true",
                            help="Disable UPnP when you have public network IP address (or NAT has been manually configured)")

        parser.add_argument("-v6", dest="ipv6", default="",
                            help="Enable this option to use IPv6 address (only use it if you have one)")
        print("""ArkC Client V""" + VERSION + """,  by ArkC Technology.
The programs is distributed under GNU General Public License Version 2.
""")

        options = parser.parse_args()

        if options.vv:
            logging.basicConfig(
                stream=sys.stdout, level=logging.DEBUG, format="%(levelname)s: %(asctime)s; %(message)s")
        elif options.v:
            logging.basicConfig(
                stream=sys.stdout, level=logging.INFO, format="%(levelname)s: %(asctime)s; %(message)s")
        else:
            logging.basicConfig(
                stream=sys.stdout, level=logging.WARNING, format="%(levelname)s: %(asctime)s; %(message)s")

        if options.gae:
            Mode = "GAE"
            logging.info("Using GAE mode.")
        else:
            Mode = "VPS"
            logging.info("Using VPS mode.")

        if options.version:
            print("ArkC Client Version " + VERSION)
            sys.exit()
        elif options.kg:
            genkey(options)
        elif options.dlmeek:
            dlmeek()
        elif options.config is None:
            logging.fatal("Config file (-c or --config) must be specified.\n")
            parser.print_help()
            sys.exit()

        data = {}

        # Load json configuration file
        try:
            data_file = open(options.config)
            data = json.load(data_file)
            data_file.close()
        except Exception as err:
            logging.fatal(
                "Fatal error while loading configuration file.\n" + err)
            sys.exit()

        if "control_domain" not in data:
            logging.fatal("missing control domain")
            sys.exit()

        # Apply default values
        if "local_host" not in data:
            data["local_host"] = DEFAULT_LOCAL_HOST

        if "local_port" not in data:
            data["local_port"] = DEFAULT_LOCAL_PORT

        if "remote_host" not in data:
            data["remote_host"] = DEFAULT_REMOTE_HOST

        if "remote_port" not in data:
            data["remote_port"] = random.randint(20000, 60000)
            logging.info(
                "Using random port " + str(data["remote_port"]) + " as remote listening port")

        if "number" not in data:
            data["number"] = DEFAULT_REQUIRED
        elif data["number"] > 20:
            logging.warning(
                "Requesting " + str(data["number"]) + " connections. Note: most servers impose a limit of 20. You may not receive response at all.")

        if data["number"] > 100:
            data["number"] = 100

        if "dns_servers" not in data:
            if "dns_server" in data:
                data["dns_servers"] = data["dns_server"]
            else:
                data["dns_servers"] = DEFAULT_DNS_SERVERS

        if "pt_exec" not in data:
            data["pt_exec"] = DEFAULT_OBFS4_EXECADDR

        if "debug_ip" not in data:
            data["debug_ip"] = None

        if Mode == "VPS":
            if "obfs_level" not in data:
                data["obfs_level"] = 0
            elif 1 <= int(data["obfs_level"]) <= 2:
                logging.error(
                    "Support for obfs4proxy is experimental with known bugs. Run this mode at your own risk.")
        else:
            data["obfs_level"] = 3

        # Load certificates
        try:
            serverpub_data = open(data["remote_cert"], "r").read()
            serverpub = certloader(serverpub_data).importKey()
        except KeyError as e:
            logging.fatal(
                e.tostring() + "is not found in the config file. Quitting.")
            sys.exit()
        except Exception as err:
            print ("Fatal error while loading remote host certificate.")
            print (err)
            sys.exit()

        try:
            clientpri_data = open(data["local_cert"], "r").read()
            clientpri_data = clientpri_data.strip(' ').lstrip('\n')
            clientpri = certloader(clientpri_data).importKey()
            clientpri_sha1 = certloader(clientpri_data).getSHA1()
            print("Using private key with SHA1: " + clientpri_sha1 +
                  ". Please make sure it is identical the string in server-side config.")
            if not clientpri.has_private():
                print(
                    "Fatal error, no private key included in local certificate.")
        except KeyError as e:
            logging.fatal(
                e.tostring() + "is not found in the config file. Quitting.")
            sys.exit()
        except Exception as err:
            print ("Fatal error while loading local certificate.")
            print (err)
            sys.exit()

        try:
            clientpub_data = open(data["local_cert_pub"], "r").read()
            clientpub_data = clientpub_data.strip(' ').lstrip('\n')
            clientpub_sha1 = certloader(clientpub_data).getSHA1()
        except KeyError as e:
            logging.fatal(
                e.tostring() + "is not found in the config file. Quitting.")
            sys.exit()
        except Exception as err:
            print ("Fatal error while calculating SHA1 digest.")
            print (err)
            sys.exit()

        # TODO: make it more elegant

        if options.fs:
            swapfq = 3
        else:
            swapfq = 8

    except IOError as e:
        print ("An error occurred: \n")
        print(e)

    # Start the main event loop

    try:
        ctl = Coordinate(
            data["control_domain"],
            clientpri,
            clientpri_sha1,
            serverpub,
            clientpub_sha1,
            data["number"],
            data["remote_host"],
            data["remote_port"],
            data["dns_servers"],
            data["debug_ip"],
            swapfq,
            data["pt_exec"],
            data["obfs_level"],
            options.ipv6,
            options.pn
        )
        sctl = ServerControl(
            data["remote_host"],
            ctl.remote_port,
            ctl,
            pt=bool(data["obfs_level"])
        )
        cctl = ClientControl(
            ctl,
            data["local_host"],
            data["local_port"]
        )
    except KeyError as e:
        print(e)
        logging.fatal("Bad config file. Quitting.")
        sys.exit()

    except Exception as e:
        print ("An error occurred: \n")
        print(e)

    logging.info("Listening to local services at " +
                 data["local_host"] + ":" + str(data["local_port"]))
    logging.info("Listening to remote server at " +
                 data["remote_host"] + ":" + str(ctl.remote_port))

    try:
        asyncore.loop(use_poll=1)
    except KeyboardInterrupt:
        pass

Example 31

Project: arkc-server
Source File: main.py
View license
def main():
    parser = argparse.ArgumentParser(description=None)
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="show detailed logs")
    parser.add_argument("-vv", action="store_true", dest="vv",
                        help="show debug logs")
    parser.add_argument(
        "--version", dest="version", action="store_true", help="show version number")
    parser.add_argument('-kg', '--keygen', dest="kg", action="store_true",
                        help="Generate a key string and quit, overriding other options")
    parser.add_argument('--get-meek', dest="dlmeek", action="store_true",
                        help="Download meek to home directory, overriding normal options")
    parser.add_argument('-c', '--config', dest="config", default=None,
                        help="specify a configuration files, required for ArkC to start")
    parser.add_argument("-t", action="store_true", dest="transmit",
                        help="use transmit server")

    parser.add_argument('-ep', "--use-external-proxy", action="store_true",
                        help="""use an external proxy server or handler running locally,e.g. polipo, for better performance.
Use this option to support other types of proxy other than HTTP, or use authentication at client-end proxy.
Fall back to in-built python proxy server otherwise.""")
    print(
        """ArkC Server V""" + VERSION + """ by ArkC Technology.
The programs is distributed under GNU General Public License Version 2.
""")

    args = parser.parse_args()
    if args.version:
        print("ArkC Server Version " + VERSION)
        sys.exit()
    elif args.kg:
        print("Generating 2048 bit RSA key.")
        print("Writing to home directory " + os.path.expanduser('~'))
        generate_RSA(os.path.expanduser(
            '~') + os.sep + 'arkc_pri.asc', os.path.expanduser('~') + os.sep + 'arkc_pub.asc')
        print(
            "Please save the above settings to client and server side config files.")
        sys.exit()
    elif args.dlmeek:
        if sys.platform == 'linux2':
            link = "https://github.com/projectarkc/meek/releases/download/v0.2.2/meek-client"
            localfile = os.path.expanduser('~') + os.sep + "meek-client"
        elif sys.platform == 'win32':
            link = "https://github.com/projectarkc/meek/releases/download/v0.2.2/meek-client.exe"
            localfile = os.path.expanduser('~') + os.sep + "meek-client.exe"
        else:
            print(
                "MEEK for ArkC has no compiled executable for your OS platform. Please compile and install from source.")
            print(
                "Get source at https://github.com/projectarkc/meek/tree/master/meek-client")
            sys.exit()
        print(
            "Downloading meek plugin (meek-client) from github to " + localfile)
        urllib.urlretrieve(link, localfile)
        if sys.platform == 'linux2':
            st = os.stat(localfile)
            os.chmod(localfile, st.st_mode | stat.S_IEXEC)
            print("File made executable.")
        print("Finished. If no error, you may change obfs_level and update pt_exec to " +
              localfile + " to use meek.")
        sys.exit()
    elif args.config is None:
        logging.fatal("Config file (-c or --config) must be specified.\n")
        parser.print_help()
        sys.exit()

    # mapping client public sha1 --> (RSA key object, client private sha1)
    certs = dict()

    data = {}

    # Load json configuration file
    try:
        data_file = open(args.config)
        data = json.load(data_file)
        data_file.close()
    except Exception as err:
        logging.error("Fatal error while loading configuration file.")
        print(err)  # TODO: improve error processing
        sys.exit()

    try:
        for client in data["clients"]:
            with open(client[0], "r") as f:
                remote_cert_txt = f.read()
                remote_cert = RSA.importKey(remote_cert_txt)
                remote_cert_txt = remote_cert_txt.strip(
                    ' ').lstrip('\n')
                certs[sha1(remote_cert_txt).hexdigest()] =\
                     [remote_cert, client[1]]
    except KeyError:
        pass
    except Exception as err:
        print ("Fatal error while loading client certificate.")
        print (err)
        sys.exit()

    try:
        certsdbpath = data["clients_db"]
    except KeyError:
        certsdbpath = None

    try:
        certs_db = certstorage(certs, certsdbpath)
    except Exception as err:
        print ("Fatal error while loading clients' certificate.")
        print (err)
        sys.exit()

    if args.transmit:
        try:
            with open(data["central_cert"], "r") as f:
                central_cert_txt = f.read()
                central_cert = RSA.importKey(central_cert_txt)
        except Exception as err:
            print ("Fatal error while loading client certificate.")
            print (err)
            sys.exit()
    else:
        central_cert = None

    try:
        with open(data["local_cert_path"], "r") as f:
            local_cert = RSA.importKey(f.read())
        if not local_cert.has_private():
            print("Fatal error, no private key included in local certificate.")
    except IOError as err:
        print ("Fatal error while loading local certificate.")
        print (err)
        sys.exit()

    if args.vv:
        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG,
                            format="%(levelname)s: %(asctime)s; %(message)s")
    elif args.verbose:
        logging.basicConfig(stream=sys.stdout, level=logging.INFO,
                            format="%(levelname)s: %(asctime)s; %(message)s")
    else:
        logging.basicConfig(stream=sys.stdout, level=logging.WARNING,
                            format="%(levelname)s: %(asctime)s; %(message)s")

    if not args.use_external_proxy:
        if "proxy_port" not in data:
            data["proxy_port"] = 8100
        start_proxy(data["proxy_port"])
    else:
        if "proxy_port" not in data:
            data["proxy_port"] = 8123

    if "udp_port" not in data:
        if args.transmit:
            data["udp_port"] = 8000
        else:
            data["udp_port"] = 53

    if "socks_proxy" not in data:
        data["socks_proxy"] = None

    if "delegated_domain" not in data:
        data["delegated_domain"] = "public.arkc.org"

    if "self_domain" not in data:
        data["self_domain"] = "freedom.arkc.org"

    if "pt_exec" not in data:
        data["pt_exec"] = "obfs4proxy"

    if "obfs_level" not in data:
        data["obfs_level"] = 0
    elif 1 <= int(data["obfs_level"]) <= 2:
        logging.error(
            "Support for obfs4proxy is experimental with known bugs. Run this mode at your own risk.")

    if "meek_url" not in data:
        data["meek_url"] = "https://arkc-reflect1.appspot.com/"

    # Start the loop
    try:
        reactor.listenUDP(
            data["udp_port"],
            Coordinator(
                data["proxy_port"],
                data["socks_proxy"],
                local_cert,
                certs_db,
                central_cert,
                data["delegated_domain"],
                data["self_domain"],
                data["pt_exec"],
                data["obfs_level"],
                data["meek_url"],
                args.transmit
            )
        )
    except CannotListenError as err:
        print(err.socketError)
        if data["udp_port"] <= 1024 and str(err.socketError) == "[Errno 13] \
                Permission denied":
            print("root privilege may be required to listen to low ports")
        exit()

    try:
        reactor.run()
    except KeyboardInterrupt:
        pass

Example 32

Project: PyPXE
Source File: server.py
View license
def main():
    global SETTINGS, args
    try:
        # warn the user that they are starting PyPXE as non-root user
        if os.getuid() != 0:
            print '\nWARNING: Not root. Servers will probably fail to bind.\n'

        # configure
        args = parse_cli_arguments()
        if args.JSON_CONFIG: # load from configuration file if specified
            try:
                config_file = open(args.JSON_CONFIG, 'rb')
            except IOError:
                sys.exit('Failed to open {0}'.format(args.JSON_CONFIG))
            try:
                loaded_config = json.load(config_file)
                config_file.close()
            except ValueError:
                sys.exit('{0} does not contain valid JSON'.format(args.JSON_CONFIG))
            for setting in loaded_config:
                if type(loaded_config[setting]) is unicode:
                    loaded_config[setting] = loaded_config[setting].encode('ascii')
            SETTINGS.update(loaded_config) # update settings with JSON config
            args = parse_cli_arguments() # re-parse, CLI options take precedence

        # ideally this would be in dhcp itself, but the chroot below *probably*
        # breaks the ability to open the config file.
        if args.STATIC_CONFIG:
            try:
                static_config = open(args.STATIC_CONFIG, 'rb')
            except IOError:
                sys.exit("Failed to open {0}".format(args.STATIC_CONFIG))
            try:
                loaded_statics = json.load(static_config)
                static_config.close()
            except ValueError:
                sys.exit("{0} does not contain valid json".format(args.STATIC_CONFIG))
        else:
            loaded_statics = dict()

        # setup main logger
        sys_logger = logging.getLogger('PyPXE')
        if args.SYSLOG_SERVER:
            handler = logging.handlers.SysLogHandler(address = (args.SYSLOG_SERVER, int(args.SYSLOG_PORT)))
        else:
            handler = logging.StreamHandler()
        formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s %(message)s')
        handler.setFormatter(formatter)
        sys_logger.addHandler(handler)
        sys_logger.setLevel(logging.INFO)

        # pass warning to user regarding starting HTTP server without iPXE
        if args.USE_HTTP and not args.USE_IPXE and not args.USE_DHCP:
            sys_logger.warning('HTTP selected but iPXE disabled. PXE ROM must support HTTP requests.')

        # if the argument was pased to enabled ProxyDHCP then enable the DHCP server
        if args.DHCP_MODE_PROXY:
            args.USE_DHCP = True

        # if the network boot file name was not specified in the argument,
        # set it based on what services were enabled/disabled
        if args.NETBOOT_FILE == '':
            if not args.USE_IPXE:
                args.NETBOOT_FILE = 'pxelinux.0'
            elif not args.USE_HTTP:
                args.NETBOOT_FILE = 'boot.ipxe'
            else:
                args.NETBOOT_FILE = 'boot.http.ipxe'

        if args.NBD_WRITE and not args.NBD_COW:
            sys_logger.warning('NBD Write enabled but copy-on-write is not. Multiple clients may cause corruption')

        if args.NBD_COW_IN_MEM or args.NBD_COPY_TO_RAM:
            sys_logger.warning('NBD cowinmem and copytoram can cause high RAM usage')

        if args.NBD_COW and not args.NBD_WRITE:
            # cow implies write
            args.NBD_WRITE = True

        # make a list of running threads for each service
        running_services = []

        # configure/start TFTP server
        if args.USE_TFTP:

            # setup TFTP logger
            tftp_logger = sys_logger.getChild('TFTP')
            sys_logger.info('Starting TFTP server...')

            # setup the thread
            tftp_server = tftp.TFTPD(mode_debug = do_debug('tftp'), mode_verbose = do_verbose('tftp'), logger = tftp_logger, netboot_directory = args.NETBOOT_DIR)
            tftpd = threading.Thread(target = tftp_server.listen)
            tftpd.daemon = True
            tftpd.start()
            running_services.append(tftpd)

        # configure/start DHCP server
        if args.USE_DHCP:

            # setup DHCP logger
            dhcp_logger = sys_logger.getChild('DHCP')
            if args.DHCP_MODE_PROXY:
                sys_logger.info('Starting DHCP server in ProxyDHCP mode...')
            else:
                sys_logger.info('Starting DHCP server...')

            # setup the thread
            dhcp_server = dhcp.DHCPD(
                ip = args.DHCP_SERVER_IP,
                port = args.DHCP_SERVER_PORT,
                offer_from = args.DHCP_OFFER_BEGIN,
                offer_to = args.DHCP_OFFER_END,
                subnet_mask = args.DHCP_SUBNET,
                router = args.DHCP_ROUTER,
                dns_server = args.DHCP_DNS,
                broadcast = args.DHCP_BROADCAST,
                file_server = args.DHCP_FILESERVER,
                file_name = args.NETBOOT_FILE,
                use_ipxe = args.USE_IPXE,
                use_http = args.USE_HTTP,
                mode_proxy = args.DHCP_MODE_PROXY,
                mode_debug = do_debug('dhcp'),
                mode_verbose = do_verbose('dhcp'),
                whitelist = args.DHCP_WHITELIST,
                static_config = loaded_statics,
                logger = dhcp_logger,
                saveleases = args.LEASES_FILE)
            dhcpd = threading.Thread(target = dhcp_server.listen)
            dhcpd.daemon = True
            dhcpd.start()
            running_services.append(dhcpd)

        # configure/start HTTP server
        if args.USE_HTTP:

            # setup HTTP logger
            http_logger = sys_logger.getChild('HTTP')
            sys_logger.info('Starting HTTP server...')

            # setup the thread
            http_server = http.HTTPD(mode_debug = do_debug('http'), mode_verbose = do_debug('http'), logger = http_logger, netboot_directory = args.NETBOOT_DIR)
            httpd = threading.Thread(target = http_server.listen)
            httpd.daemon = True
            httpd.start()
            running_services.append(httpd)

        # configure/start NBD server
        if args.NBD_BLOCK_DEVICE:
            # setup NBD logger
            nbd_logger = sys_logger.getChild('NBD')
            sys_logger.info('Starting NBD server...')
            nbd_server = nbd.NBD(
                block_device = args.NBD_BLOCK_DEVICE,
                write = args.NBD_WRITE,
                cow = args.NBD_COW,
                in_mem = args.NBD_COW_IN_MEM,
                copy_to_ram = args.NBD_COPY_TO_RAM,
                ip = args.NBD_SERVER_IP,
                port = args.NBD_PORT,
                mode_debug = do_debug('nbd'),
                mode_verbose = do_verbose('nbd'),
                logger = nbd_logger,
                netboot_directory = args.NETBOOT_DIR)
            nbdd = threading.Thread(target = nbd_server.listen)
            nbdd.daemon = True
            nbdd.start()
            running_services.append(nbdd)

        sys_logger.info('PyPXE successfully initialized and running!')

        while map(lambda x: x.isAlive(), running_services):
            sleep(1)

    except KeyboardInterrupt:
        sys.exit('\nShutting down PyPXE...\n')

Example 33

Project: Pyfa
Source File: jsonToSql.py
View license
def main(db, json_path):

    jsonPath = os.path.expanduser(json_path)

    # Import eos.config first and change it
    import eos.config
    eos.config.gamedata_connectionstring = db
    eos.config.debug = False

    # Now thats done, we can import the eos modules using the config
    import eos.db
    import eos.gamedata

    # Create the database tables
    eos.db.gamedata_meta.create_all()

    # Config dict
    tables = {
        "dgmattribs": eos.gamedata.AttributeInfo,
        "dgmeffects": eos.gamedata.EffectInfo,
        "dgmtypeattribs": eos.gamedata.Attribute,
        "dgmtypeeffects": eos.gamedata.Effect,
        "dgmunits": eos.gamedata.Unit,
        "icons": eos.gamedata.Icon,
        "evecategories": eos.gamedata.Category,
        "evegroups": eos.gamedata.Group,
        "invmetagroups": eos.gamedata.MetaGroup,
        "invmetatypes": eos.gamedata.MetaType,
        "evetypes": eos.gamedata.Item,
        "phbtraits": eos.gamedata.Traits,
        "phbmetadata": eos.gamedata.MetaData,
        "marketProxy()_GetMarketGroups()": eos.gamedata.MarketGroup
    }

    fieldMapping = {
        "dgmattribs": {
            "displayName_en-us": "displayName"
        },
        "dgmeffects": {
            "displayName_en-us": "displayName",
            "description_en-us": "description"
        },
        "dgmunits": {
            "displayName_en-us": "displayName"
        },
        #icons???
        "evecategories": {
            "categoryName_en-us": "categoryName"
        },
        "evegroups": {
            "groupName_en-us": "groupName"
        },
        "invmetagroups": {
            "metaGroupName_en-us": "metaGroupName"
        },
        "evetypes": {
            "typeName_en-us": "typeName",
            "description_en-us": "description"
        },
        #phbtraits???
        "marketProxy()_GetMarketGroups()": {
            "marketGroupName_en-us": "marketGroupName",
            "description_en-us": "description"
        }

    }

    rowsInValues = (
        "evetypes",
        "evegroups",
        "evecategories"
    )

    def convertIcons(data):
        new = []
        for k, v in data.items():
            v["iconID"] = k
            new.append(v)
        return new

    def convertTraits(data):

        def convertSection(sectionData):
            sectionLines = []
            headerText = u"<b>{}</b>".format(sectionData["header"])
            sectionLines.append(headerText)
            for bonusData in sectionData["bonuses"]:
                prefix = u"{} ".format(bonusData["number"]) if "number" in bonusData else ""
                bonusText = u"{}{}".format(prefix, bonusData["text"].replace(u"\u00B7", u"\u2022 "))
                sectionLines.append(bonusText)
            sectionLine = u"<br />\n".join(sectionLines)
            return sectionLine

        newData = []
        for row in data:
            typeLines = []
            typeId = row["typeID"]
            traitData = row["traits_en-us"]
            for skillData in sorted(traitData.get("skills", ()), key=lambda i: i["header"]):
                typeLines.append(convertSection(skillData))
            if "role" in traitData:
                typeLines.append(convertSection(traitData["role"]))
            if "misc" in traitData:
                typeLines.append(convertSection(traitData["misc"]))
            traitLine = u"<br />\n<br />\n".join(typeLines)
            newRow = {"typeID": typeId, "traitText": traitLine}
            newData.append(newRow)
        return newData

    def convertTypes(typesData):
        """
        Add factionID column to evetypes table.
        """
        factionMap = {}
        with open(os.path.join(jsonPath, "fsdTypeOverrides.json")) as f:
            overridesData = json.load(f)
        for typeID, typeData in overridesData.items():
            factionID = typeData.get("factionID")
            if factionID is not None:
                factionMap[int(typeID)] = factionID
        for row in typesData:
            row['factionID'] = factionMap.get(int(row['typeID']))
        return typesData

    data = {}

    # Dump all data to memory so we can easely cross check ignored rows
    for jsonName, cls in tables.iteritems():
        with open(os.path.join(jsonPath, "{}.json".format(jsonName))) as f:
            tableData = json.load(f)
        if jsonName in rowsInValues:
            tableData = list(tableData.values())
        if jsonName == "icons":
            tableData = convertIcons(tableData)
        if jsonName == "phbtraits":
            tableData = convertTraits(tableData)
        if jsonName == "evetypes":
            tableData = convertTypes(tableData)
        data[jsonName] = tableData

    # Set with typeIDs which we will have in our database
    # Sometimes CCP unpublishes some items we want to have published, we
    # can do it here - just add them to initial set
    eveTypes = set()
    for row in data["evetypes"]:
        # 1306 - group Ship Modifiers, for items like tactical t3 ship modes
        # (3638, 3634, 3636, 3640) - Civilian weapons
        # (41549, 41548, 41551, 41550) - Micro Bombs (Fighters)
        if (row["published"] or row['groupID'] == 1306
            or row['typeID'] in (3638, 3634, 3636, 3640)
            or row['typeID'] in (41549, 41548, 41551,41550)):
            eveTypes.add(row["typeID"])

    # ignore checker
    def isIgnored(file, row):
        if file in ("evetypes", "dgmtypeeffects", "dgmtypeattribs", "invmetatypes") and row['typeID'] not in eveTypes:
            return True
        return False

    # Loop through each json file and write it away, checking ignored rows
    for jsonName, table in data.iteritems():
        fieldMap = fieldMapping.get(jsonName, {})
        print "processing {}".format(jsonName)
        for row in table:
            # We don't care about some kind of rows, filter it out if so
            if not isIgnored(jsonName, row):
                instance = tables[jsonName]()
                # fix for issue 80
                if jsonName is "icons" and "res:/ui/texture/icons/" in str(row["iconFile"]).lower():
                    row["iconFile"] = row["iconFile"].lower().replace("res:/ui/texture/icons/", "").replace(".png", "")
                    # with res:/ui... references, it points to the actual icon file (including it's size variation of #_size_#)
                    # strip this info out and get the identifying info
                    split = row['iconFile'].split('_')
                    if len(split) == 3:
                        row['iconFile'] = "{}_{}".format(split[0], split[2])
                if jsonName is "icons" and "modules/" in str(row["iconFile"]).lower():
                    row["iconFile"] = row["iconFile"].lower().replace("modules/", "").replace(".png", "")

                for k, v in row.iteritems():
                    if (isinstance(v, basestring)):
                        v = v.strip()
                    setattr(instance, fieldMap.get(k, k), v)

                eos.db.gamedata_session.add(instance)

    eos.db.gamedata_session.commit()

    print("done")

Example 34

Project: pyomo
Source File: ph_checker.py
View license
def validate_ph(options):

    history = None
    try:
        with open(options.ph_history_filename) as f:
            history = json.load(f)
    except:
        history = None
        try:
            history = shelve.open(options.ph_history_filename,
                                  flag='r')
        except:
            history = None

    if history is None:
        raise RuntimeError("Unable to open ph history file as JSON "
                           "or python Shelve DB format")

    scenario_tree = history['scenario tree']

    try:
        validate_probabilities(options,
                               scenario_tree)
    except AssertionError as e:
        msg = "FAILURE:\n"
        msg += ("Problem validating probabilities for scenario "
                "tree in ph history from file: %s"
                % (options.ph_history_filename))
        _update_exception_message(msg, e)
        raise

    try:
        iter_keys = history['results keys']
    except KeyError:
        # we are using json format (which loads the entire file
        # anyway)
        iter_keys = list(history.keys())
        iter_keys.remove('scenario tree')
    iterations = sorted(int(k) for k in iter_keys)
    iterations = [str(k) for k in iterations]

    prev_iter_results = None
    for i in iterations:

        iter_results = history[i]

        convergence = iter_results['convergence']
        scenario_solutions = iter_results['scenario solutions']
        node_solutions = iter_results['node solutions']

        try:
            validate_variable_set(options,
                                  scenario_tree,
                                  scenario_solutions,
                                  node_solutions)
        except AssertionError as e:
            msg = "FAILURE:\n"
            msg += ("Problem validating variable set for iteration %s "
                    "of ph history in file: %s"
                    % (i, options.ph_history_filename))
            _update_exception_message(msg, e)
            raise


        try:
            validate_leaf_stage_costs(options,
                                      scenario_tree,
                                      scenario_solutions,
                                      node_solutions)
        except AssertionError as e:
            msg = "FAILURE:\n"
            msg += ("Problem validating leaf-stage costs for iteration %s "
                    "of ph history in file: %s"
                    % (i, options.ph_history_filename))
            _update_exception_message(msg, e)
            raise


        try:
            validate_scenario_costs(options, scenario_tree, scenario_solutions)
        except AssertionError as e:
            msg = "FAILURE:\n"
            msg += ("Problem validating scenario costs for iteration %s "
                    "of ph history in file: %s"
                    % (i, options.ph_history_filename))
            _update_exception_message(msg, e)
            raise


        try:
            validate_scenario_costs(options, scenario_tree, scenario_solutions)
        except AssertionError as e:
            msg = "FAILURE:\n"
            msg += ("Problem validating scenario costs for iteration %s "
                    "of ph history in file: %s"
                    % (i, options.ph_history_filename))
            _update_exception_message(msg, e)
            raise

        try:
            validate_variable_statistics(options,
                                         scenario_tree,
                                         node_solutions,
                                         scenario_solutions)
        except AssertionError as e:
            msg = "FAILURE:\n"
            msg += ("Problem validating variable statistics for iteration %s "
                    "of ph history in file: %s"
                    % (i, options.ph_history_filename))
            _update_exception_message(msg, e)
            raise

        if prev_iter_results is not None:
            try:
                validate_ph_objective_parameters(
                    options,
                    scenario_tree,
                    prev_iter_results['scenario solutions'],
                    prev_iter_results['node solutions'],
                    scenario_solutions,
                    node_solutions)
            except AssertionError as e:
                msg = "FAILURE:\n"
                msg += ("Problem validating ph objective parameters "
                        "for iteration %s of ph history in file: %s"
                        % (i, options.ph_history_filename))
                _update_exception_message(msg, e)
                raise
        prev_iter = i

    return 0

Example 35

Project: pyomo
Source File: model_types.py
View license
    def validateCurrentSolution(self,**kwds):
        assert self.model is not None
        assert self.results_file is not None
        model = self.model
        suffixes = dict((suffix, getattr(model,suffix))
                        for suffix in kwds.pop('suffixes',[]))
        for suf in suffixes.values():
            assert isinstance(suf,Suffix)
            assert suf.import_enabled()
        solution = None
        error_str = ("Difference in solution for {0}.{1}:\n\tBaseline "
                     "- {2}\n\tCurrent - {3}")
        with open(self.results_file,'r') as f:
            try:
                solution = json.load(f)
            except:
                return (False,"Problem reading file "+self.results_file)
        for var in model.component_data_objects(Var):
            var_value_sol = solution[var.name]['value']
            var_value = var.value
            if not ((var_value is None) and (var_value_sol is None)):
                if ((var_value is None) ^ (var_value_sol is None)) or \
                   (abs(var_value_sol - var_value) > self.diff_tol):
                    return (False,
                            error_str.format(var.name,
                                             'value',
                                             var_value_sol,
                                             var_value))
            if not (solution[var.name]['stale'] is var.stale):
                return (False,
                        error_str.format(var.name,
                                         'stale',
                                         solution[var.name]['stale'],
                                         var.stale))
            for suffix_name, suffix in suffixes.items():
                if suffix_name in solution[var.name]:
                    if suffix.get(var) is None:
                        if not(solution[var.name][suffix_name] in \
                               solution["suffix defaults"][suffix_name]):
                            return (False,
                                    error_str.format(
                                        var.name,
                                        suffix,
                                        solution[var.name][suffix_name],
                                        "none defined"))
                    elif not abs(solution[var.name][suffix_name] - \
                                 suffix.get(var)) < self.diff_tol:
                        return (False,
                                error_str.format(
                                    var.name,
                                    suffix,
                                    solution[var.name][suffix_name],
                                    suffix.get(var)))
        for con in model.component_data_objects(Constraint):
            con_value_sol = solution[con.name]['value']
            con_value = con(exception=False)
            if not ((con_value is None) and (con_value_sol is None)):
                if ((con_value is None) ^ (con_value_sol is None)) or \
                   (abs(con_value_sol - con_value) > self.diff_tol):
                    return (False,
                            error_str.format(con.name,
                                             'value',
                                             con_value_sol,
                                             con_value))
            for suffix_name, suffix in suffixes.items():
                if suffix_name in solution[con.name]:
                    if suffix.get(con) is None:
                        if not (solution[con.name][suffix_name] in \
                                solution["suffix defaults"][suffix_name]):
                            return (False,
                                    error_str.format(
                                        con.name,
                                        suffix,
                                        solution[con.name][suffix_name],
                                        "none defined"))
                    elif not abs(solution[con.name][suffix_name] - \
                                 suffix.get(con)) < self.diff_tol:
                        return (False,
                                error_str.format(
                                    con.name,
                                    suffix,
                                    solution[con.name][suffix_name],
                                    suffix.get(con)))
        for obj in model.component_data_objects(Objective):
            obj_value_sol = solution[obj.name]['value']
            obj_value = obj(exception=False)
            if not ((obj_value is None) and (obj_value_sol is None)):
                if ((obj_value is None) ^ (obj_value_sol is None)) or \
                   (abs(obj_value_sol - obj_value) > self.diff_tol):
                    return (False,
                            error_str.format(obj.name,
                                             'value',
                                             obj_value_sol,
                                             obj_value))
            for suffix_name, suffix in suffixes.items():
                if suffix_name in solution[obj.name]:
                    if suffix.get(obj) is None:
                        if not(solution[obj.name][suffix_name] in \
                               solution["suffix defaults"][suffix_name]):
                            return (False,
                                    error_str.format(
                                        obj.name,
                                        suffix,
                                        solution[obj.name][suffix_name],
                                        "none defined"))
                    elif not abs(solution[obj.name][suffix_name] - \
                                 suffix.get(obj)) < self.diff_tol:
                        return (False,
                                error_str.format(
                                    obj.name,
                                    suffix,
                                    solution[obj.name][suffix_name],
                                    suffix.get(obj)))
        for block in model.block_data_objects():
            for suffix_name, suffix in suffixes.items():
                if (solution[block.name] is not None) and \
                   (suffix_name in solution[block.name]):
                    if suffix.get(block) is None:
                        if not(solution[block.name][suffix_name] in \
                               solution["suffix defaults"][suffix_name]):
                            return (False,
                                    error_str.format(
                                        block.name,
                                        suffix,
                                        solution[block.name][suffix_name],
                                        "none defined"))
                    elif not abs(solution[block.name][suffix_name] - \
                                 suffix.get(block)) < sefl.diff_tol:
                        return (False,
                                error_str.format(
                                    block.name,
                                    suffix,
                                    solution[block.name][suffix_name],
                                    suffix.get(block)))
        return (True,"")

Example 36

View license
    @patch('pagure.lib.notify.send_email')
    def test_dumping_reloading_ticket(self, send_email):
        """ Test dumping a ticket into a JSON blob. """
        send_email.return_value = True

        tests.create_projects(self.session)

        # Create repo
        self.gitrepo = os.path.join(self.path, 'tickets', 'test.git')
        repopath = os.path.join(self.path, 'tickets')
        os.makedirs(self.gitrepo)
        repo_obj = pygit2.init_repository(self.gitrepo, bare=True)

        repo = pagure.lib.get_project(self.session, 'test')
        # Create an issue to play with
        msg = pagure.lib.new_issue(
            session=self.session,
            repo=repo,
            title='Test issue',
            content='We should work on this',
            user='pingou',
            ticketfolder=repopath
        )
        self.assertEqual(msg.title, 'Test issue')

        # Need another two issue to test the dependencie chain
        msg = pagure.lib.new_issue(
            session=self.session,
            repo=repo,
            title='Test issue #2',
            content='Another bug',
            user='pingou',
            ticketfolder=repopath
        )
        self.assertEqual(msg.title, 'Test issue #2')
        msg = pagure.lib.new_issue(
            session=self.session,
            repo=repo,
            title='Test issue #3',
            content='That would be nice feature no?',
            user='foo',
            ticketfolder=repopath
        )
        self.assertEqual(msg.title, 'Test issue #3')

        issue = pagure.lib.search_issues(self.session, repo, issueid=1)
        issue2 = pagure.lib.search_issues(self.session, repo, issueid=2)
        issue3 = pagure.lib.search_issues(self.session, repo, issueid=3)

        # Add a couple of comment on the ticket
        msg = pagure.lib.add_issue_comment(
            session=self.session,
            issue=issue,
            comment='Hey look a comment!',
            user='foo',
            ticketfolder=repopath,
        )
        self.session.commit()
        self.assertEqual(msg, 'Comment added')
        msg = pagure.lib.add_issue_comment(
            session=self.session,
            issue=issue,
            comment='crazy right?',
            user='pingou',
            ticketfolder=repopath,
        )
        self.session.commit()
        self.assertEqual(msg, 'Comment added')
        # Assign the ticket to someone
        msg = pagure.lib.add_issue_assignee(
            session=self.session,
            issue=issue,
            assignee='pingou',
            user='pingou',
            ticketfolder=repopath,
        )
        self.session.commit()
        self.assertEqual(msg, 'Issue assigned')
        # Add a couple of tags on the ticket
        msg = pagure.lib.add_tag_obj(
            session=self.session,
            obj=issue,
            tags=[' feature ', 'future '],
            user='pingou',
            ticketfolder=repopath,
        )
        self.session.commit()
        self.assertEqual(msg, 'Tag added: feature, future')
        # Add dependencies
        msg = pagure.lib.add_issue_dependency(
            session=self.session,
            issue=issue,
            issue_blocked=issue2,
            user='pingou',
            ticketfolder=repopath,
        )
        self.session.commit()
        self.assertEqual(msg, 'Dependency added')
        msg = pagure.lib.add_issue_dependency(
            session=self.session,
            issue=issue3,
            issue_blocked=issue,
            user='foo',
            ticketfolder=repopath,
        )
        self.session.commit()
        self.assertEqual(msg, 'Dependency added')

        # Dump the JSON
        pagure.lib.git.update_git(issue, repo, repopath)
        repo = pygit2.Repository(self.gitrepo)
        cnt = len([commit
            for commit in repo.walk(
                repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL)])
        self.assertEqual(cnt, 10)

        last_commit = repo.revparse_single('HEAD')
        patch = pagure.lib.git.commit_to_patch(repo, last_commit)
        for line in patch.split('\n'):
            if line.startswith('--- a/'):
                fileid = line.split('--- a/')[1]
                break

        newpath = tempfile.mkdtemp(prefix='pagure-dump-load')
        clone_repo = pygit2.clone_repository(self.gitrepo, newpath)

        self.assertEqual(len(os.listdir(newpath)), 4)

        ticket_json = os.path.join(self.path, 'test_ticket.json')
        self.assertFalse(os.path.exists(ticket_json))
        shutil.copyfile(os.path.join(newpath, fileid), ticket_json)
        self.assertTrue(os.path.exists(ticket_json))
        jsondata = None
        with open(ticket_json) as stream:
            jsondata = json.load(stream)
        self.assertNotEqual(jsondata, None)

        shutil.rmtree(newpath)

        # Test reloading the JSON
        self.tearDown()
        self.setUp()
        tests.create_projects(self.session)

        pagure.lib.git.update_ticket_from_git(
            self.session,
            reponame='test',
            namespace=None,
            username=None,
            issue_uid='foobar',
            json_data=jsondata,
        )

        # Post loading
        repo = pagure.lib.get_project(self.session, 'test')
        self.assertEqual(len(repo.issues), 1)
        issue = pagure.lib.search_issues(self.session, repo, issueid=1)

        # Check after re-loading
        self.assertEqual(len(issue.comments), 2)
        self.assertEqual(len(issue.tags), 2)
        self.assertEqual(issue.tags_text, ['future', 'feature'])
        self.assertEqual(issue.assignee.username, 'pingou')
        self.assertEqual(issue.children, [])
        self.assertEqual(issue.parents, [])

Example 37

Project: youtube-dl
Source File: test_download.py
View license
def generator(test_case):

    def test_template(self):
        ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
        other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
        is_playlist = any(k.startswith('playlist') for k in test_case)
        test_cases = test_case.get(
            'playlist', [] if is_playlist else [test_case])

        def print_skipping(reason):
            print('Skipping %s: %s' % (test_case['name'], reason))
        if not ie.working():
            print_skipping('IE marked as not _WORKING')
            return

        for tc in test_cases:
            info_dict = tc.get('info_dict', {})
            if not (info_dict.get('id') and info_dict.get('ext')):
                raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')

        if 'skip' in test_case:
            print_skipping(test_case['skip'])
            return
        for other_ie in other_ies:
            if not other_ie.working():
                print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
                return

        params = get_params(test_case.get('params', {}))
        if is_playlist and 'playlist' not in test_case:
            params.setdefault('extract_flat', 'in_playlist')
            params.setdefault('skip_download', True)

        ydl = YoutubeDL(params, auto_init=False)
        ydl.add_default_info_extractors()
        finished_hook_called = set()

        def _hook(status):
            if status['status'] == 'finished':
                finished_hook_called.add(status['filename'])
        ydl.add_progress_hook(_hook)
        expect_warnings(ydl, test_case.get('expected_warnings', []))

        def get_tc_filename(tc):
            return ydl.prepare_filename(tc.get('info_dict', {}))

        res_dict = None

        def try_rm_tcs_files(tcs=None):
            if tcs is None:
                tcs = test_cases
            for tc in tcs:
                tc_filename = get_tc_filename(tc)
                try_rm(tc_filename)
                try_rm(tc_filename + '.part')
                try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
        try_rm_tcs_files()
        try:
            try_num = 1
            while True:
                try:
                    # We're not using .download here sine that is just a shim
                    # for outside error handling, and returns the exit code
                    # instead of the result dict.
                    res_dict = ydl.extract_info(
                        test_case['url'],
                        force_generic_extractor=params.get('force_generic_extractor', False))
                except (DownloadError, ExtractorError) as err:
                    # Check if the exception is not a network related one
                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
                        raise

                    if try_num == RETRIES:
                        report_warning('Failed due to network errors, skipping...')
                        return

                    print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))

                    try_num += 1
                else:
                    break

            if is_playlist:
                self.assertTrue(res_dict['_type'] in ['playlist', 'multi_video'])
                self.assertTrue('entries' in res_dict)
                expect_info_dict(self, res_dict, test_case.get('info_dict', {}))

            if 'playlist_mincount' in test_case:
                assertGreaterEqual(
                    self,
                    len(res_dict['entries']),
                    test_case['playlist_mincount'],
                    'Expected at least %d in playlist %s, but got only %d' % (
                        test_case['playlist_mincount'], test_case['url'],
                        len(res_dict['entries'])))
            if 'playlist_count' in test_case:
                self.assertEqual(
                    len(res_dict['entries']),
                    test_case['playlist_count'],
                    'Expected %d entries in playlist %s, but got %d.' % (
                        test_case['playlist_count'],
                        test_case['url'],
                        len(res_dict['entries']),
                    ))
            if 'playlist_duration_sum' in test_case:
                got_duration = sum(e['duration'] for e in res_dict['entries'])
                self.assertEqual(
                    test_case['playlist_duration_sum'], got_duration)

            for tc in test_cases:
                tc_filename = get_tc_filename(tc)
                if not test_case.get('params', {}).get('skip_download', False):
                    self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
                    self.assertTrue(tc_filename in finished_hook_called)
                    expected_minsize = tc.get('file_minsize', 10000)
                    if expected_minsize is not None:
                        if params.get('test'):
                            expected_minsize = max(expected_minsize, 10000)
                        got_fsize = os.path.getsize(tc_filename)
                        assertGreaterEqual(
                            self, got_fsize, expected_minsize,
                            'Expected %s to be at least %s, but it\'s only %s ' %
                            (tc_filename, format_bytes(expected_minsize),
                                format_bytes(got_fsize)))
                    if 'md5' in tc:
                        md5_for_file = _file_md5(tc_filename)
                        self.assertEqual(md5_for_file, tc['md5'])
                info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
                self.assertTrue(
                    os.path.exists(info_json_fn),
                    'Missing info file %s' % info_json_fn)
                with io.open(info_json_fn, encoding='utf-8') as infof:
                    info_dict = json.load(infof)

                expect_info_dict(self, info_dict, tc.get('info_dict', {}))
        finally:
            try_rm_tcs_files()
            if is_playlist and res_dict is not None and res_dict.get('entries'):
                # Remove all other files that may have been extracted if the
                # extractor returns full results even with extract_flat
                res_tcs = [{'info_dict': e} for e in res_dict['entries']]
                try_rm_tcs_files(res_tcs)

    return test_template

Example 38

Project: nYNABapi
Source File: csvimport.py
View license
def do_csvimport(args,client=None):
    if client is None:
        client = clientfromargs(args)
    logger=get_logger(args)

    logger.debug('selected schema %s' % (args.schema,))
    if os.path.exists(args.schema):
        schemafile = args.schema
    else:
        schemafile = os.path.join(schemas_dir, args.schema + '.json')
        if not os.path.exists(schemafile):
            logger.error('This schema doesn''t exist in csv_schemas')
            exit(-1)
    try:
        schema = SchemaModel(schemafile, case_insensitive_headers=True)
        with open(schemafile,'r') as sf:
            schemacontent = json.load(sf)
            try:
                nheaders = schemacontent['nheaders']
            except KeyError:
                nheaders = 1
    except InvalidSchemaError:
        logger.error('Invalid CSV schema')
        raise
    logger.debug('schema headers %s' % schema.headers)

    if 'account' not in schema.headers and args.accountname is None:
        logger.error('This schema does not have an account column and no account name was provided')
        exit(-1)

    accounts = {x.account_name: x for x in client.budget.be_accounts}
    payees = {p.name: p for p in client.budget.be_payees}
    mastercategories_perid = {m.id: m for m in client.budget.be_master_categories}
    subcategories = {}
    for s in client.budget.be_subcategories:
        m=mastercategories_perid[s.entities_master_category_id]
        subcategories[m.name+':'+s.name]=s

    def getaccount(accountname):
        try:
            logger.debug('searching for account %s' % accountname)
            return accounts[accountname]
        except KeyError:
            logger.error('Couldn''t find this account: %s' % accountname)
            exit(-1)

    def getpayee(payeename):
        try:
            logger.debug('searching for payee %s' % payeename)
            return payees[payeename]
        except KeyError:
            logger.debug('Couldn''t find this payee: %s' % payeename)
            payee=Payee(name=payeename)
            client.budget.be_payees.append(payee)
            return payee

    def getsubcategory(categoryname):
        try:
            logger.debug('searching for subcategory %s' % categoryname)
            return subcategories[categoryname]
        except KeyError:
            get_logger(args).debug('Couldn''t find this category: %s' % categoryname)
            exit(-1)

    if 'account' not in schema.headers:
        entities_account_id = getaccount(args.accountname).id

    if 'inflow' in schema.headers and 'outflow' in schema.headers:
        pass
    elif 'amount' in schema.headers:
        pass
    else:
        logger.error('This schema doesn''t provide an amount column or (inflow,outflow) columns')
        exit(-1)

    csvrow = namedtuple('CSVrow', field_names=schema.headers)
    transactions = []

    imported_date=datetime.now().date()

    get_logger(args).debug('OK starting the import from %s '%os.path.abspath(args.csvfile))
    with open(args.csvfile, 'r') as inputfile:
        header = inputfile.readline()
        for row in csv.reader(inputfile):
            if sys.version[0] == '2':
                row = [cell.decode('utf-8') for cell in row]
            if all(map(lambda x:x.strip()=='',row)):
                continue
            get_logger(args).debug('read line %s' % row)
            result = csvrow(*list(schema.convert_row(*row, fail_fast=True)))
            if 'account' in schema.headers:
                entities_account_id = getaccount(result.account).id
            if 'inflow' in schema.headers and 'outflow' in schema.headers:
                amount = result.inflow - result.outflow
            elif 'amount' in schema.headers:
                amount = result.amount
            else:
                get_logger(args).error('Couldn''t find this account: %s' % args.accountname)
                exit(-1)

            if 'category' in schema.headers and result.category:
                entities_subcategory_id = getsubcategory(result.category).id
            else:
                entities_subcategory_id = None
            if 'payee' in schema.headers:
                imported_payee=result.payee
            else:
                imported_payee=''
            entities_payee_id = getpayee(imported_payee).id
            if 'memo' in schema.headers:
                memo=result.memo
            else:
                memo=''


            transaction=Transaction(
                entities_account_id=entities_account_id,
                amount=amount,
                date=result.date,
                entities_payee_id=entities_payee_id,
                entities_subcategory_id=entities_subcategory_id,
                imported_date=imported_date,
                imported_payee=imported_payee,
                memo=memo,
                source="Imported"
            )
            if args.import_duplicates or (not client.budget.be_transactions.containsduplicate(transaction)):
                get_logger(args).debug('Appending transaction %s '%transaction.getdict())
                transactions.append(transaction)
            else:
                get_logger(args).debug('Duplicate transaction found %s '%transaction.getdict())



    client.add_transactions(transactions)

Example 39

Project: rivulet
Source File: __init__.py
View license
def session_thread(receive_queue, queue_poll_interval, torrents_by_hash, config, session_obj, session):
    # TODO make sure the torrent_handles are hashable
    def alert_to_torrent(alert):
        for torrent_handle, torrent in torrents.items():
            if torrent_handle == alert.handle:
                return torrent
        raise LibtorrentErrorException('Got alert for unknown torrent')

    def send_events(streamer_infos, alert):
        # Timeout
        # TODO: this code is unreachable
        if isinstance(alert, type(None)):
            for torrent in torrents:
                for si in streamer_infos:
                    si.time_waited += queue_poll_interval
                    if si.time_waited >= si.timeout:
                        err = "TorrentStream %d timed out waiting for data" % my_id
                        si.put(('kill', TorrentTimeoutException(err)))
        # Error
        elif alert.category() == lt.alert.category_t.error_notification:
            if isinstance(alert, lt.torrent_alert):
                torrent = alert_to_torrent(alert)
                # kill all streamers for this torrent
                for si in torrent.streamers.items().values():
                    si.put(('kill', LibtorrentErrorException(alert)))
            else:
                raise LibtorrentErrorException(alert)
        # Metadata
        elif isinstance(alert, lt.metadata_received_alert):
            torrent = alert_to_torrent(alert)
            for si in torrent.streamers:
                metadata = get_torrent_info(torrent.torrent_handle)
                print >> sys.stderr, 'XXXXXXXXMETADATA', metadata
                torrents_by_hash[str(metadata.info_hash())] = torrent.torrent_handle
                si.put(('metadata', metadata))
        # Data
        elif isinstance(alert, lt.read_piece_alert):
            print >> sys.stderr, 'received data', alert
            torrent = alert_to_torrent(alert)
            print >> sys.stderr, 'found streamers', torrent.streamers
            # TODO: figure out which streamer requested the piece and send the data only to that streamer
            for si in torrent.streamers:
                print >> sys.stderr, 'found streamer', si
                si.time_waited = 0
                # very important! duplicate the alert! Otherwise accessing the alert from another thread segfaults
                si.put(('data', {'piece': alert.piece, 'size': alert.size, 'buffer': alert.buffer[:]}))
        else:
            print >> sys.stderr, 'unknown alert', alert, type(alert), alert.category()

    root_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "..", "..", os.path.expanduser(config.get('private_data_path', DEFAULT_PRIVATE_DATA_PATH)))
    resume_data_path = os.path.join(root_path, os.path.expanduser(config.get('resume_data_path', DEFAULT_RESUME_DATA_PATH)))
    mkdir_p(os.path.dirname(resume_data_path))
    session.listen_on(6881, 6891)
    session.set_alert_mask(lt.alert.category_t.status_notification)
    session.add_dht_router("dht.transmissionbt.com", 6881)
    session.add_dht_router("router.bittorrent.com", 6881)
    session.add_dht_router("router.utorrent.com", 6881)
    torrents = {}
    streamer_infos = {}

    # load resume data for each torrent if available
    try:
        with open(resume_data_path, 'r') as f:
            resume_data = json.load(f)
            for info_hash, resume_data in resume_data.items():
                params = session_obj.torrent_params_from_info_hash(info_hash)
                params['resume_data'] = base64.b64decode(resume_data)
                th = session.add_torrent(params)
                th.set_sequential_download(True)
                torrents_by_hash[str(get_torrent_info(th).info_hash())] = th
    except IOError:
        pass

    while True:
        # alert processing loop forever
        alert = session.wait_for_alert(queue_poll_interval)
        if alert:
          print >> sys.stderr, 'alert:', alert, type(alert), alert.category()
          send_events(streamer_infos, alert)
          session.pop_alert()
          #TODO: check if any recipients are done and remove them from the list


        # every X seconds or every alert check for requests from new callers
        if not receive_queue.empty():
            action, caller, data = receive_queue.get()
            print >> sys.stderr, 'action:', action, caller, data
            if action == 'subscribe_streamer':
                torrent_params, file_index_or_name, streamer_info = data
                streamer_infos[caller] = streamer_info

                #TODO check if this is safe to do on another thread so we can get rid of the queue
                torrent_handle = session.add_torrent(torrent_params)
                torrent_handle.set_sequential_download(True)

                # TODO: make this check work even if the metadata is not already available
                # if this torrent already exists, use the existing Torrent object instead of creating a new one
                torrent = None
                for th, t in torrents.items():
                    if torrent_handle == th:
                        torrent = t
                # we have to assume at first that this is a new torrent; if it turns out that it is not,
                # when we receive the alert telling us this, we merge it with the existing torrent
                if torrent is None:
                    torrent = Torrent(torrent_handle)
                torrents[torrent.torrent_handle] = torrent
                # this is done here so that metadata can be delivered to this streamer; the streamer removes itself
                torrent.add_streamer(streamer_info)

                # if the metadata is already available, there will be no event, so we have to do this here:
                torrent_info = None
                try:
                    torrent_info = get_torrent_info(torrent_handle)
                    torrents_by_hash[str(torrent_info.info_hash())] = torrent.torrent_handle
                except:
                    # and if the metadata is not available, it will come via an event
                    pass

                streamer_info.torrent = torrent
                streamer_info.put((torrent, torrent_info))

            elif action == 'unsubscribe_streamer':
                streamer_info = data
                last_user = streamer_info.torrent.remove_streamer(streamer_info)
                if last_user == True:
                    print >> sys.stderr, 'Torrent not needed any more, maybe we should remove it?'
                    del torrents_by_hash[str(get_torrent_info(streamer_info.torrent.torrent_handle).info_hash())]
                streamer_info.torrent = None

            elif action == 'quit':
                print 'shutting down gracefully and saving resume data'
                # go through each torrent and save their resume_data
                outstanding = 0
                for th, t in torrents.items():
                    if not th.is_valid():
                        continue
                    if not th.status().has_metadata:
                        continue
                    if not th.need_save_resume_data():
                        continue
                    th.save_resume_data(3)  # TODO: use the proper named flags
                    outstanding += 1

                save = {}
                try:
                    with open(resume_data_path, 'r') as old_f:
                        save = json.load(old_f)
                except IOError:
                    pass

                while outstanding > 0:
                    alert = session.wait_for_alert(10000) #wait extra long in this case
                    if not alert:
                        # give up if it times out
                        print 'timed out on shutdown'
                        break

                    if isinstance(alert, lt.save_resume_data_failed_alert):
                        print 'failed to save resume data'

                    elif isinstance(alert, lt.save_resume_data_alert):
                        th = alert.handle
                        hsh = str(get_torrent_info(th).info_hash())
                        # Sorry. If you feel like improving this, be my guest, but I didn't want to deal with it
                        save[hsh] = base64.b64encode(lt.bencode(alert.resume_data))

                    session.pop_alert()
                    outstanding -= 1

                print 'dumping resume_data'
                with open(resume_data_path, 'w') as f:
                    json.dump(save, f)
                print 'dumped resume_data'
                return
            else:
                raise Exception('WTF')

Example 40

Project: orchestra
Source File: test_load.py
View license
    def test_load_workflow_version(self):
        """ Ensure that workflow version loading works as desired. """
        # Verify initial DB state.
        assert_test_dir_workflow_not_loaded(self)
        assert_test_dir_v1_not_loaded(self)
        assert_test_dir_v2_not_loaded(self)

        # Load V1 of the workflow.
        load_workflow(WORKFLOWS['valid']['app_label'], VERSION_1)
        workflow = Workflow.objects.get(slug='test_dir')
        assert_test_dir_workflow_loaded(self)
        assert_test_dir_v1_loaded(self)
        assert_test_dir_v2_not_loaded(self)

        # Load the JSON data for the versions.
        v1_file_path = os.path.join(WORKFLOWS['valid']['dir'],
                                    'v1/version.json')
        with open(v1_file_path, 'r') as v1_file:
            v1_data = json.load(v1_file)

        v2_file_path = os.path.join(WORKFLOWS['valid']['dir'],
                                    'v2/version.json')
        with open(v2_file_path, 'r') as v2_file:
            v2_data = json.load(v2_file)

        # Without --force, can't overwrite a version.
        # We wrap calls to load_workflow_version in transaction.atomic, because
        # the call might create corrupt database state otherwise.
        force_error = 'Version {} already exists'.format(VERSION_1)
        with self.assertRaisesMessage(WorkflowError, force_error):
            with transaction.atomic():
                load_workflow_version(v1_data, workflow)

        # Even with --force, can't overwrite a version with a new step
        v1_data['steps'].append({'slug': 'invalid_new_step'})
        step_change_err_msg = ('Even with --force, cannot change the steps of '
                               'a workflow.')
        with self.assertRaisesMessage(WorkflowError, step_change_err_msg):
            with transaction.atomic():
                load_workflow_version(v1_data, workflow, force=True)
        v1_data['steps'] = v1_data['steps'][:-1]

        # Even with --force, can't change a step's creation dependencies.
        step_2_create_dependencies = v1_data['steps'][1]['creation_depends_on']
        step_2_create_dependencies.append('s3')
        topology_change_err_msg = ('Even with --force, cannot change the '
                                   'topology of a workflow.')
        with self.assertRaisesMessage(WorkflowError, topology_change_err_msg):
            with transaction.atomic():
                load_workflow_version(v1_data, workflow, force=True)
        v1_data['steps'][1]['creation_depends_on'] = (
            step_2_create_dependencies[:-1])

        # Even with --force, can't change a step's submission dependencies.
        step_3_submit_dependencies = v1_data['steps'][2][
            'submission_depends_on']
        step_3_submit_dependencies.append('s1')
        with self.assertRaisesMessage(WorkflowError, topology_change_err_msg):
            with transaction.atomic():
                load_workflow_version(v1_data, workflow, force=True)
        v1_data['steps'][2]['submission_depends_on'] = (
            step_3_submit_dependencies[:-1])

        # Otherwise, --force should reload versions correctly.
        with transaction.atomic():
            load_workflow_version(v1_data, workflow, force=True)
        assert_test_dir_workflow_loaded(self)
        assert_test_dir_v1_loaded(self)
        assert_test_dir_v2_not_loaded(self)

        # New versions with bad slugs should not load correctly
        v2_step_2 = v2_data['steps'][1]
        v2_step_2_create_dependencies = v2_step_2['creation_depends_on']
        v2_step_2_create_dependencies.append('not_a_real_step')
        bad_slug_error = '{}.{} contains a non-existent slug'
        with self.assertRaisesMessage(
                WorkflowError,
                bad_slug_error.format('s2', 'creation_depends_on')):
            with transaction.atomic():
                load_workflow_version(v2_data, workflow)
        v2_step_2['creation_depends_on'] = (
            v2_step_2_create_dependencies[:-1])

        v2_step_2_submit_dependencies = v2_step_2['submission_depends_on']
        v2_step_2_submit_dependencies.append('not_a_real_step')
        with self.assertRaisesMessage(
                WorkflowError,
                bad_slug_error.format('s2', 'submission_depends_on')):
            with transaction.atomic():
                load_workflow_version(v2_data, workflow)
        v2_step_2['submission_depends_on'] = (
            v2_step_2_submit_dependencies[:-1])

        v2_step_2_certification_dependencies = v2_step_2[
            'required_certifications']
        v2_step_2_certification_dependencies.append('not_a_real_certification')
        with self.assertRaisesMessage(
                WorkflowError,
                bad_slug_error.format('s2', 'required_certifications')):
            with transaction.atomic():
                load_workflow_version(v2_data, workflow)
        v2_step_2['required_certifications'] = (
            v2_step_2_certification_dependencies[:-1])

        # Otherwise, new versions should load correctly
        with transaction.atomic():
            load_workflow_version(v2_data, workflow)
        assert_test_dir_workflow_loaded(self)
        assert_test_dir_v1_loaded(self)
        assert_test_dir_v2_loaded(self)

Example 41

View license
def run_training(cmd_line_args=None):
    import argparse
    parser = argparse.ArgumentParser(description='Perform reinforcement learning to improve given policy network. Second phase of pipeline.')  # noqa: E501
    parser.add_argument("model_json", help="Path to policy model JSON.")
    parser.add_argument("initial_weights", help="Path to HDF5 file with inital weights (i.e. result of supervised training).")  # noqa: E501
    parser.add_argument("out_directory", help="Path to folder where the model params and metadata will be saved after each epoch.")  # noqa: E501
    parser.add_argument("--learning-rate", help="Keras learning rate (Default: 0.001)", type=float, default=0.001)  # noqa: E501
    parser.add_argument("--policy-temp", help="Distribution temperature of players using policies (Default: 0.67)", type=float, default=0.67)  # noqa: E501
    parser.add_argument("--save-every", help="Save policy as a new opponent every n batches (Default: 500)", type=int, default=500)  # noqa: E501
    parser.add_argument("--game-batch", help="Number of games per mini-batch (Default: 20)", type=int, default=20)  # noqa: E501
    parser.add_argument("--move-limit", help="Maximum number of moves per game", type=int, default=500)  # noqa: E501
    parser.add_argument("--iterations", help="Number of training batches/iterations (Default: 10000)", type=int, default=10000)  # noqa: E501
    parser.add_argument("--resume", help="Load latest weights in out_directory and resume", default=False, action="store_true")  # noqa: E501
    parser.add_argument("--verbose", "-v", help="Turn on verbose mode", default=False, action="store_true")  # noqa: E501
    # Baseline function (TODO) default lambda state: 0  (receives either file
    # paths to JSON and weights or None, in which case it uses default baseline 0)
    if cmd_line_args is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(cmd_line_args)

    ZEROTH_FILE = "weights.00000.hdf5"

    if args.resume:
        if not os.path.exists(os.path.join(args.out_directory, "metadata.json")):
            raise ValueError("Cannot resume without existing output directory")

    if not os.path.exists(args.out_directory):
        if args.verbose:
            print "creating output directory {}".format(args.out_directory)
        os.makedirs(args.out_directory)

    if not args.resume:
        # make a copy of weights file, "weights.00000.hdf5" in the output directory
        copyfile(args.initial_weights, os.path.join(args.out_directory, ZEROTH_FILE))
        if args.verbose:
            print "copied {} to {}".format(args.initial_weights,
                                           os.path.join(args.out_directory, ZEROTH_FILE))
        player_weights = ZEROTH_FILE
    else:
        # if resuming, we expect initial_weights to be just a
        # "weights.#####.hdf5" file, not a full path
        args.initial_weights = os.path.join(args.out_directory,
                                            os.path.basename(args.initial_weights))
        if not os.path.exists(args.initial_weights):
            raise ValueError("Cannot resume; weights {} do not exist".format(args.initial_weights))
        elif args.verbose:
            print "Resuming with weights {}".format(args.initial_weights)
        player_weights = os.path.basename(args.initial_weights)

    # Set initial conditions
    policy = CNNPolicy.load_model(args.model_json)
    policy.model.load_weights(args.initial_weights)
    player = ProbabilisticPolicyPlayer(policy, temperature=args.policy_temp,
                                       move_limit=args.move_limit)

    # different opponents come from simply changing the weights of 'opponent.policy.model'. That
    # is, only 'opp_policy' needs to be changed, and 'opponent' will change.
    opp_policy = CNNPolicy.load_model(args.model_json)
    opponent = ProbabilisticPolicyPlayer(opp_policy, temperature=args.policy_temp,
                                         move_limit=args.move_limit)

    if args.verbose:
        print "created player and opponent with temperature {}".format(args.policy_temp)

    if not args.resume:
        metadata = {
            "model_file": args.model_json,
            "init_weights": args.initial_weights,
            "learning_rate": args.learning_rate,
            "temperature": args.policy_temp,
            "game_batch": args.game_batch,
            "opponents": [ZEROTH_FILE],  # which weights from which to sample an opponent each batch
            "win_ratio": {}  # map from player to tuple of (opponent, win ratio) Useful for
                             # validating in lieu of 'accuracy/loss'
        }
    else:
        with open(os.path.join(args.out_directory, "metadata.json"), "r") as f:
            metadata = json.load(f)

    # Append args of current run to history of full command args.
    metadata["cmd_line_args"] = metadata.get("cmd_line_args", []).append(vars(args))

    def save_metadata():
        with open(os.path.join(args.out_directory, "metadata.json"), "w") as f:
            json.dump(metadata, f, sort_keys=True, indent=2)

    optimizer = BatchedReinforcementLearningSGD(lr=args.learning_rate, ng=args.game_batch)
    player.policy.model.compile(loss=log_loss, optimizer=optimizer)
    for i_iter in xrange(1, args.iterations + 1):
        # Randomly choose opponent from pool (possibly self), and playing
        # game_batch games against them.
        opp_weights = np.random.choice(metadata["opponents"])
        opp_path = os.path.join(args.out_directory, opp_weights)

        # Load new weights into opponent's network, but keep the same opponent object.
        opponent.policy.model.load_weights(opp_path)
        if args.verbose:
            print "Batch {}\tsampled opponent is {}".format(i_iter, opp_weights)

        # Run games (and learn from results). Keep track of the win ratio vs
        # each opponent over time.
        win_ratio = run_n_games(optimizer, player, opponent, args.game_batch)
        metadata["win_ratio"][player_weights] = (opp_weights, win_ratio)

        # Save all intermediate models.
        player_weights = "weights.%05d.hdf5" % i_iter
        player.policy.model.save_weights(os.path.join(args.out_directory, player_weights))

        # Add player to batch of oppenents once in a while.
        if i_iter % args.save_every == 0:
            metadata["opponents"].append(player_weights)
        save_metadata()

Example 42

View license
def run_training(cmd_line_args=None):
    """Run training. command-line args may be passed in as a list
    """
    import argparse
    parser = argparse.ArgumentParser(description='Perform supervised training on a policy network.')
    # required args
    parser.add_argument("model", help="Path to a JSON model file (i.e. from CNNPolicy.save_model())")  # noqa: E501
    parser.add_argument("train_data", help="A .h5 file of training data")
    parser.add_argument("out_directory", help="directory where metadata and weights will be saved")
    # frequently used args
    parser.add_argument("--minibatch", "-B", help="Size of training data minibatches. Default: 16", type=int, default=16)  # noqa: E501
    parser.add_argument("--epochs", "-E", help="Total number of iterations on the data. Default: 10", type=int, default=10)  # noqa: E501
    parser.add_argument("--epoch-length", "-l", help="Number of training examples considered 'one epoch'. Default: # training data", type=int, default=None)  # noqa: E501
    parser.add_argument("--learning-rate", "-r", help="Learning rate - how quickly the model learns at first. Default: .03", type=float, default=.03)  # noqa: E501
    parser.add_argument("--decay", "-d", help="The rate at which learning decreases. Default: .0001", type=float, default=.0001)  # noqa: E501
    parser.add_argument("--verbose", "-v", help="Turn on verbose mode", default=False, action="store_true")  # noqa: E501
    # slightly fancier args
    parser.add_argument("--weights", help="Name of a .h5 weights file (in the output directory) to load to resume training", default=None)  # noqa: E501
    parser.add_argument("--train-val-test", help="Fraction of data to use for training/val/test. Must sum to 1. Invalid if restarting training", nargs=3, type=float, default=[0.93, .05, .02])  # noqa: E501
    parser.add_argument("--symmetries", help="Comma-separated list of transforms, subset of noop,rot90,rot180,rot270,fliplr,flipud,diag1,diag2", default='noop,rot90,rot180,rot270,fliplr,flipud,diag1,diag2')  # noqa: E501
    # TODO - an argument to specify which transformations to use, put it in metadata

    if cmd_line_args is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(cmd_line_args)

    # TODO - what follows here should be refactored into a series of small functions

    resume = args.weights is not None

    if args.verbose:
        if resume:
            print("trying to resume from %s with weights %s" %
                  (args.out_directory, os.path.join(args.out_directory, args.weights)))
        else:
            if os.path.exists(args.out_directory):
                print("directory %s exists. any previous data will be overwritten" %
                      args.out_directory)
            else:
                print "starting fresh output directory %s" % args.out_directory

    # load model from json spec
    model = CNNPolicy.load_model(args.model).model
    if resume:
        model.load_weights(os.path.join(args.out_directory, args.weights))

    # TODO - (waiting on game_converter) verify that features of model match
    # features of training data
    dataset = h5.File(args.train_data)
    n_total_data = len(dataset["states"])
    n_train_data = int(args.train_val_test[0] * n_total_data)
    # Need to make sure training data is divisible by minibatch size or get
    # warning mentioning accuracy from keras
    n_train_data = n_train_data - (n_train_data % args.minibatch)
    n_val_data = n_total_data - n_train_data
    # n_test_data = n_total_data - (n_train_data + n_val_data)

    if args.verbose:
        print "datset loaded"
        print "\t%d total samples" % n_total_data
        print "\t%d training samples" % n_train_data
        print "\t%d validaion samples" % n_val_data

    # ensure output directory is available
    if not os.path.exists(args.out_directory):
        os.makedirs(args.out_directory)

    # create metadata file and the callback object that will write to it
    meta_file = os.path.join(args.out_directory, "metadata.json")
    meta_writer = MetadataWriterCallback(meta_file)
    # load prior data if it already exists
    if os.path.exists(meta_file) and resume:
        with open(meta_file, "r") as f:
            meta_writer.metadata = json.load(f)
        if args.verbose:
            print("previous metadata loaded: %d epochs. new epochs will be appended." %
                  len(meta_writer.metadata["epochs"]))
    elif args.verbose:
        print "starting with empty metadata"
    # the MetadataWriterCallback only sets 'epoch' and 'best_epoch'. We can add
    # in anything else we like here
    #
    # TODO - model and train_data are saved in meta_file; check that they match
    # (and make args optional when restarting?)
    meta_writer.metadata["training_data"] = args.train_data
    meta_writer.metadata["model_file"] = args.model
    # Record all command line args in a list so that all args are recorded even
    # when training is stopped and resumed.
    meta_writer.metadata["cmd_line_args"] \
        = meta_writer.metadata.get("cmd_line_args", []).append(vars(args))

    # create ModelCheckpoint to save weights every epoch
    checkpoint_template = os.path.join(args.out_directory, "weights.{epoch:05d}.hdf5")
    checkpointer = ModelCheckpoint(checkpoint_template)

    # load precomputed random-shuffle indices or create them
    # TODO - save each train/val/test indices separately so there's no danger of
    # changing args.train_val_test when resuming
    shuffle_file = os.path.join(args.out_directory, "shuffle.npz")
    if os.path.exists(shuffle_file) and resume:
        with open(shuffle_file, "r") as f:
            shuffle_indices = np.load(f)
        if args.verbose:
            print "loading previous data shuffling indices"
    else:
        # create shuffled indices
        shuffle_indices = np.random.permutation(n_total_data)
        with open(shuffle_file, "w") as f:
            np.save(f, shuffle_indices)
        if args.verbose:
            print "created new data shuffling indices"
    # training indices are the first consecutive set of shuffled indices, val
    # next, then test gets the remainder
    train_indices = shuffle_indices[0:n_train_data]
    val_indices = shuffle_indices[n_train_data:n_train_data + n_val_data]
    # test_indices = shuffle_indices[n_train_data + n_val_data:]

    symmetries = [BOARD_TRANSFORMATIONS[name] for name in args.symmetries.strip().split(",")]

    # create dataset generators
    train_data_generator = shuffled_hdf5_batch_generator(
        dataset["states"],
        dataset["actions"],
        train_indices,
        args.minibatch,
        symmetries)
    val_data_generator = shuffled_hdf5_batch_generator(
        dataset["states"],
        dataset["actions"],
        val_indices,
        args.minibatch,
        symmetries)

    sgd = SGD(lr=args.learning_rate, decay=args.decay)
    model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=["accuracy"])

    samples_per_epoch = args.epoch_length or n_train_data

    if args.verbose:
        print "STARTING TRAINING"

    model.fit_generator(
        generator=train_data_generator,
        samples_per_epoch=samples_per_epoch,
        nb_epoch=args.epochs,
        callbacks=[checkpointer, meta_writer],
        validation_data=val_data_generator,
        nb_val_samples=n_val_data)

Example 43

Project: Shadowsocks-For-WHMCS
Source File: utils.py
View license
def get_config(is_local):
    logging.basicConfig(level=logging.INFO,
                        format='%(levelname)-s: %(message)s', filemode='a+')
    if is_local:
        shortopts = 'hs:b:p:k:l:m:c:t:vq'
        longopts = ['fast-open']
    else:
        shortopts = 'hs:p:k:m:c:t:vq'
        longopts = ['fast-open', 'workers:']
    try:
        config_path = find_config()
        optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
        for key, value in optlist:
            if key == '-c':
                config_path = value

        if config_path:
            logging.info('loading config from %s' % config_path)
            with open(config_path, 'rb') as f:
                try:
                    config = json.load(f, object_hook=_decode_dict)
                except ValueError as e:
                    logging.error('found an error in config.json: %s',
                                  e.message)
                    sys.exit(1)
        else:
            config = {}

        optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
        v_count = 0
        for key, value in optlist:
            if key == '-p':
                config['server_port'] = int(value)
            elif key == '-k':
                config['password'] = value
            elif key == '-l':
                config['local_port'] = int(value)
            elif key == '-s':
                config['server'] = value
            elif key == '-m':
                config['method'] = value
            elif key == '-b':
                config['local_address'] = value
            elif key == '-v':
                v_count += 1
                # '-vv' turns on more verbose mode
                config['verbose'] = v_count
            elif key == '-t':
                config['timeout'] = int(value)
            elif key == '--fast-open':
                config['fast_open'] = True
            elif key == '--workers':
                config['workers'] = value
            elif key == '-h':
                if is_local:
                    print_local_help()
                else:
                    print_server_help()
                sys.exit(0)
            elif key == '-q':
                v_count -= 1
                config['verbose'] = v_count
    except getopt.GetoptError as e:
        print >>sys.stderr, e
        print_help(is_local)
        sys.exit(2)

    if not config:
        logging.error('config not specified')
        print_help(is_local)
        sys.exit(2)

    config['password'] = config.get('password', None)
    config['method'] = config.get('method', 'aes-256-cfb')
    config['port_password'] = config.get('port_password', None)
    config['timeout'] = int(config.get('timeout', 300))
    config['fast_open'] = config.get('fast_open', False)
    config['workers'] = config.get('workers', 1)
    config['verbose'] = config.get('verbose', False)
    config['local_address'] = config.get('local_address', '127.0.0.1')
    config['local_port'] = config.get('local_port', 1080)
    if is_local:
        if config.get('server', None) is None:
            logging.error('server addr not specified')
            print_local_help()
            sys.exit(2)
    else:
        config['server'] = config.get('server', '0.0.0.0')
    config['server_port'] = config.get('server_port', 8388)

    if not ('password' in config and config['password']):
        logging.error('password not specified')
        print_help(is_local)
        sys.exit(2)

    logging.getLogger('').handlers = []
    logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE')
    if config['verbose'] >= 2:
        level = VERBOSE_LEVEL
    elif config['verbose'] == 1:
        level = logging.DEBUG
    elif config['verbose'] == -1:
        level = logging.WARN
    elif config['verbose'] <= -2:
        level = logging.ERROR
    else:
        level = logging.INFO
    logging.basicConfig(level=level,
                        format='%(asctime)s %(levelname)-8s %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S', filemode='a+')

    check_config(config)

    return config

Example 44

Project: ibus-uniemoji
Source File: uniemoji.py
View license
    def __init__(self):
        super(UniEmoji, self).__init__()
        self.table = defaultdict(UniEmojiChar)
        self.unicode_chars_to_names = {}
        self.unicode_chars_to_shortnames = {}
        self.ascii_table = {}
        self.reverse_ascii_table = {}
        self.alias_table = {}
        with open(os.path.join(__base_dir__, 'UnicodeData.txt'), encoding='utf-8') as unicodedata:
            for line in unicodedata.readlines():
                if not line.strip(): continue
                code, name, category, _ = line.split(';', 3)
                code = int(code, 16)
                if category not in VALID_CATEGORIES:
                    continue
                if not in_range(code):
                    continue
                name = name.lower()
                unicode_char = chr(code)
                self.table[name] = UniEmojiChar(unicode_char)
                self.unicode_chars_to_names[unicode_char] = name

        # Load emojione file
        alias_counter = Counter()
        temp_alias_table = defaultdict(set)

        emojione = json.load(open(os.path.join(__base_dir__, 'emojione.json'), encoding='utf-8'))
        for emoji_shortname, info in emojione.items():

            # ZWJ emojis such as 'family', 'couple', and 'kiss' appear in an
            # alternate field
            alternate_form = info.get('unicode_alternates')
            if alternate_form and '200d' in alternate_form:
                chars = alternate_form
            else:
                chars = info['unicode']

            unicode_str = ''.join(chr(int(codepoint, 16)) for codepoint in chars.split('-'))
            self.unicode_chars_to_shortnames[unicode_str] = emoji_shortname

            emoji_shortname = emoji_shortname.replace('_', ' ')

            if emoji_shortname in self.table:
                # Check for clashes between emojione's names and the existing unicode name.
                # Clashes turn into aliases.
                if unicode_str != self.table[emoji_shortname].unicode_str:
                    self.table[emoji_shortname].aliasing.append(unicode_str)
            elif info['category'] == 'flags':
                flag_name = 'flag of ' + info['name']
                self.table[flag_name] = UniEmojiChar(unicode_str, is_emojione=True)
                self.unicode_chars_to_names[unicode_str] = flag_name
            else:
                self.table[emoji_shortname] = UniEmojiChar(unicode_str, is_emojione=True)

            # When the string defined by emojione isn't in Unicode
            # (because it's a combination of characters), use emojione's
            # descriptive name, and set the shortname as an alias
            if unicode_str not in self.unicode_chars_to_names:
                long_name = info['name']
                self.unicode_chars_to_names[unicode_str] = long_name
                if long_name not in self.table:
                    self.table[long_name] = UniEmojiChar(unicode_str)

            # EmojiOne has duplicate entries in the keywords array
            keywords = set(info.get('keywords', []))
            for alias in keywords:
                alias_counter[alias] += 1
                temp_alias_table[alias].add(unicode_str)

            for ascii_aliases in info.get('aliases_ascii', []):
                self.ascii_table[ascii_aliases] = unicode_str
                self.reverse_ascii_table[unicode_str] = info['name']

        # Load less-frequent aliases from emojione file
        for alias, n in alias_counter.most_common():
            if n >= 25:
                continue
            self.table[alias].aliasing.extend(temp_alias_table[alias])

        # Load custom file(s)
        for d in reversed(SETTINGS_DIRS):
            custom_filename = os.path.join(d, 'custom.json')
            debug('Loading custom emoji from {}'.format(custom_filename))
            if os.path.isfile(custom_filename):
                custom_table = None
                try:
                    with open(custom_filename, encoding='utf-8') as f:
                        custom_table = json.loads(f.read())
                except:
                    error = sys.exc_info()[1]
                    debug(error)
                    self.table = {
                        'Failed to load custom file {}: {}'.format(custom_filename, error): 'ERROR'
                    }
                    break
                else:
                    debug(custom_table)
                    for k, v in custom_table.items():
                        self.table[k] = UniEmojiChar(v, is_custom=True)

Example 45

Project: scalarizr
Source File: api.py
View license
    def bootstrap(self, dry_run=False):
        # [SCALARIZR-1797]
        # Cleanup RPMDb from old entries
        if linux.os.redhat_family and not dry_run:
            out = linux.system(('rpm', '-qa', 'scalarizr*', '--queryformat', '%{NAME}|%{VERSION}\n'))[0]
            for line in out.strip().split('\n'):
                name, version = line.strip().split('|')
                if not version == __version__:
                    linux.system(('rpm', '-e', '--nodeps', '--justdb', '{0}-{1}'.format(name, version)))
        try:
            LOG.info('Getting System ID')
            self.system_id = self.get_system_id()
            LOG.info('System ID: {}'.format(self.system_id))
        except:
            # This will force updclient to perform check updates each startup,
            # this is the optimal behavior cause that's ensure latest available package
            LOG.debug('Unable to get System ID: %s', sys.exc_info()[1])
            self.system_id = str(uuid.uuid4())
            LOG.debug('System ID (random generated): {}'.format(self.system_id))

        system_matches = False
        status_data = None
        if os.path.exists(self.status_file):
            LOG.debug('Checking %s', self.status_file)
            with open(self.status_file) as fp:
                status_data = json.load(fp)
                if 'downgrades_enabled' not in status_data:
                    # Field introduced in 2.7.12
                    # Missing field here means downgrades_enabled=False,
                    # cause it's setted by postinst migration to new update system
                    status_data['downgrades_enabled'] = False
            system_matches = status_data['system_id'] == self.system_id
            if not system_matches:
                if not status_data.get('system_id_is_server_id') \
                    and status_data['system_id'] == self.obsolete_get_system_id():
                    LOG.info('System-ID (obsolete) in status file matches current one')
                    status_data['system_id'] = self.system_id
                    status_data['system_id_is_server_id'] = True
                    system_matches = True
                else:
                    LOG.info('System ID changed: %s => %s',
                             status_data['system_id'], self.system_id)
            else:
                LOG.debug('System-ID in status file matches current one')
        else:
            LOG.debug('Status file %s not exists', self.status_file)

        if system_matches:
            LOG.info('Reading state from %s', self.status_file)
            self._update_self_dict(status_data)

        if not system_matches:
            LOG.info('Initializing UpdateClient...')
            user_data = copy.deepcopy(metadata.user_data())
            norm_user_data(user_data)
            LOG.info('Applying configuration from user-data')
            self._update_self_dict(user_data)

            crypto_dir = os.path.dirname(self.crypto_file)
            if not os.path.exists(crypto_dir):
                os.makedirs(crypto_dir)
            if os.path.exists(self.crypto_file):
                LOG.info('Testing that crypto key works (file: %s)', self.crypto_file)
                try:
                    self._init_db()
                    self._init_queryenv()
                    LOG.info('Crypto key works')
                except queryenv.InvalidSignatureError:
                    LOG.info("Crypto key doesn't work: got invalid signature error")
                    self.queryenv = None
            if not self.queryenv:
                LOG.info("Use crypto key from user-data")
                if os.path.exists(self.crypto_file):
                    os.chmod(self.crypto_file, 0600)
                with open(self.crypto_file, 'w+') as fp:
                    fp.write(user_data['szr_key'])
                os.chmod(self.crypto_file, 0400)
        self.early_bootstrapped = True

        try:
            self._init_services()
        except sqlite.OperationalError as e:
            # [SCALARIZR-2168] Scalarizr as part of post rebundle cleanup removed db.sqlite
            LOG.debug('Database error: {}'.format(e))
            wait_until(
                lambda: os.path.exists(self.db_file) and os.stat(self.db_file).st_size,
                timeout=10, sleep=0.1)
            self._init_db()
            self._init_services()

        # - my uptime is 644 days, 20 hours and 13 mins and i know nothing about 'platform' in user-data
        if not self.platform:
            self.platform = bus.cnf.rawini.get('general', 'platform')
        # - my uptime is 1086 days, 55 mins and i know nothing about 'farm_roleid' in user-data
        if not self.farm_role_id:
            self.farm_role_id = bus.cnf.rawini.get('general', 'farm_role_id')

        self.system_matches = system_matches
        if not self.system_matches:
            if dry_run:
                self._sync()
            else:
                self.update(bootstrap=True)
        else:
            # if self.state in ('completed/wait-ack', 'noop'):
            if self.state not in ('error', 'rollbacked'):
                # forcefully finish any in-progress operations
                self.state = 'completed'
            self.store()

            # Set correct repo value at start. Need this to recover in case they
            # were not set during _sync() in the previous run [SCALARIZR-1885]
            self._ensure_repos()
        if not (self.shutdown_ev.is_set() or dry_run or
                (self.state == 'error' and not system_matches) or
                self.daemon.running):
            # we shouldn't start Scalarizr
            # - when UpdateClient is terminating
            # - when UpdateClient is not performing any updates
            # - when state is 'error' and it's a first UpdateClient start on a new system
            # - when Scalarizr is already running
            self.daemon.start()
        if self.state == 'completed/wait-ack':
            obsoletes = pkg_resources.Requirement.parse('A<=2.7.5')
            inst = re.sub(r'^\d\:', '', self.installed)  # remove debian epoch
            if inst in obsoletes:
                LOG.info('UpdateClient is going to restart itself, cause ')

                def restart_self():
                    time.sleep(5)
                    name = 'ScalrUpdClient' if linux.os.windows else 'scalr-upd-client'
                    service = initdv2.Daemon(name)
                    service.restart()
                proc = multiprocessing.Process(target=restart_self)
                proc.start()

Example 46

Project: iris
Source File: __init__.py
View license
    def _assert_graphic(self, tol=_HAMMING_DISTANCE):
        """
        Check the hash of the current matplotlib figure matches the expected
        image hash for the current graphic test.

        To create missing image test results, set the IRIS_TEST_CREATE_MISSING
        environment variable before running the tests. This will result in new
        and appropriately "<hash>.png" image files being generated in the image
        output directory, and the imagerepo.json file being updated.

        """
        import imagehash
        from PIL import Image

        dev_mode = os.environ.get('IRIS_TEST_CREATE_MISSING')
        unique_id = self._unique_id()
        repo_fname = os.path.join(_RESULT_PATH, 'imagerepo.json')
        with open(repo_fname, 'rb') as fi:
            repo = json.load(codecs.getreader('utf-8')(fi))

        try:
            #: The path where the images generated by the tests should go.
            image_output_directory = os.path.join(os.path.dirname(__file__),
                                                  'result_image_comparison')
            if not os.access(image_output_directory, os.W_OK):
                if not os.access(os.getcwd(), os.W_OK):
                    raise IOError('Write access to a local disk is required '
                                  'to run image tests.  Run the tests from a '
                                  'current working directory you have write '
                                  'access to to avoid this issue.')
                else:
                    image_output_directory = os.path.join(
                        os.getcwd(), 'iris_image_test_output')
            result_fname = os.path.join(image_output_directory,
                                        'result-' + unique_id + '.png')

            if not os.path.isdir(image_output_directory):
                # Handle race-condition where the directories are
                # created sometime between the check above and the
                # creation attempt below.
                try:
                    os.makedirs(image_output_directory)
                except OSError as err:
                    # Don't care about "File exists"
                    if err.errno != 17:
                        raise

            def _create_missing():
                fname = '{}.png'.format(phash)
                base_uri = ('https://scitools.github.io/test-iris-imagehash/'
                            'images/{}')
                uri = base_uri.format(fname)
                hash_fname = os.path.join(image_output_directory, fname)
                uris = repo.setdefault(unique_id, [])
                uris.append(uri)
                print('Creating image file: {}'.format(hash_fname))
                figure.savefig(hash_fname)
                msg = 'Creating imagerepo entry: {} -> {}'
                print(msg.format(unique_id, uri))
                with open(repo_fname, 'wb') as fo:
                    json.dump(repo, codecs.getwriter('utf-8')(fo), indent=4,
                              sort_keys=True)

            # TBD: Push this fix to imagehash (done!)
            # See https://github.com/JohannesBuchner/imagehash/pull/31
            # Now need this imagehash/master pushed to pypi ...
            def _hex_to_hash(hexstr, hash_size=_HASH_SIZE):
                l = []
                count = hash_size * (hash_size // 4)
                if len(hexstr) != count:
                    emsg = 'Expected hex string size of {}.'
                    raise ValueError(emsg.format(count))
                for i in range(count // 2):
                    h = hexstr[i*2:i*2+2]
                    v = int("0x" + h, 16)
                    l.append([v & 2**i > 0 for i in range(8)])
                return imagehash.ImageHash(np.array(l))

            # Calculate the test result perceptual image hash.
            buffer = io.BytesIO()
            figure = plt.gcf()
            figure.savefig(buffer, format='png')
            buffer.seek(0)
            phash = imagehash.phash(Image.open(buffer), hash_size=_HASH_SIZE)

            if unique_id not in repo:
                if dev_mode:
                    _create_missing()
                else:
                    figure.savefig(result_fname)
                    emsg = 'Missing image test result: {}.'
                    raise ValueError(emsg.format(unique_id))
            else:
                uris = repo[unique_id]
                # Create the expected perceptual image hashes from the uris.
                expected = [_hex_to_hash(os.path.splitext(os.path.basename(uri))[0])
                            for uri in uris]

                # Calculate the hamming distance vector for the result hash.
                distances = [e - phash for e in expected]

                if np.all([hd > tol for hd in distances]):
                    if dev_mode:
                        _create_missing()
                    else:
                        figure.savefig(result_fname)
                        msg = ('Bad phash {} with hamming distance {} '
                               'for test {}.')
                        msg = msg.format(phash, distances, unique_id)
                        if _DISPLAY_FIGURES:
                            emsg = 'Image comparion would have failed: {}'
                            print(emsg.format(msg))
                        else:
                            emsg = 'Image comparison failed: {}'
                            raise ValueError(emsg.format(msg))

            if _DISPLAY_FIGURES:
                plt.show()

        finally:
            plt.close()

Example 47

Project: ipwhois
Source File: test_rdap.py
View license
    def test__RDAPLookup(self):

        data_dir = path.dirname(__file__)

        with io.open(str(data_dir) + '/rdap.json', 'r') as data_file:
            data = json.load(data_file)

        for key, val in data.items():

            log.debug('Testing: {0}'.format(key))
            net = Net(key)
            obj = RDAP(net)

            try:

                self.assertIsInstance(obj.lookup(response=val['response'],
                                                 asn_data=val['asn_data'],
                                                 depth=0), dict)

            except AssertionError as e:

                raise e

            except Exception as e:

                self.fail('Unexpected exception raised: {0}'.format(e))

            self.assertRaises(NetError, RDAP, 'a')

        for key, val in data.items():

            log.debug('Testing bootstrap and raw: {0}'.format(key))
            net = Net(key)
            obj = RDAP(net)

            try:

                self.assertIsInstance(obj.lookup(response=val['response'],
                                                 asn_data=val['asn_data'],
                                                 depth=0,
                                                 bootstrap=True,
                                                 inc_raw=True), dict)

            except AssertionError as e:

                raise e

            except Exception as e:

                self.fail('Unexpected exception raised: {0}'.format(e))

            break

        log.debug('Testing rdap.lookup response checks')
        net = Net('74.125.225.229')
        obj = RDAP(net)
        self.assertIsInstance(obj.lookup(response={
                                             'handle': 'test',
                                             'ipVersion': 'v4',
                                             'startAddress': '74.125.225.229',
                                             'endAddress': '74.125.225.229'
                                         },
                                         asn_data=val['asn_data'],
                                         depth=0), dict)

        log.debug('Testing rdap.lookup entitiy checks')
        net = Net('74.125.225.229')
        obj = RDAP(net)
        entity = [{'handle': 'test', 'roles': [
            'administrative', 'technical'], 'entities': [
            {'handle': 'GOGL', 'roles': ['administrative', 'technical']}]}]

        self.assertIsInstance(obj.lookup(response={
                                             'handle': 'test',
                                             'ipVersion': 'v4',
                                             'startAddress': '74.125.225.229',
                                             'endAddress': '74.125.225.229',
                                             'entities': entity
                                         },
                                         asn_data=val['asn_data'],
                                         depth=1), dict)

        self.assertIsInstance(obj.lookup(response={
                                             'handle': 'test',
                                             'ipVersion': 'v4',
                                             'startAddress': '74.125.225.229',
                                             'endAddress': '74.125.225.229',
                                             'entities': entity
                                         },
                                         asn_data=val['asn_data'],
                                         depth=1,
                                         bootstrap=True,
                                         inc_raw=True), dict)

Example 48

Project: shadowsocks-manyuser
Source File: utils.py
View license
def get_config(is_local):
    logging.basicConfig(level=logging.INFO,
                        format='%(levelname)-s: %(message)s', filemode='a+')
    if is_local:
        shortopts = 'hs:b:p:k:l:m:c:t:vq'
        longopts = ['fast-open']
    else:
        shortopts = 'hs:p:k:m:c:t:vq'
        longopts = ['fast-open', 'workers:']
    try:
        config_path = find_config()
        optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
        for key, value in optlist:
            if key == '-c':
                config_path = value

        if config_path:
            logging.info('loading config from %s' % config_path)
            with open(config_path, 'rb') as f:
                try:
                    config = json.load(f, object_hook=_decode_dict)
                except ValueError as e:
                    logging.error('found an error in config.json: %s',
                                  e.message)
                    sys.exit(1)
        else:
            config = {}

        optlist, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
        v_count = 0
        for key, value in optlist:
            if key == '-p':
                config['server_port'] = int(value)
            elif key == '-k':
                config['password'] = value
            elif key == '-l':
                config['local_port'] = int(value)
            elif key == '-s':
                config['server'] = value
            elif key == '-m':
                config['method'] = value
            elif key == '-b':
                config['local_address'] = value
            elif key == '-v':
                v_count += 1
                # '-vv' turns on more verbose mode
                config['verbose'] = v_count
            elif key == '-t':
                config['timeout'] = int(value)
            elif key == '--fast-open':
                config['fast_open'] = True
            elif key == '--workers':
                config['workers'] = value
            elif key == '-h':
                if is_local:
                    print_local_help()
                else:
                    print_server_help()
                sys.exit(0)
            elif key == '-q':
                v_count -= 1
                config['verbose'] = v_count
    except getopt.GetoptError as e:
        print >>sys.stderr, e
        print_help(is_local)
        sys.exit(2)

    if not config:
        logging.error('config not specified')
        print_help(is_local)
        sys.exit(2)

    config['password'] = config.get('password', None)
    config['method'] = config.get('method', 'aes-256-cfb')
    config['port_password'] = config.get('port_password', None)
    config['timeout'] = int(config.get('timeout', 300))
    config['fast_open'] = config.get('fast_open', False)
    config['workers'] = config.get('workers', 1)
    config['verbose'] = config.get('verbose', False)
    config['local_address'] = config.get('local_address', '127.0.0.1')
    config['local_port'] = config.get('local_port', 1080)
    if is_local:
        if config.get('server', None) is None:
            logging.error('server addr not specified')
            print_local_help()
            sys.exit(2)
    else:
        config['server'] = config.get('server', '0.0.0.0')
    config['server_port'] = config.get('server_port', 8388)

    if not ('password' in config and config['password']):
        logging.error('password not specified')
        print_help(is_local)
        sys.exit(2)

    logging.getLogger('').handlers = []
    logging.addLevelName(VERBOSE_LEVEL, 'VERBOSE')
    if config['verbose'] >= 2:
        level = VERBOSE_LEVEL
    elif config['verbose'] == 1:
        level = logging.DEBUG
    elif config['verbose'] == -1:
        level = logging.WARN
    elif config['verbose'] <= -2:
        level = logging.ERROR
    else:
        level = logging.INFO
    logging.basicConfig(level=level,
                        format='%(asctime)s %(levelname)-8s %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S', filemode='a+')

    check_config(config)

    return config

Example 49

Project: ppagent
Source File: main.py
View license
def entry():
    parser = argparse.ArgumentParser(
        prog='ppagent', description='A daemon for reporting mining statistics to your pool.')
    parser.add_argument('-l',
                        '--log-level',
                        choices=['DEBUG', 'INFO', 'WARN', 'ERROR'],
                        default='INFO')
    parser.add_argument('-a', '--address', default=None)
    parser.add_argument('-c', '--config', default=None)
    parser.add_argument('-p', '--port', type=int, default=None)
    parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(version))
    subparsers = parser.add_subparsers(title='main subcommands', dest='action')

    subparsers.add_parser('run', help='start the daemon')
    inst = subparsers.add_parser('install', help='install the upstart script and add user')
    inst.add_argument('type', choices=['upstart', 'sysv'],
                      help='upstart for ubuntu, sysv for debian.')
    if len(sys.argv) == 1:
        args = parser.parse_args(['run'])
    else:
        args = parser.parse_args()
    configs = vars(args)

    if configs['action'] == 'install':
        try:
            install(configs)
        except Exception:
            logger.info("Installation failed because of an unhandled exception:")
            raise
        easy_exit(0)

    # rely on command line log level until we get all configs parsed
    ch.setLevel(getattr(logging, args.log_level))
    logger.setLevel(getattr(logging, args.log_level))

    # if they didn't specify a config file lets create a default in the os
    # specific default locations
    if configs['config'] is None:
        if sys.platform == "win32":
            configs['config'] = 'ppagent.json'
            path = '.'
            name = 'ppagent.json'
        else:
            configs['config'] = os.path.join(config_home, 'config.json')
            path = config_home
            name = 'config.json'

        if not os.path.isfile(configs['config']):
            setup_folders(path, name)

    # setup or load our configuration file
    try:
        file_configs = json.load(open(configs['config']))
        logger.debug("Loaded JSON config file from {0}".format(configs['config']))
    except (IOError, OSError):
        logger.error("JSON configuration file {0} couldn't be loaded, no miners configured, exiting..."
                     .format(configs['config']), exc_info=True)
        easy_exit(1)

    # setup our collected configs by recursively overriding
    def update(d, u):
        """ Simple recursive dictionary update """
        for k, v in u.iteritems():
            if isinstance(v, collections.Mapping):
                r = update(d.get(k, {}), v)
                d[k] = r
            else:
                d[k] = u[k]
        return d

    # do some processing on the config file
    miners = []
    for section in file_configs:
        title = section.keys()[0]
        content = section.values()[0]
        # process a miner directive
        if title == "miner":
            typ = content.pop('type', 'CGMiner')
            kwargs = {}
            # apply defaults, followed by overriding with config options
            update(kwargs, miner_defaults[typ])
            update(kwargs, content)
            kwargs['remotes'] = [configs['address']]
            # init the miner class with the configs
            miners.append(globals()[typ](**kwargs))
        elif title == "daemon":
            configs.update(content)

    if not configs['address']:
        # try and fetch the address from our first miner entry. A bit of a hack
        # until 0.4 can be released to handle this more robustly
        while True:
            try:
                url = miners[0].fetch_pool()
            except Exception:
                logger.info("Couldn't fetch pool info from cgminer!", exc_info=True)
                time.sleep(1)
                continue
            configs['address'] = url.hostname
            configs['port'] = url.port + 1111
            break

    # set a default of 4444 for the port
    if not configs['port']:
        configs['port'] = 4444

    for miner in miners:
        miner.remotes = [configs['address']]

    # set our logging level based on the configs
    ch.setLevel(getattr(logging, configs['log_level']))
    logger.setLevel(getattr(logging, configs['log_level']))
    logger.debug(configs)

    sender = AgentSender(miners, configs['address'], configs['port'])
    sender.loop()

Example 50

Project: github-notify
Source File: github-notify.py
View license
def scan_github_issues(config):
    pattern = re.compile(config['regex_pattern'], flags=re.I)
    os.chdir(os.path.dirname(os.path.abspath(__file__)))
    try:
        with open('github-notify.json') as f:
            known = json.load(f)
    except IOError, ValueError:
        known = {}

    g = Github(client_id=config['github_client_id'],
               client_secret=config['github_client_secret'],
               per_page=100)

    if not isinstance(config['github_repository'], list):
        repos = [config['github_repository']]
    else:
        repos = config['github_repository']

    for repo_name in repos:
        repo = g.get_repo(repo_name)

        for pull in repo.get_pulls():
            if pull.number in known.get(repo_name, []):
                continue

            try:
                if pattern.search(pull.title):
                    alert(config, pull, known, repo_name)
                    continue
            except TypeError:
                pass

            try:
                if pattern.search(pull.body):
                    alert(config, pull, known, repo_name)
                    continue
            except TypeError:
                pass

            for pull_file in pull.get_files():
                if pattern.search(pull_file.filename):
                    alert(config, pull, known, repo_name)
                    break

            for comment in pull.get_comments():
                try:
                    if pattern.search(comment.body):
                        alert(config, pull, known, repo_name)
                        break
                except TypeError:
                    pass

            for commit in pull.get_commits():
                try:
                    if pattern.search(commit.commit.message):
                        alert(config, pull, known, repo_name)
                        break
                except TypeError:
                    pass

            for comment in pull.get_issue_comments():
                try:
                    if pattern.search(comment.body):
                        alert(config, pull, known, repo_name)
                        break
                except TypeError:
                    pass

        for issue in repo.get_issues():
            if issue.number in known.get(repo_name, []):
                continue

            if issue.pull_request is not None:
                continue

            try:
                if pattern.search(issue.title):
                    alert(config, issue, known, repo_name)
                    continue
            except TypeError:
                pass

            try:
                if pattern.search(issue.body):
                    alert(config, issue, known, repo_name)
                    continue
            except TypeError:
                pass

            for comment in issue.get_comments():
                try:
                    if pattern.search(comment.body):
                        alert(config, issue, known, repo_name)
                        break
                except TypeError:
                    pass

    try:
        with open('github-notify.json', 'w+') as f:
            json.dump(known, f, indent=4, sort_keys=True)
    except (IOError, ValueError):
        pass