django.utils.six.moves.queue.Queue

Here are the examples of the python api django.utils.six.moves.queue.Queue taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

8 Examples 7

Example 1

Project: django-mysql
Source File: utils.py
View license
    @classmethod
    def get_thread(cls):
        with cls.life_lock:
            if cls.the_thread is None:
                in_queue = Queue()
                out_queue = Queue()
                thread = cls(in_queue, out_queue)
                thread.daemon = True
                thread.in_queue = in_queue
                thread.out_queue = out_queue
                thread.start()
                cls.the_thread = thread

        return cls.the_thread

Example 2

Project: django-mysql
Source File: test_locks.py
View license
    def test_timeout_with_threads(self):
        to_me = queue.Queue()
        to_you = queue.Queue()

        def lock_until_told():
            with Lock('threading_test'):
                to_me.put("Locked")
                to_you.get(True)

        threading_test = Lock('threading_test', 0.05)
        assert not threading_test.is_held()

        other_thread = Thread(target=lock_until_told)
        other_thread.start()
        try:
            item = to_me.get(True)
            assert item == "Locked"

            cursor = connection.cursor()
            cursor.execute("SELECT CONNECTION_ID();")
            own_connection_id = cursor.fetchone()[0]

            assert threading_test.is_held()
            assert threading_test.holding_connection_id() != own_connection_id

            with pytest.raises(TimeoutError):
                with threading_test:
                        pass

            to_you.put("Stop")
        finally:
            other_thread.join()

        assert not threading_test.is_held()
        with threading_test:
            pass

Example 3

Project: django-mysql
Source File: test_locks.py
View license
    def test_threads_concurrent_access(self):
        """
        Test that the same lock object can be used in multiple threads, allows
        the definition of a lock upfront in a module.
        """
        to_me = queue.Queue()
        to_you = queue.Queue()
        the_lock = Lock('THElock', 0.05)

        def check_it_lock_it():
            assert not the_lock.is_held()
            with the_lock:
                to_me.put("Locked")
                to_you.get(True)

        other_thread = Thread(target=check_it_lock_it)
        other_thread.start()
        try:
            item = to_me.get(True)
            assert item == "Locked"

            cursor = connection.cursor()
            cursor.execute("SELECT CONNECTION_ID()")
            own_connection_id = cursor.fetchone()[0]

            assert the_lock.is_held()
            assert the_lock.holding_connection_id() != own_connection_id

            with pytest.raises(TimeoutError):
                with the_lock:
                    pass

            to_you.put("Stop")
        finally:
            other_thread.join()

        with the_lock:
            pass

Example 4

Project: django-mysql
Source File: test_locks.py
View license
    def test_read_and_write_with_threads(self):
        to_me = queue.Queue()

        def get_read_lock():
            with TableLock(read=[Alphabet]):
                to_me.put("Reading")

        try:
            # Thread can obtain lock alone
            other_thread = Thread(target=get_read_lock)
            other_thread.start()
            assert to_me.get() == "Reading"

            with TableLock(write=[Alphabet]):
                other_thread = Thread(target=get_read_lock)
                other_thread.start()

                # Doesn't lock now
                with pytest.raises(queue.Empty):
                    to_me.get(timeout=0.2)

            # Gets now
            assert to_me.get() == "Reading"
        finally:
            other_thread.join()

Example 5

Project: django-haystack
Source File: test_indexes.py
View license
    def test_thread_safety(self):
        # This is a regression. ``SearchIndex`` used to write to
        # ``self.prepared_data``, which would leak between threads if things
        # went too fast.
        exceptions = []

        def threaded_prepare(index_queue, index, model):
            try:
                index.queue = index_queue
                prepped = index.prepare(model)
            except Exception as e:
                exceptions.append(e)
                raise

        class ThreadedSearchIndex(GoodMockSearchIndex):
            def prepare_author(self, obj):
                if obj.pk == 20:
                    time.sleep(0.1)
                else:
                    time.sleep(0.5)

                index_queue.put(self.prepared_data['author'])
                return self.prepared_data['author']

        tmi = ThreadedSearchIndex()
        index_queue = queue.Queue()
        mock_1 = MockModel()
        mock_1.pk = 20
        mock_1.author = 'foo'
        mock_1.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0)
        mock_2 = MockModel()
        mock_2.pk = 21
        mock_2.author = 'daniel%s' % mock_2.id
        mock_2.pub_date = datetime.datetime(2009, 1, 31, 4, 19, 0)

        th1 = Thread(target=threaded_prepare, args=(index_queue, tmi, mock_1))
        th2 = Thread(target=threaded_prepare, args=(index_queue, tmi, mock_2))

        th1.start()
        th2.start()
        th1.join()
        th2.join()

        mock_1_result = index_queue.get()
        mock_2_result = index_queue.get()
        self.assertEqual(mock_1_result, u'foo')
        self.assertEqual(mock_2_result, u'daniel21')

Example 6

Project: django-haystack
Source File: test_views.py
View license
    def test_thread_safety(self):
        exceptions = []

        def threaded_view(resp_queue, view, request):
            import time; time.sleep(2)
            try:
                inst = view(request)
                resp_queue.put(request.GET['name'])
            except Exception as e:
                exceptions.append(e)
                raise

        class ThreadedSearchView(SearchView):
            def __call__(self, request):
                print("Name: %s" % request.GET['name'])
                return super(ThreadedSearchView, self).__call__(request)

        view = search_view_factory(view_class=ThreadedSearchView)
        resp_queue = queue.Queue()
        request_1 = HttpRequest()
        request_1.GET = {'name': 'foo'}
        request_2 = HttpRequest()
        request_2.GET = {'name': 'bar'}

        th1 = Thread(target=threaded_view, args=(resp_queue, view, request_1))
        th2 = Thread(target=threaded_view, args=(resp_queue, view, request_2))

        th1.start()
        th2.start()
        th1.join()
        th2.join()

        foo = resp_queue.get()
        bar = resp_queue.get()
        self.assertNotEqual(foo, bar)

Example 7

Project: django-smartfields
Source File: base.py
View license
    def process(self, in_file, **kwargs):
        out_file = self.get_output_file(in_file, **kwargs)
        cmd = self.cmd_template.format(
            input=self.get_input_path(in_file), output=self.get_output_path(out_file),
            **kwargs
        ).split()
        stdout_pipe, stdout_queue, stdout_reader = None, None, None
        stderr_pipe, stderr_queue, stderr_reader = None, None, None
        if callable(self.stdout_handler):
            stdout_pipe = subprocess.PIPE
            stdout_queue = queue.Queue()
        if self.stderr_handler is True:
            stderr_pipe = subprocess.STDOUT
        elif callable(self.stderr_handler):
            stderr_pipe = subprocess.PIPE
            stderr_queue = queue.Queue()
        proc = subprocess.Popen(
            cmd, stdout=stdout_pipe, stderr=stderr_pipe, universal_newlines=True)
        if stdout_queue is not None:
            stdout_reader = AsynchronousFileReader(proc.stdout, stdout_queue)
            stdout_reader.start()
        if stderr_queue is not None:
            stderr_reader = AsynchronousFileReader(proc.stderr, stderr_queue)
            stderr_reader.start()
        if stdout_reader is None and stderr_reader is None:
            proc.wait()
        else:
            stdout_args, stderr_args = () , ()
            try:
                while not (stdout_reader is None or stdout_reader.eof()) or \
                      not (stderr_reader is None or stderr_reader.eof()):
                    if stdout_queue is not None:
                        while not stdout_queue.empty():
                            stdout_args = self.stdout_handler( # pylint: disable=not-callable
                                stdout_queue.get(), *stdout_args) or ()
                    if stderr_queue is not None:
                        while not stderr_queue.empty():
                            stderr_args = self.stderr_handler( # pylint: disable=not-callable
                                stderr_queue.get(), *stderr_args) or ()
                    time.sleep(self.sleep_time)
            except ProcessingError:
                if proc.poll() is None:
                    proc.terminate()
                raise
            finally:
                # wait for process to finish, so we can check the return value
                if proc.poll() is None:
                    proc.wait()
                if stdout_reader is not None:
                    stdout_reader.join()
                    proc.stdout.close()
                if stderr_reader is not None:
                    stderr_reader.join()
                    proc.stderr.close()
        if proc.returncode < 0:
            raise ProcessingError("There was a problem processing this file.")
        return out_file

Example 8

Project: django-smartfields
Source File: base.py
View license
    def process(self, in_file, **kwargs):
        out_file = self.get_output_file(in_file, **kwargs)
        cmd = self.cmd_template.format(
            input=self.get_input_path(in_file), output=self.get_output_path(out_file),
            **kwargs
        ).split()
        stdout_pipe, stdout_queue, stdout_reader = None, None, None
        stderr_pipe, stderr_queue, stderr_reader = None, None, None
        if callable(self.stdout_handler):
            stdout_pipe = subprocess.PIPE
            stdout_queue = queue.Queue()
        if self.stderr_handler is True:
            stderr_pipe = subprocess.STDOUT
        elif callable(self.stderr_handler):
            stderr_pipe = subprocess.PIPE
            stderr_queue = queue.Queue()
        proc = subprocess.Popen(
            cmd, stdout=stdout_pipe, stderr=stderr_pipe, universal_newlines=True)
        if stdout_queue is not None:
            stdout_reader = AsynchronousFileReader(proc.stdout, stdout_queue)
            stdout_reader.start()
        if stderr_queue is not None:
            stderr_reader = AsynchronousFileReader(proc.stderr, stderr_queue)
            stderr_reader.start()
        if stdout_reader is None and stderr_reader is None:
            proc.wait()
        else:
            stdout_args, stderr_args = () , ()
            try:
                while not (stdout_reader is None or stdout_reader.eof()) or \
                      not (stderr_reader is None or stderr_reader.eof()):
                    if stdout_queue is not None:
                        while not stdout_queue.empty():
                            stdout_args = self.stdout_handler( # pylint: disable=not-callable
                                stdout_queue.get(), *stdout_args) or ()
                    if stderr_queue is not None:
                        while not stderr_queue.empty():
                            stderr_args = self.stderr_handler( # pylint: disable=not-callable
                                stderr_queue.get(), *stderr_args) or ()
                    time.sleep(self.sleep_time)
            except ProcessingError:
                if proc.poll() is None:
                    proc.terminate()
                raise
            finally:
                # wait for process to finish, so we can check the return value
                if proc.poll() is None:
                    proc.wait()
                if stdout_reader is not None:
                    stdout_reader.join()
                    proc.stdout.close()
                if stderr_reader is not None:
                    stderr_reader.join()
                    proc.stderr.close()
        if proc.returncode < 0:
            raise ProcessingError("There was a problem processing this file.")
        return out_file