System.Threading.CancellationTokenSource.CreateLinkedTokenSource(params System.Threading.CancellationToken[])

public void Start(CancellationToken cancellationToken)
        {
            if (this._source != null)
            {
                throw new InvalidOperationException(
                    "Already started. Call Stop first.");
            }

            this._logger.LogDebug("Starting token cleanup");

            this._source = CancellationTokenSource
                .CreateLinkedTokenSource(cancellationToken);

            Task.Factory.StartNew(() => StartInternal(this._source.Token));
        }

public async Task StartAsync(CancellationToken stopToken)
        {
            if (Disposed)
            {
                throw new ObjectDisposedException(DisplayName);
            }

            // create a token that is cancelled when either _stopTokenSource or stopToken is cancelled
            _stopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stopToken);

            await _bookmarkManager.StartAsync(_stopTokenSource.Token);

            LoadCredentialProviders();

            await LoadGenericPlugins(_stopTokenSource.Token);

            await LoadBuiltInSinks(_stopTokenSource.Token);

            await LoadEventSinks(_stopTokenSource.Token);

            (var sourcesLoaded, var sourcesFailed) = LoadEventSources();

            await LoadPipes(_stopTokenSource.Token);

            await StartEventSources(sourcesLoaded, sourcesFailed, _stopTokenSource.Token);

            _logger.LogInformation("Started");
        }

public IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var sourceCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new SampleEnumerator(_source.GetAsyncEnumerator(sourceCTS.Token), _period, _emitLast, sourceCTS);
            en.StartTimer();
            en.MoveNext();
            return en;
        }

private static async Task<HealthCheckResult> ExecuteHealthCheckWithRetriesAsync(
            Uri uri,
            int retries,
            TimeSpan delayBetweenRetries,
            TimeSpan timeoutPerRequest,
            bool degradedOnError,
            CancellationToken cancellationToken)
        {
            var sw = new Stopwatch();
            var attempts = 0;
            try
            {
                sw.Start();

                do
                {
                    try
                    {
                        attempts++;

                        using (var tokenWithTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
                        {
                            tokenWithTimeout.CancelAfter(timeoutPerRequest);

                            var response = await HttpClient.GetAsync(uri, tokenWithTimeout.Token);

                            if (attempts == retries || response.IsSuccessStatusCode)
                            {
                                return response.IsSuccessStatusCode
                                    ? HealthCheckResult.Healthy(
                                        $"OK. '{uri}' success. Total Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.")
                                    : HealthCheckResultOnError(
                                        $"FAILED. '{uri}' status code was {response.StatusCode}. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
                                        degradedOnError);
                            }

                            if (response.StatusCode == HttpStatusCode.GatewayTimeout ||
                                response.StatusCode == HttpStatusCode.ServiceUnavailable)
                            {
                                Logger.Error(
                                    $"HTTP Health Check '{uri}' failed with status code {response.StatusCode}. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.");

                                Logger.Info(
                                    $"Retrying HTTP Health Check '{uri}' in {delayBetweenRetries.TotalMilliseconds}ms. {attempts} / {retries} retries.");

                                await Task.Delay(delayBetweenRetries, cancellationToken);
                            }
                        }
                    }
                    catch (Exception ex) when (ex is TaskCanceledException)
                    {
                        Logger.ErrorException(
                            $"HTTP Health Check '{uri}' did not respond within '{timeoutPerRequest.TotalMilliseconds}'ms. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
                            ex);

                        if (attempts == retries)
                        {
                            return HealthCheckResultOnError(
                                $"FAILED. '{uri}' did not respond within {timeoutPerRequest.TotalMilliseconds}ms. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
                                degradedOnError);
                        }

                        await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
                    }
                    catch (Exception ex) when (ex is TimeoutException)
                    {
                        Logger.ErrorException(
                            $"HTTP Health Check '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
                            ex);

                        if (attempts == retries)
                        {
                            return HealthCheckResultOnError(
                                $"FAILED. '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
                                degradedOnError);
                        }

                        await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
                    }
                    catch (Exception ex) when (ex is HttpRequestException)
                    {
                        Logger.ErrorException(
                            $"HTTP Health Check '{uri}' failed. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
                            ex);

                        if (attempts == retries)
                        {
                            return HealthCheckResultOnError(
                                $"FAILED. '{uri}' request failed with an unexpected error. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.",
                                degradedOnError);
                        }

                        await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
                    }
                    catch (Exception ex)
                    {
                        var message =
                            $"HTTP Health Check failed to request '{uri}'. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.";

                        if (attempts == retries)
                        {
                            Logger.ErrorException(message, ex);

                            return HealthCheckResultOnError(
                                $"FAILED. {message}.",
                                degradedOnError);
                        }

                        await Retry(uri, retries, delayBetweenRetries, attempts, ex, cancellationToken);
                    }
                }
                while (true);
            }
            catch (Exception ex) when (ex is TaskCanceledException)
            {
                Logger.ErrorException(
                    $"HTTP Health Check '{uri}' did not respond within '{timeoutPerRequest.TotalMilliseconds}'ms. Attempts: {attempts}.",
                    ex);

                return HealthCheckResultOnError(
                    $"FAILED. '{uri}' did not respond within {timeoutPerRequest.TotalMilliseconds}ms. Attempts: {attempts}.",
                    degradedOnError);
            }
            catch (Exception ex)
            {
                var message = $"HTTP Health Check failed to request '{uri}'. Time taken: {sw.ElapsedMilliseconds}ms. Attempts: {attempts}.";

                Logger.ErrorException(message, ex);

                return HealthCheckResultOnError(
                    $"FAILED. {message}",
                    degradedOnError);
            }
            finally
            {
                sw.Stop();
            }
        }

public ValueTask<bool> MoveNextAsync()
            {
                var idx = Volatile.Read(ref _index);

                var result = new TaskCompletionSource<bool>();

                _token = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);

                Interlocked.Increment(ref _disposeWip);

                Task.Delay(_timeout, _token.Token)
                    .ContinueWith(t => Timeout(idx, result), _token.Token);

                _source.MoveNextAsync()
                    .AsTask().ContinueWith(t => Next(idx, t, result));

                return new ValueTask<bool>(result.Task);
            }

public IAsyncEnumerator<TSource> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var cancelMain = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var cancelOther = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new TakeUntilEnumerator(_source.GetAsyncEnumerator(cancelMain.Token), _other.GetAsyncEnumerator(cancelOther.Token), cancelMain, cancelOther);
            en.MoveNextOther();
            return en;
        }

public static async ValueTask<bool> GetIsEC2Instance(CancellationToken cancellationToken)
        {
            // return the cache value if possible
            if (_isEC2 >= 0)
            {
                return _isEC2 > 0;
            }

            // try to fetch the EC2 idenreplacedy doreplacedent
            // see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-idenreplacedy-doreplacedents.html

            const string doreplacedentEndpoint = "http://169.254.169.254/latest/dynamic/instance-idenreplacedy";
            using var client = new HttpClient();
            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);

            // allow 2 seconds for the request to complete, if this is EC2 instance that should be way more than enough
            cts.CancelAfter(2000);

            try
            {
                _ = await client.GetStringAsync(doreplacedentEndpoint, cts.Token);
                Interlocked.Exchange(ref _isEC2, 1);
                return true;
            }
            catch (Exception)
            {
                Interlocked.Exchange(ref _isEC2, 0);
            }

            return false;
        }

public IAsyncEnumerator<TCollection> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var mainCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var otherCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new BufferBoundaryExactEnumerator(_source.GetAsyncEnumerator(mainCancel.Token), 
                _boundary.GetAsyncEnumerator(otherCancel.Token), _collectionSupplier, _maxSize, mainCancel, otherCancel);

            en.MoveNextOther();
            en.MoveNextSource();
            return en;
        }

public override async Task RunAsync(CancellationToken shutdownToken)
        {
            await this.observer.OpenAsync(this.lease.CurrentLeaseToken).ConfigureAwait(false);

            this.processorCancellation = CancellationTokenSource.CreateLinkedTokenSource(shutdownToken);

            Task processorTask = this.processor.RunAsync(this.processorCancellation.Token);
            processorTask.ContinueWith(_ => this.renewerCancellation.Cancel()).LogException();

            Task renewerTask = this.renewer.RunAsync(this.renewerCancellation.Token);
            renewerTask.ContinueWith(_ => this.processorCancellation.Cancel()).LogException();

            ChangeFeedObserverCloseReason closeReason = shutdownToken.IsCancellationRequested ?
                ChangeFeedObserverCloseReason.Shutdown :
                ChangeFeedObserverCloseReason.Unknown;

            try
            {
                await Task.WhenAll(processorTask, renewerTask).ConfigureAwait(false);
            }
            catch (LeaseLostException)
            {
                closeReason = ChangeFeedObserverCloseReason.LeaseLost;
                throw;
            }
            catch (FeedRangeGoneException)
            {
                closeReason = ChangeFeedObserverCloseReason.LeaseGone;
                throw;
            }
            catch (CosmosException)
            {
                closeReason = ChangeFeedObserverCloseReason.CosmosException;
                throw;
            }
            catch (OperationCanceledException) when (shutdownToken.IsCancellationRequested)
            {
                closeReason = ChangeFeedObserverCloseReason.Shutdown;
            }
            catch (ChangeFeedProcessorUserException)
            {
                closeReason = ChangeFeedObserverCloseReason.ObserverError;
                throw;
            }
            catch (Exception) when (processorTask.IsFaulted)
            {
                closeReason = ChangeFeedObserverCloseReason.Unknown;
                throw;
            }
            finally
            {
                await this.observer.CloseAsync(this.lease.CurrentLeaseToken, closeReason).ConfigureAwait(false);
            }
        }

private void NextHandler(Task<bool> t)
            {
                if (t.IsCanceled)
                {
                    ExceptionHelper.AddException(ref _error, new OperationCanceledException());
                    _sourceDone = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                } else
                if (t.IsFaulted)
                {
                    ExceptionHelper.AddException(ref _error, ExceptionHelper.Extract(t.Exception));
                    _sourceDone = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
                else if (t.Result)
                {
                    var cts = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);
                    IAsyncEnumerator<TResult> src;
                    try
                    {
                        src = _mapper(_source.Current).GetAsyncEnumerator(cts.Token);
                    }
                    catch (Exception ex)
                    {
                        ExceptionHelper.AddException(ref _error, ex);
                        _sourceDone = true;
                        src = null;
                        if (TryDispose())
                        {
                            ResumeHelper.Resume(ref _resume);
                            return;
                        }
                    }

                    if (src != null)
                    {
                        Interlocked.Increment(ref _disposeWip);
                        var inner = new InnerHandler(src, this, cts);
                        _inners.Enqueue(inner);

                        if (_disposeRequested)
                        {
                            while (_inners.TryDequeue(out var inner2))
                            {
                                inner2.Dispose();
                            }
                        }

                        if (TryDispose())
                        {
                            inner.MoveNext();
                            if (Interlocked.Decrement(ref _sourceOutstanding) != 0)
                            {
                                MoveNextSource();
                            }
                            ResumeHelper.Resume(ref _resume);
                        }
                    }
                }
                else
                {
                    _sourceDone = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
            }

public async Task StartAsync(CancellationToken? token = null)
        {
            _tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token ?? new CancellationToken());
            _token = _tokenSource.Token;
            _tcpListener.Start();
            _listening = true;

            try
            {
                while (!_token.IsCancellationRequested)
                {
                    var tcpClientTask = _tcpListener.AcceptTcpClientAsync();
                    var result = await tcpClientTask;
                    Console.WriteLine("New client");
                    _ = Task.Run(() =>
                      {
                          HandleReceivedClient(result);
                      }, _token);
                }
            }
            finally
            {
                _tcpListener.Stop();
                _listening = false;
            }
        }

public static async Task Main(string[] _)
        {
            var ctx = new CancellationToken();
            var ctxs = CancellationTokenSource.CreateLinkedTokenSource(ctx);

            Console.CancelKeyPress += (x, y) =>
            {
                y.Cancel = true;
                ctxs.Cancel(false);
            };

            var builder = new ContainerBuilder();

            builder.RegisterMediatR(typeof(CustomerLoadQuery).replacedembly);

            builder.RegisterType<CustomersRepository>()
                .As<ICustomersRepository>()
                .SingleInstance();

            var container = builder.Build();

            var lifetimeScope = container.Resolve<ILifetimeScope>();

            var googleCustomerAddCommand = new CustomerAddCommand(Guid.NewGuid(), "google");

            await using (var scope = lifetimeScope.BeginLifetimeScope())
            {
                var mediator = scope.Resolve<IMediator>();

                await mediator.Send(googleCustomerAddCommand, ctx);
            }

            await using (var scope = lifetimeScope.BeginLifetimeScope())
            {
                var mediator = scope.Resolve<IMediator>();

                var customer = await mediator.Send(new CustomerLoadQuery(googleCustomerAddCommand.Id), ctx);

                Console.WriteLine(googleCustomerAddCommand.Name == customer.Name);

                try
                {
                    await mediator.Send(new CustomerLoadQuery(Guid.Empty), ctx);
                }
                catch (CustomerNotFoundException)
                {
                    Console.WriteLine("Expected that the customer could not be found bc we didn't add him b4.");
                }
            }

            Console.ReadKey();
        }

private void Handle(Task<bool> task)
            {
                if (task.IsCanceled)
                {
                    AddException(new OperationCanceledException());
                    _done = true;
                    if (TryDispose())
                    {
                        Signal();
                    }
                }
                else if (task.IsFaulted)
                {
                    AddException(task.Exception);
                    _done = true;
                    if (TryDispose())
                    {
                        Signal();
                    }
                }
                else
                {
                    if (task.Result)
                    {
                        var v = _source.Current;

                        if (TryDispose())
                        {
                            var cts = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);
                            IAsyncEnumerator<TResult> innerSource;
                            try
                            {
                                innerSource = _mapper(v)
                                    .GetAsyncEnumerator(cts.Token);
                            }
                            catch (Exception ex)
                            {
                                _source.DisposeAsync();

                                AddException(ex);
                                _done = true;
                                Signal();
                                return;
                            }

                            var handler = new InnerHandler(this, innerSource, _prefetch, cts);
                            Interlocked.Increment(ref _allDisposeWip);
                            if (Add(handler))
                            {
                                handler.MoveNext();

                                if (Interlocked.Decrement(ref _outstanding) != 0)
                                {
                                    MoveNext();
                                }
                            }
                            else
                            {
                                // This will decrement _allDisposeWip so
                                // that the DisposeAsync() can be released eventually
                                DisposeOne();
                            }
                        }
                    }
                    else
                    {
                        _done = true;
                        if (TryDispose())
                        {
                            Signal();
                        }
                    }
                }
            }

private static async Task<HealthCheckResult> ExecuteHttpCheckNoRetriesAsync(
            Uri uri,
            TimeSpan timeout,
            bool degradedOnError,
            CancellationToken cancellationToken)
        {
            var sw = new Stopwatch();

            try
            {
                using (var tokenWithTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
                {
                    tokenWithTimeout.CancelAfter(timeout);

                    sw.Start();

                    var response = await HttpClient.GetAsync(uri, tokenWithTimeout.Token).ConfigureAwait(false);

                    return response.IsSuccessStatusCode
                        ? HealthCheckResult.Healthy($"OK. '{uri}' success. Time taken: {sw.ElapsedMilliseconds}ms.")
                        : HealthCheckResultOnError(
                            $"FAILED. '{uri}' status code was {response.StatusCode}. Time taken: {sw.ElapsedMilliseconds}ms.",
                            degradedOnError);
                }
            }
            catch (Exception ex) when (ex is TaskCanceledException)
            {
                Logger.ErrorException($"HTTP Health Check '{uri}' did not respond within '{timeout.TotalMilliseconds}'ms.", ex);

                return HealthCheckResultOnError($"FAILED. '{uri}' did not respond within {timeout.TotalMilliseconds}ms", degradedOnError);
            }
            catch (Exception ex) when (ex is TimeoutException)
            {
                Logger.ErrorException($"HTTP Health Check '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms.", ex);

                return HealthCheckResultOnError($"FAILED. '{uri}' timed out. Time taken: {sw.ElapsedMilliseconds}ms.", degradedOnError);
            }
            catch (Exception ex) when (ex is HttpRequestException)
            {
                Logger.ErrorException($"HTTP Health Check '{uri}' failed. Time taken: {sw.ElapsedMilliseconds}ms.", ex);

                return HealthCheckResultOnError(
                    $"FAILED. '{uri}' request failed with an unexpected error. Time taken: {sw.ElapsedMilliseconds}ms.",
                    degradedOnError);
            }
            catch (Exception ex)
            {
                var message = $"HTTP Health Check failed to request '{uri}'. Time taken: {sw.ElapsedMilliseconds}ms.";

                Logger.ErrorException(message, ex);

                return HealthCheckResultOnError($"FAILED. {message}", degradedOnError);
            }
            finally
            {
                sw.Stop();
            }
        }

public static void DeleteDirectory(string path, bool contentsOnly, bool continueOnContentDeleteError, CancellationToken cancellationToken)
        {
            ArgUtil.NotNullOrEmpty(path, nameof(path));
            DirectoryInfo directory = new DirectoryInfo(path);
            if (!directory.Exists)
            {
                return;
            }

            if (!contentsOnly)
            {
                // Remove the readonly flag.
                RemoveReadOnly(directory);

                // Check if the directory is a reparse point.
                if (directory.Attributes.HasFlag(FileAttributes.ReparsePoint))
                {
                    // Delete the reparse point directory and short-circuit.
                    directory.Delete();
                    return;
                }
            }

            // Initialize a concurrent stack to store the directories. The directories
            // cannot be deleted until the files are deleted.
            var directories = new ConcurrentStack<DirectoryInfo>();

            if (!contentsOnly)
            {
                directories.Push(directory);
            }

            // Create a new token source for the parallel query. The parallel query should be
            // canceled after the first error is encountered. Otherwise the number of exceptions
            // could get out of control for a large directory with access denied on every file.
            using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                try
                {
                    // Recursively delete all files and store all subdirectories.
                    Enumerate(directory, tokenSource)
                        .AsParallel()
                        .WithCancellation(tokenSource.Token)
                        .ForAll((FileSystemInfo item) =>
                        {
                            bool success = false;
                            try
                            {
                                // Remove the readonly attribute.
                                RemoveReadOnly(item);

                                // Check if the item is a file.
                                if (item is FileInfo)
                                {
                                    // Delete the file.
                                    item.Delete();
                                }
                                else
                                {
                                    // Check if the item is a directory reparse point.
                                    var subdirectory = item as DirectoryInfo;
                                    ArgUtil.NotNull(subdirectory, nameof(subdirectory));
                                    if (subdirectory.Attributes.HasFlag(FileAttributes.ReparsePoint))
                                    {
                                        try
                                        {
                                            // Delete the reparse point.
                                            subdirectory.Delete();
                                        }
                                        catch (DirectoryNotFoundException)
                                        {
                                            // The target of the reparse point directory has been deleted.
                                            // Therefore the item is no longer a directory and is now a file.
                                            //
                                            // Deletion of reparse point directories happens in parallel. This case can occur
                                            // when reparse point directory FOO points to some other reparse point directory BAR,
                                            // and BAR is deleted after the DirectoryInfo for FOO has already been initialized.
                                            File.Delete(subdirectory.FullName);
                                        }
                                    }
                                    else
                                    {
                                        // Store the directory.
                                        directories.Push(subdirectory);
                                    }
                                }

                                success = true;
                            }
                            catch (Exception) when (continueOnContentDeleteError)
                            {
                                // ignore any exception when continueOnContentDeleteError is true.
                                success = true;
                            }
                            finally
                            {
                                if (!success)
                                {
                                    tokenSource.Cancel(); // Cancel is thread-safe.
                                }
                            }
                        });
                }
                catch (Exception)
                {
                    tokenSource.Cancel();
                    throw;
                }
            }

            // Delete the directories.
            foreach (DirectoryInfo dir in directories.OrderByDescending(x => x.FullName.Length))
            {
                cancellationToken.ThrowIfCancellationRequested();
                dir.Delete();
            }
        }

public Task Start(CancellationToken cancellationToken)
        {
            if (_tokenSource != null)
            {
                var message = string.Format(CultureInfo.CurrentCulture, AzureStorageResources.DequeueManager_Started, this.GetType().Name);
                _logger.Error(message);
                throw new InvalidOperationException(message);
            }

            _tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            return DequeueAndSendWebHooks(_tokenSource.Token);
        }

public IAsyncEnumerator<TResult> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var cancelMain = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var cancelOther = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new WithLatestFromEnumerator(_source.GetAsyncEnumerator(cancelMain.Token), _other.GetAsyncEnumerator(cancelOther.Token), _func, cancelMain, cancelOther);
            en.MoveNextOther();
            en.MoveNextMain();
            return en;
        }

private async Task TerminateSession(ISession session, CancellationToken cancellationToken)
        {
            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            cts.CancelAfter(SessionStopTimeoutMs);

            _logger.LogDebug("Terminating session '{0}'", session.DisplayName);

            await session.StopAsync(cts.Token);
            session.Dispose();
        }

public IExecutionContext CreateEmbeddedChild(string scopeName, string contextName, Guid embeddedId, ActionRunStage stage, Dictionary<string, string> intraActionState = null, string siblingScopeName = null)
        {
            return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token), intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName);
        }

public static void Run(Action<CancellationToken> body, int msTimeout, Action timeout, CancellationToken? cancel = null)
    {
      body.NonNull(nameof(body));
      timeout.NonNull(nameof(timeout));
      msTimeout.IsTrue( a => a > 0, nameof(msTimeout));

      using (var cts = cancel.HasValue ? CancellationTokenSource.CreateLinkedTokenSource(cancel.Value) : new CancellationTokenSource())
      {
        Task.Delay(msTimeout, cts.Token)
            .ContinueWith(d => { try{ if (!d.IsCanceled) timeout(); }catch{ } });

        try
        {
          body(cts.Token);
        }
        finally
        {
          cts.Cancel();
        }
      }
    }

private void NextHandler(Task<bool> t)
            {
                if (t.IsCanceled)
                {
                    ExceptionHelper.AddException(ref _error, new OperationCanceledException());
                    _done = true;
                    if (TryDispose())
                    {
                        Signal();
                    }
                }
                else if (t.IsFaulted)
                {
                    ExceptionHelper.AddException(ref _error, ExceptionHelper.Extract(t.Exception));
                    _done = true;
                    if (TryDispose())
                    {
                        Signal();
                    }
                }
                else if (t.Result)
                {
                    var cts = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);
                    IAsyncEnumerator<TResult> src;
                    try
                    {
                        src = _mapper(_source.Current).GetAsyncEnumerator(cts.Token);
                    }
                    catch (Exception ex)
                    {
                        ExceptionHelper.AddException(ref _error, ex);
                        _done = true;
                        Dispose(_source);
                        Signal();
                        return;
                    }

                    if (TryDispose())
                    {
                        Interlocked.Increment(ref _allDisposeWip);
                        var inner = new InnerHandler(src, this, cts);

                        for (; ; )
                        {
                            var curr = Volatile.Read(ref _current);
                            if (curr == DisposedInnerHandler)
                            {
                                inner.Dispose();
                                break;
                            }
                            if (Interlocked.CompareExchange(ref _current, inner, curr) == curr)
                            {
                                curr?.Dispose();
                                inner.MoveNext();
                                MoveNext();
                                return;
                            }
                        }
                    }
                }
                else
                {
                    _done = true;
                    if (TryDispose())
                    {
                        Signal();
                    }
                }
            }

public IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var sourceCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            return new TimeoutEnumerator(_source.GetAsyncEnumerator(sourceCTS.Token), _timeout, sourceCTS);
        }

public async Task<int> RunAsync(string pipeIn, string pipeOut)
        {
            try
            {
                // Setup way to handle SIGTERM/unloading signals
                _completedCommand.Reset();
                HostContext.Unloading += Worker_Unloading;

                // Validate args.
                ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn));
                ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut));
                VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
                var jobRunner = HostContext.CreateService<IJobRunner>();

                using (var channel = HostContext.CreateService<IProcessChannel>())
                using (var jobRequestCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(HostContext.RunnerShutdownToken))
                using (var channelTokenSource = new CancellationTokenSource())
                {
                    // Start the channel.
                    channel.StartClient(pipeIn, pipeOut);

                    // Wait for up to 30 seconds for a message from the channel.
                    HostContext.WritePerfCounter("WorkerWaitingForJobMessage");
                    Trace.Info("Waiting to receive the job message from the channel.");
                    WorkerMessage channelMessage;
                    using (var csChannelMessage = new CancellationTokenSource(_workerStartTimeout))
                    {
                        channelMessage = await channel.ReceiveAsync(csChannelMessage.Token);
                    }

                    // Deserialize the job message.
                    Trace.Info("Message received.");
                    ArgUtil.Equal(MessageType.NewJobRequest, channelMessage.MessageType, nameof(channelMessage.MessageType));
                    ArgUtil.NotNullOrEmpty(channelMessage.Body, nameof(channelMessage.Body));
                    var jobMessage = StringUtil.ConvertFromJson<Pipelines.AgentJobRequestMessage>(channelMessage.Body);
                    ArgUtil.NotNull(jobMessage, nameof(jobMessage));
                    HostContext.WritePerfCounter($"WorkerJobMessageReceived_{jobMessage.RequestId.ToString()}");

                    // Initialize the secret masker and set the thread culture.
                    InitializeSecretMasker(jobMessage);
                    SetCulture(jobMessage);

                    // Start the job.
                    Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(jobMessage)}");
                    Task<TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);

                    // Start listening for a cancel message from the channel.
                    Trace.Info("Listening for cancel message from the channel.");
                    Task<WorkerMessage> channelTask = channel.ReceiveAsync(channelTokenSource.Token);

                    // Wait for one of the tasks to complete.
                    Trace.Info("Waiting for the job to complete or for a cancel message from the channel.");
                    Task.WaitAny(jobRunnerTask, channelTask);
                    // Handle if the job completed.
                    if (jobRunnerTask.IsCompleted)
                    {
                        Trace.Info("Job completed.");
                        channelTokenSource.Cancel(); // Cancel waiting for a message from the channel.
                        return TaskResultUtil.TranslateToReturnCode(await jobRunnerTask);
                    }

                    // Otherwise a cancel message was received from the channel.
                    Trace.Info("Cancellation/Shutdown message received.");
                    channelMessage = await channelTask;
                    switch (channelMessage.MessageType)
                    {
                        case MessageType.CancelRequest:
                            jobRequestCancellationToken.Cancel();   // Expire the host cancellation token.
                            break;
                        case MessageType.RunnerShutdown:
                            HostContext.ShutdownRunner(ShutdownReason.UserCancelled);
                            break;
                        case MessageType.OperatingSystemShutdown:
                            HostContext.ShutdownRunner(ShutdownReason.OperatingSystemShutdown);
                            break;
                        default:
                            throw new ArgumentOutOfRangeException(nameof(channelMessage.MessageType), channelMessage.MessageType, nameof(channelMessage.MessageType));
                    }

                    // Await the job.
                    return TaskResultUtil.TranslateToReturnCode(await jobRunnerTask);
                }
            }
            finally
            {
                HostContext.Unloading -= Worker_Unloading;
                _completedCommand.Set();
            }
        }

public IAsyncEnumerator<TResult> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var enumerators = new IAsyncEnumerator<TSource>[_sources.Length];
            var tokenSources = new CancellationTokenSource[_sources.Length];
            for (var i = 0; i < _sources.Length; i++)
            {
                var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
                tokenSources[i] = cts;
                enumerators[i] = _sources[i].GetAsyncEnumerator(cts.Token);
            }
            return new ZipArrayEnumerator(enumerators, _zipper, tokenSources);
        }

public IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var sourceCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new DebounceEnumerator(_source.GetAsyncEnumerator(sourceCTS.Token), _delay, _emitLast, sourceCTS);
            en.MoveNext();
            return en;
        }

private async Task<Response> SendRequest(Request request, CancellationToken cancellationToken)
		{
			try
			{
				logger?.LogTrace("ModbusClient.SendRequest enter");
				CheckDisposed();

				lock (reconnectLock)
				{
					if (!IsConnected)
					{
						if (!isReconnecting)
							ConnectingTask = GetReconnectTask(true);

						throw new InvalidOperationException("Modbus client is not connected");
					}
				}

				if (stream == null)
					throw new InvalidOperationException("Modbus client failed to open stream");

				using var sendCts = CancellationTokenSource.CreateLinkedTokenSource(stopCts.Token, cancellationToken);
				
				var queueItem = new QueuedRequest
				{
					TaskCompletionSource = new TaskCompletionSource<Response>()
				};

				lock (trxIdLock)
				{
					queueItem.TransactionId = transactionId++;
				}

				await queueLock.WaitAsync(sendCts.Token);
				try
				{
					awaitingResponses.Add(queueItem);
					logger?.LogDebug($"Added transaction #{queueItem.TransactionId} to receive queue");
				}
				finally
				{
					queueLock.Release();
				}

				await sendLock.WaitAsync(sendCts.Token);
				try
				{
					request.TransactionId = queueItem.TransactionId;
					logger?.LogDebug($"Sending {request}");

					byte[] bytes = request.Serialize();
					using var timeCts = new CancellationTokenSource(SendTimeout);
					using var cts = CancellationTokenSource.CreateLinkedTokenSource(stopCts.Token, timeCts.Token, cancellationToken);
					try
					{
						await stream.WriteAsync(bytes, 0, bytes.Length, cts.Token);
						logger?.LogDebug($"Request for transaction #{request.TransactionId} sent");

						queueItem.TimeoutCancellationTokenSource = new CancellationTokenSource(ReceiveTimeout);
						queueItem.CancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stopCts.Token, cancellationToken, queueItem.TimeoutCancellationTokenSource.Token);
						queueItem.Registration = queueItem.CancellationTokenSource.Token.Register(() => RemoveQueuedItem(queueItem));
					}
					catch (OperationCanceledException) when (timeCts.IsCancellationRequested)
					{
						queueItem.TaskCompletionSource.TrySetCanceled();
						await queueLock.WaitAsync(stopCts.Token);
						try
						{
							awaitingResponses.Remove(queueItem);
						}
						finally
						{
							queueLock.Release();
						}
					}
				}
				catch (Exception ex)
				{
					logger?.LogError(ex, $"Unexpected error ({ex.GetType().Name}) on send: {ex.GetMessage()}");
					queueItem.TaskCompletionSource.TrySetException(ex);
					await queueLock.WaitAsync(stopCts.Token);
					try
					{
						awaitingResponses.Remove(queueItem);
					}
					finally
					{
						queueLock.Release();
					}
				}
				finally
				{
					sendLock.Release();
				}

				return await queueItem.TaskCompletionSource.Task;
			}
			finally
			{
				logger?.LogTrace("ModbusClient.SendRequest leave");
			}
		}

public void OnRequestAuthorization(object sender, RequestAuthorizationEventArgs e)
        {
            if (!(sender is Server authenticatingServer))
                return;

            e.TokenOrigin = RequestAuthorizationEventArgs.TokenOriginType.None;
            e.AccessToken = null;

            lock (Properties.Settings.Default.AccessTokenCache)
            {
                if (e.SourcePolicy != RequestAuthorizationEventArgs.SourcePolicyType.ForceAuthorization)
                {
                    var key = authenticatingServer.Base.AbsoluteUri;
                    if (Properties.Settings.Default.AccessTokenCache.TryGetValue(key, out var accessToken))
                    {
                        if (!e.ForceRefresh && DateTime.Now < accessToken.Expires)
                        {
                            e.TokenOrigin = RequestAuthorizationEventArgs.TokenOriginType.Saved;
                            e.AccessToken = accessToken;
                            return;
                        }

                        // Token refresh was explicitly requested or the token expired. Refresh it.
                        if (accessToken is InvalidToken)
                        {
                            // Invalid token is not refreshable.
                            Properties.Settings.Default.AccessTokenCache.Remove(key);
                        }
                        else
                        {
                            // Get API endpoints. (Not called from the UI thread or already cached by now. Otherwise it would need to be spawned as a background task to avoid deadlock.)
                            var api = authenticatingServer.GetEndpoints(Window.Abort.Token);

                            // Prepare web request.
                            var request = WebRequest.Create(api.TokenEndpoint);
                            request.CachePolicy = Xml.Response.CachePolicy;
                            request.Proxy = null;
                            if (request is HttpWebRequest requestHTTP)
                                requestHTTP.UserAgent = Xml.Response.UserAgent;

                            try
                            {
                                accessToken = accessToken.RefreshToken(request, null, Window.Abort.Token);

                                // Update access token cache.
                                Properties.Settings.Default.AccessTokenCache[key] = accessToken;

                                // If we got here, return the token.
                                e.TokenOrigin = RequestAuthorizationEventArgs.TokenOriginType.Refreshed;
                                e.AccessToken = accessToken;
                                return;
                            }
                            catch (AccessTokenException ex)
                            {
                                if (ex.ErrorCode == AccessTokenException.ErrorCodeType.InvalidGrant)
                                {
                                    // The grant has been revoked. Drop the access token.
                                    Properties.Settings.Default.AccessTokenCache.Remove(key);
                                }
                                else
                                    throw;
                            }
                        }
                    }
                }

                if (e.SourcePolicy != RequestAuthorizationEventArgs.SourcePolicyType.SavedOnly)
                {
                    // We're in the background thread - notify via dispatcher.
                    Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() =>
                    {
                        Wizard.TaskCount++;
                        ReturnPage = Wizard.CurrentPage;
                        Wizard.CurrentPage = this;
                        AuthorizationInProgress = new CancellationTokenSource();
                    }));
                    try
                    {
                        // Get API endpoints. (Not called from the UI thread. Otherwise it would need to be spawned as a background task to avoid deadlock.)
                        var api = authenticatingServer.GetEndpoints(Window.Abort.Token);

                        // Prepare new authorization grant.
                        AuthorizationGrant authorizationGrant = null;
                        Uri callbackUri = null;
                        var httpListener = new eduOAuth.HttpListener(IPAddress.Loopback, 0);
                        httpListener.HttpCallback += (object _, HttpCallbackEventArgs eHTTPCallback) =>
                        {
                            Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() =>
                            {
                                callbackUri = eHTTPCallback.Uri;
                                AuthorizationInProgress.Cancel();
                                Wizard.CurrentPage = ReturnPage;
                            }));
                        };
                        httpListener.HttpRequest += (object _, HttpRequestEventArgs eHTTPRequest) =>
                        {
                            if (eHTTPRequest.Uri.AbsolutePath.ToLowerInvariant() == "/favicon.ico")
                            {
                                var res = System.Windows.Application.GetResourceStream(new Uri("pack://application:,,,/Resources/App.ico"));
                                eHTTPRequest.Type = res.ContentType;
                                eHTTPRequest.Content = res.Stream;
                            }
                        };
                        httpListener.Start();
                        try
                        {
                            // Make the authorization URI.
                            authorizationGrant = new AuthorizationGrant(
                                api.AuthorizationEndpoint,
                                new Uri(string.Format("http://{0}:{1}/callback", ((IPEndPoint)httpListener.LocalEndpoint).Address, ((IPEndPoint)httpListener.LocalEndpoint).Port)),
                                Properties.Settings.Default.ClientId + ".windows",
                                new HashSet<string>() { e.Scope },
                                AuthorizationGrant.CodeChallengeAlgorithmType.S256);
                            var authorizationUri = authorizationGrant.AuthorizationUri;
                            if (authenticatingServer is SecureInternetServer srv &&
                                srv.AuthenticationUriTemplate != null)
                            {
                                // Envelope authorization URI and organization identifier.
                                authorizationUri = new Uri(srv.AuthenticationUriTemplate
                                    .Replace("@[email protected]", HttpUtility.UrlEncode(authorizationUri.ToString()))
                                    .Replace("@[email protected]", HttpUtility.UrlEncode(srv.OrganizationId)));
                            }

                            // Trigger authorization.
                            Process.Start(authorizationUri.ToString());

                            // Wait for a change: either callback is invoked, either user cancels.
                            CancellationTokenSource.CreateLinkedTokenSource(AuthorizationInProgress.Token, Window.Abort.Token).Token.WaitHandle.WaitOne();
                        }
                        finally
                        {
                            // Delay HTTP server shutdown allowing browser to finish loading content.
                            new Thread(new ThreadStart(() =>
                            {
                                Window.Abort.Token.WaitHandle.WaitOne(5 * 1000);
                                httpListener.Stop();
                            })).Start();
                        }

                        if (callbackUri == null)
                            throw new OperationCanceledException();

                        // Get access token from authorization grant.
                        var request = WebRequest.Create(api.TokenEndpoint);
                        request.CachePolicy = Xml.Response.CachePolicy;
                        request.Proxy = null;
                        if (request is HttpWebRequest requestHTTP)
                            requestHTTP.UserAgent = Xml.Response.UserAgent;
                        e.AccessToken = authorizationGrant.ProcessResponse(
                            HttpUtility.ParseQueryString(callbackUri.Query),
                            request,
                            null,
                            Window.Abort.Token);
                        Window.Abort.Token.ThrowIfCancellationRequested();

                        // Save access token to the cache.
                        e.TokenOrigin = RequestAuthorizationEventArgs.TokenOriginType.Authorized;
                        Properties.Settings.Default.AccessTokenCache[authenticatingServer.Base.AbsoluteUri] = e.AccessToken;
                    }
                    finally { Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() => Wizard.TaskCount--)); }
                }
            }
        }

public IAsyncEnumerator<TResult> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var sourceCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new FlatMapEnumerator(_source.GetAsyncEnumerator(sourceCTS.Token), _mapper, _maxConcurrency, _prefetch, sourceCTS);
            en.MoveNext();
            return en;
        }

public async Task Invoke(HttpContext context)
            // ReSharper restore UnusedMember.Global
        {
            _logger.MiddlewareExecuting<HealthCheckEndpointMiddleware>();

            var healthCheckCancellationTokenSource = new CancellationTokenSource(_timeout);

            using (var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(healthCheckCancellationTokenSource.Token, context.RequestAborted))
            {
                try
                {
                    var healthStatus = await _healthCheckRunner.ReadAsync(cancellationTokenSource.Token);

                    await _healthResponseWriter.WriteAsync(context, healthStatus, cancellationTokenSource.Token);
                }
                catch (OperationCanceledException e)
                {
                    var responseFeature = context.Response.HttpContext.Features.Get<IHttpResponseFeature>();

                    if (healthCheckCancellationTokenSource.IsCancellationRequested)
                    {
                        responseFeature.ReasonPhrase = "Health Check Middleware timed out.";
                        _logger.MiddlewareFailed<HealthCheckEndpointMiddleware>(e, responseFeature.ReasonPhrase);
                    }
                    else if (context.RequestAborted.IsCancellationRequested)
                    {
                        responseFeature.ReasonPhrase = "Health Check Middleware request aborted.";
                        _logger.MiddlewareFailed<HealthCheckEndpointMiddleware>(e, responseFeature.ReasonPhrase);
                    }

                    context.SetNoCacheHeaders();
                    context.Response.StatusCode = StatusCodes.Status503ServiceUnavailable;
                    context.Response.Headers[HeaderNames.ContentType] = new[] { context.Request.ContentType };
                }
            }

            _logger.MiddlewareExecuted<HealthCheckEndpointMiddleware>();
        }

protected override async Task<HttpResponseMessage> SendAsync(
            HttpRequestMessage request,
            CancellationToken cancellationToken)
        {
            VssTraceActivity traceActivity = VssTraceActivity.Current;

            var traceInfo = VssHttpMessageHandlerTraceInfo.GetTraceInfo(request);
            traceInfo?.TraceHandlerStartTime();

            if (!m_appliedClientCertificatesToTransportHandler &&
                request.RequestUri.Scheme == "https")
            {
                HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
                if (httpClientHandler != null &&
                    this.Settings.ClientCertificateManager != null &&
                    this.Settings.ClientCertificateManager.ClientCertificates != null &&
                    this.Settings.ClientCertificateManager.ClientCertificates.Count > 0)
                {
                    httpClientHandler.ClientCertificates.AddRange(this.Settings.ClientCertificateManager.ClientCertificates);
                }
                m_appliedClientCertificatesToTransportHandler = true;
            }

            if (!m_appliedServerCertificateValidationCallbackToTransportHandler &&
                request.RequestUri.Scheme == "https")
            {
                HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler;
                if (httpClientHandler != null &&
                    this.Settings.ServerCertificateValidationCallback != null)
                {
                    httpClientHandler.ServerCertificateCustomValidationCallback = this.Settings.ServerCertificateValidationCallback;
                }
                m_appliedServerCertificateValidationCallbackToTransportHandler = true;
            }

            // The .NET Core 2.1 runtime switched its HTTP default from HTTP 1.1 to HTTP 2.
            // This causes problems with some versions of the Curl handler on Linux.
            // See GitHub issue https://github.com/dotnet/corefx/issues/32376
            if (Settings.UseHttp11)
            {
                request.Version = HttpVersion.Version11;
            }

            IssuedToken token = null;
            IssuedTokenProvider provider;
            if (this.Credentials.TryGetTokenProvider(request.RequestUri, out provider))
            {
                token = provider.CurrentToken;
            }

            // Add ourselves to the message so the underlying token issuers may use it if necessary
            request.Properties[VssHttpMessageHandler.PropertyName] = this;

            Boolean succeeded = false;
            Boolean lastResponseDemandedProxyAuth = false;
            Int32 retries = m_maxAuthRetries;
            HttpResponseMessage response = null;
            HttpResponseMessageWrapper responseWrapper;
            CancellationTokenSource tokenSource = null;

            try
            {
                tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);

                if (this.Settings.SendTimeout > TimeSpan.Zero)
                {
                    tokenSource.CancelAfter(this.Settings.SendTimeout);
                }

                do
                {
                    if (response != null)
                    {
                        response.Dispose();
                    }

                    ApplyHeaders(request);

                    // In the case of a Windows token, only apply it to the web proxy if it
                    // returned a 407 Proxy Authentication Required. If we didn't get this
                    // status code back, then the proxy (if there is one) is clearly working fine,
                    // so we shouldn't mess with its credentials.
                    ApplyToken(request, token, applyICredentialsToWebProxy: lastResponseDemandedProxyAuth);
                    lastResponseDemandedProxyAuth = false;

                    // The WinHttpHandler will chunk any content that does not have a computed length which is
                    // not what we want. By loading into a buffer up-front we bypreplaced this behavior and there is
                    // no difference in the normal HttpClientHandler behavior here since this is what they were
                    // already doing.
                    await BufferRequestContentAsync(request, tokenSource.Token).ConfigureAwait(false);

                    traceInfo?.TraceBufferedRequestTime();

                    // ConfigureAwait(false) enables the continuation to be run outside any captured 
                    // SyncronizationContext (such as ASP.NET's) which keeps things from deadlocking...
                    response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false);

                    traceInfo?.TraceRequestSendTime();

                    // Now buffer the response content if configured to do so. In general we will be buffering
                    // the response content in this location, except in the few cases where the caller has 
                    // specified HttpCompletionOption.ResponseHeadersRead.
                    // Trace content type in case of error
                    await BufferResponseContentAsync(request, response, () => $"[ContentType: {response.Content.GetType().Name}]", tokenSource.Token).ConfigureAwait(false);

                    traceInfo?.TraceResponseContentTime();

                    responseWrapper = new HttpResponseMessageWrapper(response);

                    if (!this.Credentials.IsAuthenticationChallenge(responseWrapper))
                    {
                        // Validate the token after it has been successfully authenticated with the server.
                        if (provider != null)
                        {
                            provider.ValidateToken(token, responseWrapper);
                        }

                        // Make sure that once we can authenticate with the service that we turn off the 
                        // Expect100Continue behavior to increase performance.
                        this.ExpectContinue = false;
                        succeeded = true;
                        break;
                    }
                    else
                    {
                        // In the case of a Windows token, only apply it to the web proxy if it
                        // returned a 407 Proxy Authentication Required. If we didn't get this
                        // status code back, then the proxy (if there is one) is clearly working fine,
                        // so we shouldn't mess with its credentials.
                        lastResponseDemandedProxyAuth = responseWrapper.StatusCode == HttpStatusCode.ProxyAuthenticationRequired;

                        // Invalidate the token and ensure that we have the correct token provider for the challenge
                        // which we just received
                        VssHttpEventSource.Log.AuthenticationFailed(traceActivity, response);

                        if (provider != null)
                        {
                            provider.InvalidateToken(token);
                        }

                        // Ensure we have an appropriate token provider for the current challenge
                        provider = this.Credentials.CreateTokenProvider(request.RequestUri, responseWrapper, token);

                        // Make sure we don't invoke the provider in an invalid state
                        if (provider == null)
                        {
                            VssHttpEventSource.Log.IssuedTokenProviderNotFound(traceActivity);
                            break;
                        }
                        else if (provider.GetTokenIsInteractive && this.Credentials.PromptType == CredentialPromptType.DoNotPrompt)
                        {
                            VssHttpEventSource.Log.IssuedTokenProviderPromptRequired(traceActivity, provider);
                            break;
                        }

                        // If the user has already tried once but still unauthorized, stop retrying. The main scenario for this condition
                        // is a user typed in a valid credentials for a hosted account but the replacedociated idenreplacedy does not have 
                        // access. We do not want to continually prompt 3 times without telling them the failure reason. In the 
                        // next release we should rethink about presenting user the failure and options between retries.
                        IEnumerable<String> headerValues;
                        Boolean hasAuthenticateError =
                            response.Headers.TryGetValues(HttpHeaders.VssAuthenticateError, out headerValues) &&
                            !String.IsNullOrEmpty(headerValues.FirstOrDefault());

                        if (retries == 0 || (retries < m_maxAuthRetries && hasAuthenticateError))
                        {
                            break;
                        }

                        // Now invoke the provider and await the result
                        token = await provider.GetTokenAsync(token, tokenSource.Token).ConfigureAwait(false);

                        // I always see 0 here, but the method above could take more time so keep for now
                        traceInfo?.TraceGetTokenTime();

                        // If we just received a token, lets ask the server for the VSID
                        request.Headers.Add(HttpHeaders.VssUserData, String.Empty);

                        retries--;
                    }
                }
                while (retries >= 0);

                if (traceInfo != null)
                {
                    traceInfo.TokenRetries = m_maxAuthRetries - retries;
                }

                // We're out of retries and the response was an auth challenge -- then the request was unauthorized
                // and we will throw a strongly-typed exception with a friendly error message.
                if (!succeeded && response != null && this.Credentials.IsAuthenticationChallenge(responseWrapper))
                {
                    String message = null;
                    IEnumerable<String> serviceError;

                    if (response.Headers.TryGetValues(HttpHeaders.TfsServiceError, out serviceError))
                    {
                        message = UriUtility.UrlDecode(serviceError.FirstOrDefault());
                    }
                    else
                    {
                        message = CommonResources.VssUnauthorized(request.RequestUri.GetLeftPart(UriPartial.Authority));
                    }

                    // Make sure we do not leak the response object when raising an exception
                    if (response != null)
                    {
                        response.Dispose();
                    }

                    VssHttpEventSource.Log.HttpRequestUnauthorized(traceActivity, request, message);
                    VssUnauthorizedException unauthorizedException = new VssUnauthorizedException(message);

                    if (provider != null)
                    {
                        unauthorizedException.Data.Add(CredentialsType, provider.CredentialType);
                    }

                    throw unauthorizedException;
                }

                return response;
            }
            catch (OperationCanceledException ex)
            {
                if (cancellationToken.IsCancellationRequested)
                {
                    VssHttpEventSource.Log.HttpRequestCancelled(traceActivity, request);
                    throw;
                }
                else
                {
                    VssHttpEventSource.Log.HttpRequestTimedOut(traceActivity, request, this.Settings.SendTimeout);
                    throw new TimeoutException(CommonResources.HttpRequestTimeout(this.Settings.SendTimeout), ex);
                }
            }
            finally
            {
                // We always dispose of the token source since otherwise we leak resources if there is a timer pending
                if (tokenSource != null)
                {
                    tokenSource.Dispose();
                }

                traceInfo?.TraceTrailingTime();
            }
        }

private static ValueTask<HealthCheckResult> ExecuteSqlCheckAsync(
            string name,
            Func<IDbConnection> newDbConnection,
            TimeSpan timeout,
            bool degradedOnError,
            CancellationToken cancellationToken)
        {
            var sw = new Stopwatch();

            try
            {
                using (var tokenWithTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
                {
                    tokenWithTimeout.CancelAfter(timeout);

                    sw.Start();
                    using (var connection = newDbConnection())
                    {
                        if (connection.State == ConnectionState.Closed)
                        {
                            connection.Open();
                        }

                        using (var command = connection.CreateCommand())
                        {
                            command.CommandType = CommandType.Text;
                            command.CommandText = "SELECT 1";
                            var commandResult = Convert.ToInt64(command.ExecuteScalar());

                            var result = commandResult == 1
                                ? HealthCheckResult.Healthy($"OK. {name}.")
                                : HealthCheckResultOnError($"FAILED. {name} SELECT failed. Time taken: {sw.ElapsedMilliseconds}ms.", degradedOnError);

                            return new ValueTask<HealthCheckResult>(result);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                var failedResult = degradedOnError
                    ? HealthCheckResult.Degraded(ex)
                    : HealthCheckResult.Unhealthy(ex);

                return new ValueTask<HealthCheckResult>(failedResult);
            }
        }

public IAsyncEnumerator<TSource> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var cancelMain = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var cancelOther = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new SkipUntilEnumerator(_source.GetAsyncEnumerator(cancelMain.Token), _other.GetAsyncEnumerator(cancelOther.Token), cancelMain, cancelOther);
            en.MoveNextOther();
            en.MoveNextMain();
            return en;
        }

public async Task<FtpFile<TPayload>> GetResource(FtpMetadata metadata, IResourceTrackedState lastState, CancellationToken ctk = default)
            {
                var contents = await Policy
                    .Handle<Exception>()
                    .RetryAsync(3)
                    .ExecuteAsync(async ct =>
                    {
                        using (var cts1 = new CancellationTokenSource(_config.DownloadTimeout))
                        using (var cts2 = CancellationTokenSource.CreateLinkedTokenSource(cts1.Token, ct))
                        {
                            return await _ftpClient.DownloadFileAsync(metadata.Entry.FullPath, ctk: cts2.Token).ConfigureAwait(false);

                        }
                    }, ctk).ConfigureAwait(false);

                var checksum = _computeChecksum(contents);
                if (lastState?.CheckSum == checksum)
                    return null;

                var file = new FtpFile<TPayload>(metadata)
                {
                    CheckSum = checksum,
                    RetrievedAt = SystemClock.Instance.GetCurrentInstant(),
                    ParsedData = _parser.Parse(metadata, contents)
                };

                return file;
            }

private async Task<int> RunAsync(RunnerSettings settings, bool runOnce = false)
        {
            try
            {
                Trace.Info(nameof(RunAsync));
                _listener = HostContext.GetService<IMessageListener>();
                if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken))
                {
                    return Constants.Runner.ReturnCode.TerminatedError;
                }

                HostContext.WritePerfCounter("SessionCreated");

                _term.WriteLine($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
                _term.WriteLine($"{DateTime.UtcNow:u}: Listening for Jobs");

                IJobDispatcher jobDispatcher = null;
                CancellationTokenSource messageQueueLoopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(HostContext.RunnerShutdownToken);
                
                // Should we try to cleanup ephemeral runners
                bool runOnceJobCompleted = false;
                try
                {
                    var notification = HostContext.GetService<IJobNotification>();

                    notification.StartClient(settings.MonitorSocketAddress);

                    bool autoUpdateInProgress = false;
                    Task<bool> selfUpdateTask = null;
                    bool runOnceJobReceived = false;
                    jobDispatcher = HostContext.CreateService<IJobDispatcher>();

                    while (!HostContext.RunnerShutdownToken.IsCancellationRequested)
                    {
                        TaskAgentMessage message = null;
                        bool skipMessageDeletion = false;
                        try
                        {
                            Task<TaskAgentMessage> getNextMessage = _listener.GetNextMessageAsync(messageQueueLoopTokenSource.Token);
                            if (autoUpdateInProgress)
                            {
                                Trace.Verbose("Auto update task running at backend, waiting for getNextMessage or selfUpdateTask to finish.");
                                Task completeTask = await Task.WhenAny(getNextMessage, selfUpdateTask);
                                if (completeTask == selfUpdateTask)
                                {
                                    autoUpdateInProgress = false;
                                    if (await selfUpdateTask)
                                    {
                                        Trace.Info("Auto update task finished at backend, an runner update is ready to apply exit the current runner instance.");
                                        Trace.Info("Stop message queue looping.");
                                        messageQueueLoopTokenSource.Cancel();
                                        try
                                        {
                                            await getNextMessage;
                                        }
                                        catch (Exception ex)
                                        {
                                            Trace.Info($"Ignore any exception after cancel message loop. {ex}");
                                        }

                                        if (runOnce)
                                        {
                                            return Constants.Runner.ReturnCode.RunOnceRunnerUpdating;
                                        }
                                        else
                                        {
                                            return Constants.Runner.ReturnCode.RunnerUpdating;
                                        }
                                    }
                                    else
                                    {
                                        Trace.Info("Auto update task finished at backend, there is no available runner update needs to apply, continue message queue looping.");
                                    }
                                }
                            }

                            if (runOnceJobReceived)
                            {
                                Trace.Verbose("One time used runner has start running its job, waiting for getNextMessage or the job to finish.");
                                Task completeTask = await Task.WhenAny(getNextMessage, jobDispatcher.RunOnceJobCompleted.Task);
                                if (completeTask == jobDispatcher.RunOnceJobCompleted.Task)
                                {
                                    runOnceJobCompleted = true;
                                    Trace.Info("Job has finished at backend, the runner will exit since it is running under onetime use mode.");
                                    Trace.Info("Stop message queue looping.");
                                    messageQueueLoopTokenSource.Cancel();
                                    try
                                    {
                                        await getNextMessage;
                                    }
                                    catch (Exception ex)
                                    {
                                        Trace.Info($"Ignore any exception after cancel message loop. {ex}");
                                    }

                                    return Constants.Runner.ReturnCode.Success;
                                }
                            }

                            message = await getNextMessage; //get next message
                            HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}");
                            if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase))
                            {
                                if (autoUpdateInProgress == false)
                                {
                                    autoUpdateInProgress = true;
                                    var runnerUpdateMessage = JsonUtility.FromString<AgentRefreshMessage>(message.Body);
                                    var selfUpdater = HostContext.GetService<ISelfUpdater>();
                                    selfUpdateTask = selfUpdater.SelfUpdate(runnerUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.RunnerShutdownToken);
                                    Trace.Info("Refresh message received, kick-off selfupdate background process.");
                                }
                                else
                                {
                                    Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running.");
                                }
                            }
                            else if (string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase))
                            {
                                if (autoUpdateInProgress || runOnceJobReceived)
                                {
                                    skipMessageDeletion = true;
                                    Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
                                }
                                else
                                {
                                    var jobMessage = StringUtil.ConvertFromJson<Pipelines.AgentJobRequestMessage>(message.Body);
                                    jobDispatcher.Run(jobMessage, runOnce);
                                    if (runOnce)
                                    {
                                        Trace.Info("One time used runner received job message.");
                                        runOnceJobReceived = true;
                                    }
                                }
                            }
                            else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
                            {
                                var cancelJobMessage = JsonUtility.FromString<JobCancelMessage>(message.Body);
                                bool jobCancelled = jobDispatcher.Cancel(cancelJobMessage);
                                skipMessageDeletion = (autoUpdateInProgress || runOnceJobReceived) && !jobCancelled;

                                if (skipMessageDeletion)
                                {
                                    Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'.");
                                }
                            }
                            else
                            {
                                Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
                            }
                        }
                        finally
                        {
                            if (!skipMessageDeletion && message != null)
                            {
                                try
                                {
                                    await _listener.DeleteMessageAsync(message);
                                }
                                catch (Exception ex)
                                {
                                    Trace.Error($"Catch exception during delete message from message queue. message id: {message.MessageId}");
                                    Trace.Error(ex);
                                }
                                finally
                                {
                                    message = null;
                                }
                            }
                        }
                    }
                }
                finally
                {
                    if (jobDispatcher != null)
                    {
                        await jobDispatcher.ShutdownAsync();
                    }

                    try
                    {
                        await _listener.DeleteSessionAsync();
                    }
                    catch (Exception ex) when (runOnce)
                    {
                        // ignore exception during delete session for ephemeral runner since the runner might already be deleted from the server side
                        // and the delete session call will ends up with 401.
                        Trace.Info($"Ignore any exception during DeleteSession for an ephemeral runner. {ex}");
                    }

                    messageQueueLoopTokenSource.Dispose();

                    if (settings.Ephemeral && runOnceJobCompleted)
                    {
                        var configManager = HostContext.GetService<IConfigurationManager>();
                        configManager.DeleteLocalRunnerConfig();
                    }
                }
            }
            catch (TaskAgentAccessTokenExpiredException)
            {
                Trace.Info("Runner OAuth token has been revoked. Shutting down.");
            }

            return Constants.Runner.ReturnCode.Success;
        }

internal async Task CleanupAsync()
        {
            switch (_webSocket.State)
            {
                case WebSocketState.Closed: // Closed gracefully, no action needed. 
                case WebSocketState.Aborted: // Closed abortively, no action needed.                       
                    break;
                case WebSocketState.CloseReceived:
                    // Attempt a graceful closure on behalf of the application, but don't wait too long.
                    using (var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken))
                    {
                        timeoutCts.CancelAfter(TimeSpan.FromSeconds(15));
                        // Echo what the client said, if anything.
                        await _webSocket.CloseAsync(_webSocket.CloseStatus ?? WebSocketCloseStatus.NormalClosure,
                            _webSocket.CloseStatusDescription ?? string.Empty, timeoutCts.Token);
                    }
                    break;
                case WebSocketState.Open:
                case WebSocketState.CloseSent: // No close received, abort so we don't have to drain the pipe.
                    _webSocket.Abort();
                    break;
                default:
                    throw new ArgumentOutOfRangeException("state", _webSocket.State, string.Empty);
            }
        }

public IAsyncEnumerator<TResult> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var subject = new MulticastAsyncEnumerable<TSource>();
            IAsyncEnumerable<TResult> result;
            try
            {
                result = _func(subject);
            }
            catch (Exception ex)
            {
                return new Error<TResult>.ErrorEnumerator(ex);
            }
            var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new MulticastEnumerator<TSource, TResult>(_source.GetAsyncEnumerator(cts.Token), subject, result.GetAsyncEnumerator(cancellationToken), cts);
            return en;
        }

async Task IHostedService.StartAsync(CancellationToken cancellationToken)
        {
            Logger.LogDebug($"{ServiceName} service started. Timer first/interval {FirstInterval ?? Interval}/{Interval}.");
            _cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            await StartingAsync(_cts.Token).ConfigureAwait(false);
            _timer = new Timer(Execute, null, (FirstInterval ?? Interval).Add(TimeSpan.FromMilliseconds(_random.Next(0, 1000))), Interval);
        }

public Task StartAsync(CancellationToken cancellationToken)
        {
            _logger.LogInformation("Starting tokens removal");

            Cancel();
            _source = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);

            return Task.Factory.StartNew(() => CleanupAsync(_source.Token));
        }

public Task StartAsync(CancellationToken stopToken)
        {
            if (!_loadSuccess)
            {
                throw new Exception();
            }

            _stopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stopToken);

            _execution = ExecutionTask(_stopTokenSource.Token);

            return Task.CompletedTask;
        }

public IAsyncEnumerator<TResult> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var subject = new ReplayAsyncEnumerable<TSource>();
            IAsyncEnumerable<TResult> result;
            try
            {
                result = _func(subject);
            }
            catch (Exception ex)
            {
                return new Error<TResult>.ErrorEnumerator(ex);
            }
            var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new MulticastEnumerator<TSource, TResult>(_source.GetAsyncEnumerator(cts.Token), subject, result.GetAsyncEnumerator(cancellationToken), cts);
            return en;
        }

public static async Task<IEnumerable<UnicastIPAddressInformation>> GetAddrInfoOfConnectionAsync(string host, int port,
            int timeoutMs = 5000,
            CancellationToken cancelToken = default)
        {
            //TODO right now we're using TCP connection because that's what most Internet endpoint are using.
            //howerver HTTP3 uses QUIC instead of TCP so this might need revisions in the future
            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancelToken);
            cts.CancelAfter(timeoutMs);
            using var tcpClient = new TcpClient();

            try
            {
                await tcpClient.ConnectAsync(host, port, cts.Token);
                if (!tcpClient.Connected || tcpClient.Client.LocalEndPoint is not IPEndPoint localEndpoint)
                {
                    throw new SocketException();
                }

                var localAddress = localEndpoint.Address;
                var localAddressIPv4 = localAddress.AddressFamily == AddressFamily.InterNetwork
                    ? localAddress
                    : localAddress.IsIPv4MappedToIPv6 ? localAddress.MapToIPv4() : null;

                return NetworkInterface.GetAllNetworkInterfaces()
                     .Where(n => n.OperationalStatus == OperationalStatus.Up &&
                         n.NetworkInterfaceType != NetworkInterfaceType.Tunnel &&
                         n.NetworkInterfaceType != NetworkInterfaceType.Loopback)
                     .Where(n => n.GetIPProperties().UnicastAddresses.Any(
                         ip => ip.Address.AddressFamily == AddressFamily.InterNetwork || ip.Address.AddressFamily == AddressFamily.InterNetworkV6))
                     .SelectMany(ni => ni.GetIPProperties().UnicastAddresses)
                     .Where(unicastAddr =>
                     {
                         var addr = unicastAddr.Address;
                         switch (addr.AddressFamily)
                         {
                             case AddressFamily.InterNetwork:
                                 return addr.Equals(localAddressIPv4);
                             case AddressFamily.InterNetworkV6:
                                 if (addr.IsIPv4MappedToIPv6)
                                 {
                                     addr = addr.MapToIPv4();
                                 }
                                 return addr.Equals(localAddress);
                             default:
                                 return false;
                         }
                     });
            }
            catch (OperationCanceledException)
            {
                if (cancelToken.IsCancellationRequested)
                {
                    // the cancel signal comes from the stopToken parameter, we throw it here so the caller can catch it
                    cancelToken.ThrowIfCancellationRequested();
                }
                // throw a time-out 10060 exception
                throw new SocketException(10060);
            }
        }

public async Task DownloadFromContainerAsync(
            RunnerActionPluginExecutionContext context,
            String destination,
            CancellationToken cancellationToken)
        {
            // Find out all container items need to be processed
            List<FileContainerItem> containerItems = new List<FileContainerItem>();
            int retryCount = 0;
            while (retryCount < 3)
            {
                try
                {
                    containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId,
                                                                                             _projectId,
                                                                                             _containerPath,
                                                                                             cancellationToken: cancellationToken);
                    break;
                }
                catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
                {
                    context.Debug($"Container query has been cancelled.");
                    throw;
                }
                catch (Exception ex) when (retryCount < 2)
                {
                    retryCount++;
                    context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}");
                    context.Debug(ex.ToString());
                }

                var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
                context.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
                await Task.Delay(backOff);
            }

            if (containerItems.Count == 0)
            {
                context.Output($"There is nothing under #/{_containerId}/{_containerPath}");
                return;
            }

            // container items will include both folders, files and even file with zero size
            // Create all required empty folders and emptry files, gather a list of files that we need to download from server.
            int foldersCreated = 0;
            int emptryFilesCreated = 0;
            List<DownloadInfo> downloadFiles = new List<DownloadInfo>();
            foreach (var item in containerItems.OrderBy(x => x.Path))
            {
                if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase))
                {
                    throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}");
                }

                var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/');
                var localPath = Path.Combine(destination, localRelativePath);

                if (item.ItemType == ContainerItemType.Folder)
                {
                    context.Debug($"Ensure folder exists: {localPath}");
                    Directory.CreateDirectory(localPath);
                    foldersCreated++;
                }
                else if (item.ItemType == ContainerItemType.File)
                {
                    if (item.FileLength == 0)
                    {
                        context.Debug($"Create empty file at: {localPath}");
                        var parentDirectory = Path.GetDirectoryName(localPath);
                        Directory.CreateDirectory(parentDirectory);
                        IOUtil.DeleteFile(localPath);
                        using (new FileStream(localPath, FileMode.Create))
                        {
                        }
                        emptryFilesCreated++;
                    }
                    else
                    {
                        context.Debug($"Prepare download {item.Path} to {localPath}");
                        downloadFiles.Add(new DownloadInfo(item.Path, localPath));
                    }
                }
                else
                {
                    throw new NotSupportedException(item.ItemType.ToString());
                }
            }

            if (foldersCreated > 0)
            {
                context.Output($"{foldersCreated} folders created.");
            }

            if (emptryFilesCreated > 0)
            {
                context.Output($"{emptryFilesCreated} empty files created.");
            }

            if (downloadFiles.Count == 0)
            {
                context.Output($"There is nothing to download");
                return;
            }

            // Start multi-task to download all files.
            using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                // try download all files for the first time.
                DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
                if (downloadResult.FailedFiles.Count == 0)
                {
                    // all files have been download succeed.
                    context.Output($"{downloadFiles.Count} files download succeed.");
                    return;
                }
                else
                {
                    context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute.");
                }

                // Delay 1 min then retry failed files.
                for (int timer = 60; timer > 0; timer -= 5)
                {
                    context.Output($"Retry file download after {timer} seconds.");
                    await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
                }

                // Retry download all failed files.
                context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload.");
                DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
                if (retryDownloadResult.FailedFiles.Count == 0)
                {
                    // all files have been download succeed after retry.
                    context.Output($"{downloadResult.FailedFiles} files download succeed after retry.");
                    return;
                }
                else
                {
                    throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry.");
                }
            }
        }

public static TResult Run<TResult>(Func<CancellationToken, TResult> body, int msTimeout, Action timeout, CancellationToken? cancel = null)
    {
      body.NonNull(nameof(body));
      timeout.NonNull(nameof(timeout));
      msTimeout.IsTrue(a => a > 0, nameof(msTimeout));

      using (var cts = cancel.HasValue ? CancellationTokenSource.CreateLinkedTokenSource(cancel.Value) : new CancellationTokenSource())
      {
        Task.Delay(msTimeout, cts.Token)
            .ContinueWith(d => { try { if (!d.IsCanceled) timeout(); } catch { } });

        try
        {
          return body(cts.Token);
        }
        finally
        {
          cts.Cancel();
        }
      }
    }

public IAsyncEnumerator<TResult> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var sourceCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new SwitchMapEnumerator(_source.GetAsyncEnumerator(sourceCTS.Token), _mapper, sourceCTS);
            en.MoveNext();
            return en;
        }

public IAsyncEnumerator<TResult> GetAsyncEnumerator(CancellationToken cancellationToken)
        {
            var sourceCTS = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            var en = new ConcatMapEagerEnumerator(_source.GetAsyncEnumerator(sourceCTS.Token), _mapper, _maxConcurrency, _prefetch,
                sourceCTS);
            en.MoveNextSource();
            return en;
        }

private void DumpDisc(object sender, DoWorkEventArgs doWorkEventArgs)
        {
            var backgroundWorker = (BackgroundWorker)sender;
            var dumper = (Dumper)doWorkEventArgs.Argument;
            try
            {
                var threadCts = new CancellationTokenSource();
                var combinedToken = CancellationTokenSource.CreateLinkedTokenSource(threadCts.Token, dumper.Cts.Token);
                var monitor = new Thread(() =>
                                         {
                                             try
                                             {
                                                 do
                                                 {
                                                     if (dumper.TotalSectors > 0 && backgroundWorker.IsBusy && !backgroundWorker.CancellationPending)
                                                        try { backgroundWorker.ReportProgress((int)(dumper.CurrentSector * 10000L / dumper.TotalSectors), dumper); } catch { }
                                                     Task.Delay(1000, combinedToken.Token).GetAwaiter().GetResult();
                                                 } while (!combinedToken.Token.IsCancellationRequested);
                                             }
                                             catch (TaskCanceledException)
                                             {
                                             }
                                         });
                monitor.Start();
                dumper.DumpAsync(settings.OutputDir).Wait(dumper.Cts.Token);
                threadCts.Cancel();
                monitor.Join(100);
            }
            catch (Exception e)
            {
                MessageBox.Show(e.Message, "Disc dumping error", MessageBoxButtons.OK, MessageBoxIcon.Error);
            }
            doWorkEventArgs.Result = dumper;
        }

public async Task<long> CopyToContainerAsync(
            RunnerActionPluginExecutionContext context,
            String source,
            CancellationToken cancellationToken)
        {
            //set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
            int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
            //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");

            List<String> files;
            if (File.Exists(source))
            {
                files = new List<String>() { source };
                _sourceParentDirectory = Path.GetDirectoryName(source);
            }
            else
            {
                files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList();
                _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
            }

            context.Output($"Uploading {files.Count()} files");
            using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                // hook up reporting event from file container client.
                _fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived;
                _fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived;

                try
                {
                    // try upload all files for the first time.
                    UploadResult uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);

                    if (uploadResult.RetryFiles.Count == 0)
                    {
                        // all files have been upload succeed.
                        context.Output("File upload complete.");
                        return uploadResult.TotalFileSizeUploaded;
                    }
                    else
                    {
                        context.Output($"{uploadResult.RetryFiles.Count} files failed to upload, retry these files after a minute.");
                    }

                    // Delay 1 min then retry failed files.
                    for (int timer = 60; timer > 0; timer -= 5)
                    {
                        context.Output($"Retry file upload after {timer} seconds.");
                        await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
                    }

                    // Retry upload all failed files.
                    context.Output($"Start retry {uploadResult.RetryFiles.Count} failed files upload.");
                    UploadResult retryUploadResult = await ParallelUploadAsync(context, uploadResult.RetryFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);

                    if (retryUploadResult.RetryFiles.Count == 0)
                    {
                        // all files have been upload succeed after retry.
                        context.Output("File upload complete after retry.");
                        return uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded;
                    }
                    else
                    {
                        throw new Exception("File upload failed even after retry.");
                    }
                }
                finally
                {
                    _fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived;
                    _fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived;
                }
            }
        }

public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
        {
            // Validation
            Trace.Entering();
            ArgUtil.NotNull(message, nameof(message));
            ArgUtil.NotNull(message.Resources, nameof(message.Resources));
            ArgUtil.NotNull(message.Variables, nameof(message.Variables));
            ArgUtil.NotNull(message.Plan, nameof(message.Plan));

            _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);

            Global = new GlobalContext();

            // Plan
            Global.Plan = message.Plan;
            Global.Features = PlanUtil.GetFeatures(message.Plan);

            // Endpoints
            Global.Endpoints = message.Resources.Endpoints;

            // Variables
            Global.Variables = new Variables(HostContext, message.Variables);

            // Environment variables shared across all actions
            Global.EnvironmentVariables = new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer);

            // Job defaults shared across all actions
            Global.JobDefaults = new Dictionary<string, IDictionary<string, string>>(StringComparer.OrdinalIgnoreCase);

            // Job Outputs
            JobOutputs = new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase);

            // Actions environment
            ActionsEnvironment = message.ActionsEnvironment;

            // ActionsStepTelemetry
            ActionsStepsTelemetry = new List<ActionsStepTelemetry>();

            JobTelemetry = new List<JobTelemetry>();

            // Service container info
            Global.ServiceContainers = new List<ContainerInfo>();

            // Steps context (StepsRunner manages adding the scoped steps context)
            Global.StepsContext = new StepsContext();

            // File table
            Global.FileTable = new List<String>(message.FileTable ?? new string[0]);

            // Expression values
            if (message.ContextData?.Count > 0)
            {
                foreach (var pair in message.ContextData)
                {
                    ExpressionValues[pair.Key] = pair.Value;
                }
            }

            ExpressionValues["secrets"] = Global.Variables.ToSecretsContext();
            ExpressionValues["runner"] = new RunnerContext();
            ExpressionValues["job"] = new JobContext();

            Trace.Info("Initialize GitHub context");
            var githubAccessToken = new StringContextData(Global.Variables.Get("system.github.token"));
            var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}"));
            HostContext.SecretMasker.AddValue(base64EncodedToken);
            var githubJob = Global.Variables.Get("system.github.job");
            var githubContext = new GitHubContext();
            githubContext["token"] = githubAccessToken;
            if (!string.IsNullOrEmpty(githubJob))
            {
                githubContext["job"] = new StringContextData(githubJob);
            }
            var githubDictionary = ExpressionValues["github"].replacedertDictionary("github");
            foreach (var pair in githubDictionary)
            {
                githubContext[pair.Key] = pair.Value;
            }
            ExpressionValues["github"] = githubContext;

            Trace.Info("Initialize Env context");
#if OS_WINDOWS
            ExpressionValues["env"] = new DictionaryContextData();
#else

            ExpressionValues["env"] = new CaseSensitiveDictionaryContextData();
#endif

            // Prepend Path
            Global.PrependPath = new List<string>();

            // JobSteps for job ExecutionContext
            JobSteps = new Queue<IStep>();

            // PostJobSteps for job ExecutionContext
            PostJobSteps = new Stack<IStep>();

            // StepsWithPostRegistered for job ExecutionContext
            StepsWithPostRegistered = new HashSet<Guid>();

            // EmbeddedStepsWithPostRegistered for job ExecutionContext
            EmbeddedStepsWithPostRegistered = new Dictionary<Guid, string>();

            // EmbeddedIntraActionState for job ExecutionContext
            EmbeddedIntraActionState = new Dictionary<Guid, Dictionary<string, string>>();

            // Job timeline record.
            InitializeTimelineRecord(
                timelineId: message.Timeline.Id,
                timelineRecordId: message.JobId,
                parentTimelineRecordId: null,
                recordType: ExecutionContextType.Job,
                displayName: message.JobDisplayName,
                refName: message.JobName,
                order: null); // The job timeline record's order is set by server.

            // Logger (must be initialized before writing warnings).
            _logger = HostContext.CreateService<IPagingLogger>();
            _logger.Setup(_mainTimelineId, _record.Id);

            // Initialize 'echo on action command success' property, default to false, unless Step_Debug is set
            EchoOnActionCommand = Global.Variables.Step_Debug ?? false;

            // Verbosity (from GitHub.Step_Debug).
            Global.WriteDebug = Global.Variables.Step_Debug ?? false;

            // Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
            _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
        }

private void HandleMain(Task<bool> t)
            {
                if (t.IsCanceled)
                {
                    _error = new OperationCanceledException();
                    _done = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
                else if (t.IsFaulted)
                {
                    CancellationHelper.Cancel(ref _cts);
                    if (_emitLast)
                    {
                        var idx = _sourceIndex;
                        if (idx != 0)
                        {
                            SetLatest(_emitLasreplacedem, idx + 1);
                            _emitLasreplacedem = default;
                        }
                    }
                    _error = ExceptionHelper.Extract(t.Exception);
                    _done = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
                else if (t.Result)
                {
                    Volatile.Read(ref _cts)?.Cancel();

                    var v = _source.Current;
                    if (TryDispose())
                    {
                        if (_emitLast)
                        {
                            _emitLasreplacedem = v;
                        }
                        var idx = ++_sourceIndex;
                        var newCts = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);
                        if (CancellationHelper.Replace(ref _cts, newCts))
                        {
                            Task.Delay(_delay, newCts.Token)
                                .ContinueWith(tt => TimerHandler(tt, v, idx), newCts.Token);
                            MoveNext();
                        }
                    }
                }
                else
                {
                    CancellationHelper.Cancel(ref _cts);
                    if (_emitLast)
                    {
                        var idx = _sourceIndex;
                        if (idx != 0)
                        {
                            SetLatest(_emitLasreplacedem, idx + 1);
                            _emitLasreplacedem = default;
                        }
                    }
                    _done = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
            }

private void Search()
        {
            SearchInProgress?.Cancel();
            SearchInProgress = new CancellationTokenSource();
            var ct = CancellationTokenSource.CreateLinkedTokenSource(SearchInProgress.Token, Window.Abort.Token).Token;
            var keywords = Query.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
            new Thread(new ThreadStart(
                () =>
                {
                    Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() => Wizard.TaskCount++));
                    try
                    {
                        var orderedServerHits = Wizard.GetDiscoveredInsreplaceduteAccessServers(keywords, ct);
                        ct.ThrowIfCancellationRequested();
                        Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() =>
                        {
                            if (ct.IsCancellationRequested) return;
                            var selected = SelectedInsreplaceduteAccessServer?.Base;
                            InsreplaceduteAccessServers = orderedServerHits;
                            SelectedInsreplaceduteAccessServer = Wizard.GetDiscoveredServer<InsreplaceduteAccessServer>(selected);
                        }));
                    }
                    catch (OperationCanceledException) { }
                    catch (Exception ex) { Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() => Wizard.Error = ex)); }
                    finally { Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() => Wizard.TaskCount--)); }
                })).Start();
            new Thread(new ThreadStart(
                () =>
                {
                    Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() => Wizard.TaskCount++));
                    try
                    {
                        var orderedOrganizationHits = Wizard.GetDiscoveredOrganizations(keywords, ct);
                        ct.ThrowIfCancellationRequested();
                        Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() =>
                        {
                            if (ct.IsCancellationRequested) return;
                            var selected = SelectedOrganization?.Id;
                            Organizations = orderedOrganizationHits;
                            SelectedOrganization = Wizard.GetDiscoveredOrganization(selected);
                        }));
                    }
                    catch (OperationCanceledException) { }
                    catch (Exception ex) { Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() => Wizard.Error = ex)); }
                    finally { Wizard.Dispatcher.Invoke(DispatcherPriority.Normal, (Action)(() => Wizard.TaskCount--)); }
                })).Start();

            if (keywords.Length == 1 && keywords[0].Split('.').Length >= 3)
            {
                var ownServers = new ObservableCollection<Server>();
                try
                {
                    var srv = new Server(new UriBuilder("https", keywords[0]).Uri);
                    srv.RequestAuthorization += Wizard.AuthorizationPage.OnRequestAuthorization;
                    srv.ForgetAuthorization += Wizard.AuthorizationPage.OnForgetAuthorization;
                    ownServers.Add(srv);
                }
                catch { }
                var selected = SelectedOwnServer?.Base;
                OwnServers = ownServers;
                SelectedOwnServer = ownServers.FirstOrDefault(srv => selected == srv.Base);
            }
            else
            {
                SelectedOwnServer = null;
                OwnServers = null;
            }
        }

See More Examples