System.Threading.Tasks.Task.WhenAny(System.Collections.Generic.IEnumerable)

public async Task Stop(TimeSpan grace) {
            if (_task == null || _task.IsCompleted) {
                return;
            }

            _proc.Cancel();

            var finished = await Task.WhenAny(_task, _proc.Delay(grace, CancellationToken.None));
            if (finished != _task) {
                _proc.Debug("Shutdown timeout. ERASING FUTURE to KILL");
                _machine.Runtime.FutureQueue.Erase(_scheduler);
            }
            
            ReleaseResources();
        }

public static async Task Main(string[] args)
        {
            if (File.Exists(ConfigurePath))
            {
                ApplicationConfig = await LoadConfigAsync().ConfigureAwait(false);
            }
            else
            {
                await SaveConfigAsync(new Config()).ConfigureAwait(false);
                return;
            }

            DiscordClientConfig = new DiscordSocketConfig()
            {
                LogLevel = LogSeverity.Verbose
            };
            DiscordClient = new DiscordSocketClient(DiscordClientConfig);
            DiscordClient.Log += RequestLogAsync;
            DiscordClient.Ready += () => Task.WhenAny(/* NotificationManager.InitAsync() ,*/ TippingManager.WorkAsync(), /* MonitorManager.WorkAsync() ,*/ TickerManager.WorkAsync(), Task.Delay(0));

            await ModuleManager.InstallAsync().ConfigureAwait(false);

            await DiscordClient.LoginAsync(TokenType.Bot, ApplicationConfig.DiscordToken).ConfigureAwait(false);
            await DiscordClient.StartAsync().ConfigureAwait(false);

            while (!CancellationTokenSource.Token.IsCancellationRequested)
            {
                await Task.Delay(1024).ConfigureAwait(false);
            }
        }

public async Task<bool> RunCheck(string url, string pat)
        {
            await File.AppendAllLinesAsync(_logFile, HostContext.WarnLog());
            await File.AppendAllLinesAsync(_logFile, HostContext.CheckProxy());

            var checkTasks = new List<Task<CheckResult>>();
            string githubApiUrl = null;
            string actionsTokenServiceUrl = null;
            string actionsPipelinesServiceUrl = null;
            var urlBuilder = new UriBuilder(url);
            if (UrlUtil.IsHostedServer(urlBuilder))
            {
                urlBuilder.Host = $"api.{urlBuilder.Host}";
                urlBuilder.Path = "";
                githubApiUrl = urlBuilder.Uri.AbsoluteUri;
                actionsTokenServiceUrl = "https://vstoken.actions.githubusercontent.com/_apis/health";
                actionsPipelinesServiceUrl = "https://pipelines.actions.githubusercontent.com/_apis/health";
            }
            else
            {
                urlBuilder.Path = "api/v3";
                githubApiUrl = urlBuilder.Uri.AbsoluteUri;
                urlBuilder.Path = "_services/vstoken/_apis/health";
                actionsTokenServiceUrl = urlBuilder.Uri.AbsoluteUri;
                urlBuilder.Path = "_services/pipelines/_apis/health";
                actionsPipelinesServiceUrl = urlBuilder.Uri.AbsoluteUri;
            }

            // check github api
            checkTasks.Add(CheckUtil.CheckDns(githubApiUrl));
            checkTasks.Add(CheckUtil.CheckPing(githubApiUrl));
            checkTasks.Add(HostContext.CheckHttpsGetRequests(githubApiUrl, pat, expectedHeader: "X-GitHub-Request-Id"));

            // check actions token service
            checkTasks.Add(CheckUtil.CheckDns(actionsTokenServiceUrl));
            checkTasks.Add(CheckUtil.CheckPing(actionsTokenServiceUrl));
            checkTasks.Add(HostContext.CheckHttpsGetRequests(actionsTokenServiceUrl, pat, expectedHeader: "x-vss-e2eid"));

            // check actions pipelines service
            checkTasks.Add(CheckUtil.CheckDns(actionsPipelinesServiceUrl));
            checkTasks.Add(CheckUtil.CheckPing(actionsPipelinesServiceUrl));
            checkTasks.Add(HostContext.CheckHttpsGetRequests(actionsPipelinesServiceUrl, pat, expectedHeader: "x-vss-e2eid"));

            // check HTTP POST to actions pipelines service
            checkTasks.Add(HostContext.CheckHttpsPostRequests(actionsPipelinesServiceUrl, pat, expectedHeader: "x-vss-e2eid"));

            var result = true;
            while (checkTasks.Count > 0)
            {
                var finishedCheckTask = await Task.WhenAny<CheckResult>(checkTasks);
                var finishedCheck = await finishedCheckTask;
                result = result && finishedCheck.Preplaced;
                await File.AppendAllLinesAsync(_logFile, finishedCheck.Logs);
                checkTasks.Remove(finishedCheckTask);
            }

            await Task.WhenAll(checkTasks);
            return result;
        }

public async Task<bool> RunCheck(string url, string pat)
        {
            await File.AppendAllLinesAsync(_logFile, HostContext.WarnLog());
            await File.AppendAllLinesAsync(_logFile, HostContext.CheckProxy());

            var checkTasks = new List<Task<CheckResult>>();
            checkTasks.Add(CheckUtil.CheckDns("https://api.github.com"));
            checkTasks.Add(CheckUtil.CheckPing("https://api.github.com"));

            // We don't need to preplaced a PAT since it might be a token for GHES.
            checkTasks.Add(HostContext.CheckHttpsGetRequests("https://api.github.com", pat: null, expectedHeader: "X-GitHub-Request-Id"));

            var result = true;
            while (checkTasks.Count > 0)
            {
                var finishedCheckTask = await Task.WhenAny<CheckResult>(checkTasks);
                var finishedCheck = await finishedCheckTask;
                result = result && finishedCheck.Preplaced;
                await File.AppendAllLinesAsync(_logFile, finishedCheck.Logs);
                checkTasks.Remove(finishedCheckTask);
            }

            await Task.WhenAll(checkTasks);
            return result;
        }

private async Task<int> RunAsync(RunnerSettings settings, bool runOnce = false)
        {
            try
            {
                Trace.Info(nameof(RunAsync));
                _listener = HostContext.GetService<IMessageListener>();
                if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken))
                {
                    return Constants.Runner.ReturnCode.TerminatedError;
                }

                HostContext.WritePerfCounter("SessionCreated");

                _term.WriteLine($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
                _term.WriteLine($"{DateTime.UtcNow:u}: Listening for Jobs");

                IJobDispatcher jobDispatcher = null;
                CancellationTokenSource messageQueueLoopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(HostContext.RunnerShutdownToken);
                
                // Should we try to cleanup ephemeral runners
                bool runOnceJobCompleted = false;
                try
                {
                    var notification = HostContext.GetService<IJobNotification>();

                    notification.StartClient(settings.MonitorSocketAddress);

                    bool autoUpdateInProgress = false;
                    Task<bool> selfUpdateTask = null;
                    bool runOnceJobReceived = false;
                    jobDispatcher = HostContext.CreateService<IJobDispatcher>();

                    while (!HostContext.RunnerShutdownToken.IsCancellationRequested)
                    {
                        TaskAgentMessage message = null;
                        bool skipMessageDeletion = false;
                        try
                        {
                            Task<TaskAgentMessage> getNextMessage = _listener.GetNextMessageAsync(messageQueueLoopTokenSource.Token);
                            if (autoUpdateInProgress)
                            {
                                Trace.Verbose("Auto update task running at backend, waiting for getNextMessage or selfUpdateTask to finish.");
                                Task completeTask = await Task.WhenAny(getNextMessage, selfUpdateTask);
                                if (completeTask == selfUpdateTask)
                                {
                                    autoUpdateInProgress = false;
                                    if (await selfUpdateTask)
                                    {
                                        Trace.Info("Auto update task finished at backend, an runner update is ready to apply exit the current runner instance.");
                                        Trace.Info("Stop message queue looping.");
                                        messageQueueLoopTokenSource.Cancel();
                                        try
                                        {
                                            await getNextMessage;
                                        }
                                        catch (Exception ex)
                                        {
                                            Trace.Info($"Ignore any exception after cancel message loop. {ex}");
                                        }

                                        if (runOnce)
                                        {
                                            return Constants.Runner.ReturnCode.RunOnceRunnerUpdating;
                                        }
                                        else
                                        {
                                            return Constants.Runner.ReturnCode.RunnerUpdating;
                                        }
                                    }
                                    else
                                    {
                                        Trace.Info("Auto update task finished at backend, there is no available runner update needs to apply, continue message queue looping.");
                                    }
                                }
                            }

                            if (runOnceJobReceived)
                            {
                                Trace.Verbose("One time used runner has start running its job, waiting for getNextMessage or the job to finish.");
                                Task completeTask = await Task.WhenAny(getNextMessage, jobDispatcher.RunOnceJobCompleted.Task);
                                if (completeTask == jobDispatcher.RunOnceJobCompleted.Task)
                                {
                                    runOnceJobCompleted = true;
                                    Trace.Info("Job has finished at backend, the runner will exit since it is running under onetime use mode.");
                                    Trace.Info("Stop message queue looping.");
                                    messageQueueLoopTokenSource.Cancel();
                                    try
                                    {
                                        await getNextMessage;
                                    }
                                    catch (Exception ex)
                                    {
                                        Trace.Info($"Ignore any exception after cancel message loop. {ex}");
                                    }

                                    return Constants.Runner.ReturnCode.Success;
                                }
                            }

                            message = await getNextMessage; //get next message
                            HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}");
                            if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase))
                            {
                                if (autoUpdateInProgress == false)
                                {
                                    autoUpdateInProgress = true;
                                    var runnerUpdateMessage = JsonUtility.FromString<AgentRefreshMessage>(message.Body);
                                    var selfUpdater = HostContext.GetService<ISelfUpdater>();
                                    selfUpdateTask = selfUpdater.SelfUpdate(runnerUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.RunnerShutdownToken);
                                    Trace.Info("Refresh message received, kick-off selfupdate background process.");
                                }
                                else
                                {
                                    Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running.");
                                }
                            }
                            else if (string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase))
                            {
                                if (autoUpdateInProgress || runOnceJobReceived)
                                {
                                    skipMessageDeletion = true;
                                    Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
                                }
                                else
                                {
                                    var jobMessage = StringUtil.ConvertFromJson<Pipelines.AgentJobRequestMessage>(message.Body);
                                    jobDispatcher.Run(jobMessage, runOnce);
                                    if (runOnce)
                                    {
                                        Trace.Info("One time used runner received job message.");
                                        runOnceJobReceived = true;
                                    }
                                }
                            }
                            else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
                            {
                                var cancelJobMessage = JsonUtility.FromString<JobCancelMessage>(message.Body);
                                bool jobCancelled = jobDispatcher.Cancel(cancelJobMessage);
                                skipMessageDeletion = (autoUpdateInProgress || runOnceJobReceived) && !jobCancelled;

                                if (skipMessageDeletion)
                                {
                                    Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'.");
                                }
                            }
                            else
                            {
                                Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
                            }
                        }
                        finally
                        {
                            if (!skipMessageDeletion && message != null)
                            {
                                try
                                {
                                    await _listener.DeleteMessageAsync(message);
                                }
                                catch (Exception ex)
                                {
                                    Trace.Error($"Catch exception during delete message from message queue. message id: {message.MessageId}");
                                    Trace.Error(ex);
                                }
                                finally
                                {
                                    message = null;
                                }
                            }
                        }
                    }
                }
                finally
                {
                    if (jobDispatcher != null)
                    {
                        await jobDispatcher.ShutdownAsync();
                    }

                    try
                    {
                        await _listener.DeleteSessionAsync();
                    }
                    catch (Exception ex) when (runOnce)
                    {
                        // ignore exception during delete session for ephemeral runner since the runner might already be deleted from the server side
                        // and the delete session call will ends up with 401.
                        Trace.Info($"Ignore any exception during DeleteSession for an ephemeral runner. {ex}");
                    }

                    messageQueueLoopTokenSource.Dispose();

                    if (settings.Ephemeral && runOnceJobCompleted)
                    {
                        var configManager = HostContext.GetService<IConfigurationManager>();
                        configManager.DeleteLocalRunnerConfig();
                    }
                }
            }
            catch (TaskAgentAccessTokenExpiredException)
            {
                Trace.Info("Runner OAuth token has been revoked. Shutting down.");
            }

            return Constants.Runner.ReturnCode.Success;
        }

public async Task RunAsync(ActionRunStage stage)
        {
            // Validate args.
            Trace.Entering();
            ArgUtil.NotNull(Data, nameof(Data));
            ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
            ArgUtil.NotNull(Inputs, nameof(Inputs));
            ArgUtil.Directory(ActionDirectory, nameof(ActionDirectory));

            // Update the env dictionary.
            AddInputsToEnvironment();
            AddPrependPathToEnvironment();

            // expose context to environment
            foreach (var context in ExecutionContext.ExpressionValues)
            {
                if (context.Value is IEnvironmentContextData runtimeContext && runtimeContext != null)
                {
                    foreach (var env in runtimeContext.GetRuntimeEnvironmentVariables())
                    {
                        Environment[env.Key] = env.Value;
                    }
                }
            }

            // Add Actions Runtime server info
            var systemConnection = ExecutionContext.Global.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
            Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri;
            Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
            if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl))
            {
                Environment["ACTIONS_CACHE_URL"] = cacheUrl;
            }
            if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl))
            {
                Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl;
                Environment["ACTIONS_ID_TOKEN_REQUEST_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
            }

            // Resolve the target script.
            string target = null;
            if (stage == ActionRunStage.Main)
            {
                target = Data.Script;
            }
            else if (stage == ActionRunStage.Pre)
            {
                target = Data.Pre;
            }
            else if (stage == ActionRunStage.Post)
            {
                target = Data.Post;
            }

            // Add Telemetry to JobContext to send with JobCompleteMessage
            if (stage == ActionRunStage.Main)
            {
                var telemetry = new ActionsStepTelemetry
                {
                    Ref = GetActionRef(),
                    HasPreStep = Data.HasPre,
                    HasPostStep = Data.HasPost,
                    IsEmbedded = ExecutionContext.IsEmbedded,
                    Type = "node12"
                };
                ExecutionContext.Root.ActionsStepsTelemetry.Add(telemetry);
            }

            ArgUtil.NotNullOrEmpty(target, nameof(target));
            target = Path.Combine(ActionDirectory, target);
            ArgUtil.File(target, nameof(target));

            // Resolve the working directory.
            string workingDirectory = ExecutionContext.GetGitHubContext("workspace");
            if (string.IsNullOrEmpty(workingDirectory))
            {
                workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
            }

            var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext);
            string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}");

            // Format the arguments preplaceded to node.
            // 1) Wrap the script file path in double quotes.
            // 2) Escape double quotes within the script file path. Double-quote is a valid
            // file name character on Linux.
            string arguments = StepHost.ResolvePathForStepHost(StringUtil.Format(@"""{0}""", target.Replace(@"""", @"\""")));

#if OS_WINDOWS
            // It appears that node.exe outputs UTF8 when not in TTY mode.
            Encoding outputEncoding = Encoding.UTF8;
#else
            // Let .NET choose the default.
            Encoding outputEncoding = null;
#endif

            // Remove environment variable that may cause conflicts with the node within the runner.
            Environment.Remove("NODE_ICU_DATA"); // https://github.com/actions/runner/issues/795

            using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager))
            using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager))
            {
                StepHost.OutputDataReceived += stdoutManager.OnDataReceived;
                StepHost.ErrorDataReceived += stderrManager.OnDataReceived;

                // Execute the process. Exit code 0 should always be returned.
                // A non-zero exit code indicates infrastructural failure.
                // Task failure should be communicated over STDOUT using ## commands.
                Task<int> step = StepHost.ExecuteAsync(workingDirectory: StepHost.ResolvePathForStepHost(workingDirectory),
                                                fileName: StepHost.ResolvePathForStepHost(file),
                                                arguments: arguments,
                                                environment: Environment,
                                                requireExitCodeZero: false,
                                                outputEncoding: outputEncoding,
                                                killProcessOnCancel: false,
                                                inheritConsoleHandler: !ExecutionContext.Global.Variables.Retain_Default_Encoding,
                                                cancellationToken: ExecutionContext.CancellationToken);

                // Wait for either the node exit or force finish through ##vso command
                await System.Threading.Tasks.Task.WhenAny(step, ExecutionContext.ForceCompleted);

                if (ExecutionContext.ForceCompleted.IsCompleted)
                {
                    ExecutionContext.Debug("The task was marked as \"done\", but the process has not closed after 5 seconds. Treating the task as complete.");
                }
                else
                {
                    var exitCode = await step;
                    ExecutionContext.Debug($"Node Action run completed with exit code {exitCode}");
                    if (exitCode != 0)
                    {
                        ExecutionContext.Result = TaskResult.Failed;
                    }
                }
            }
        }

public async Task<IssuedToken> GetTokenAsync(VssTraceActivity traceActivity)
            {
                IssuedToken token = null;
                try
                {
                    VssHttpEventSource.Log.IssuedTokenAcquiring(traceActivity, this.Provider);
                    if (this.Provider.InvokeRequired)
                    {
                        // Post to the UI thread using the scheduler. This may return a new task object which needs
                        // to be awaited, since once we get to the UI thread there may be nothing to do if someone else
                        // preempts us.

                        // The cancellation token source is used to handle race conditions between scheduling and 
                        // waiting for the UI task to begin execution. The callback is responsible for disposing of
                        // the token source, since the thought here is that the callback will run eventually as the
                        // typical reason for not starting execution within the timeout is due to a deadlock with
                        // the scheduler being used.
                        var timerTask = new TaskCompletionSource<Object>();
                        var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(3));
                        timeoutTokenSource.Token.Register(() => timerTask.SetResult(null), false);

                        var uiTask = Task.Factory.StartNew((state) => PostCallback(state, timeoutTokenSource),
                                                            this,
                                                            this.CancellationToken,
                                                            TaskCreationOptions.None,
                                                            this.Provider.Credential.Scheduler).Unwrap();

                        var completedTask = await Task.WhenAny(timerTask.Task, uiTask).ConfigureAwait(false);
                        if (completedTask == uiTask)
                        {
                            token = uiTask.Result;
                        }
                    }
                    else
                    {
                        token = await this.Provider.OnGetTokenAsync(this.FailedToken, this.CancellationToken).ConfigureAwait(false);
                    }

                    CompletionSource.TrySetResult(token);
                    return token;
                }
                catch (Exception exception)
                {
                    // Mark our completion source as failed so other waiters will get notified in all cases
                    CompletionSource.TrySetException(exception);
                    throw;
                }
                finally
                {
                    this.Provider.CurrentToken = token ?? this.FailedToken;
                    VssHttpEventSource.Log.IssuedTokenAcquired(traceActivity, this.Provider, token);
                }
            }

public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
        {
            // Validate parameters.
            Trace.Entering();
            ArgUtil.NotNull(message, nameof(message));
            ArgUtil.NotNull(message.Resources, nameof(message.Resources));
            ArgUtil.NotNull(message.Variables, nameof(message.Variables));
            ArgUtil.NotNull(message.Steps, nameof(message.Steps));
            Trace.Info("Job ID {0}", message.JobId);

            DateTime jobStartTimeUtc = DateTime.UtcNow;

            ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));

            // Setup the job server and job server queue.
            var jobServer = HostContext.GetService<IJobServer>();
            VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
            Uri jobServerUrl = systemConnection.Url;

            Trace.Info($"Creating job server with URL: {jobServerUrl}");
            // jobServerQueue is the throttling reporter.
            _jobServerQueue = HostContext.GetService<IJobServerQueue>();
            VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
            await jobServer.ConnectAsync(jobConnection);

            _jobServerQueue.Start(message);
            HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");

            IExecutionContext jobContext = null;
            CancellationTokenRegistration? runnerShutdownRegistration = null;
            try
            {
                // Create the job execution context.
                jobContext = HostContext.CreateService<IExecutionContext>();
                jobContext.InitializeJob(message, jobRequestCancellationToken);
                Trace.Info("Starting the job execution context.");
                jobContext.Start();
                jobContext.Debug($"Starting: {message.JobDisplayName}");

                runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() =>
                {
                    // log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break.
                    // the server will use Ctrl-Break to tells the runner that operating system is shutting down.
                    string errorMessage;
                    switch (HostContext.RunnerShutdownReason)
                    {
                        case ShutdownReason.UserCancelled:
                            errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.";
                            break;
                        case ShutdownReason.OperatingSystemShutdown:
                            errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'";
                            break;
                        default:
                            throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason));
                    }
                    jobContext.AddIssue(new Issue() { Type = IssueType.Error, Message = errorMessage });
                });

                // Validate directory permissions.
                string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
                Trace.Info($"Validating directory permissions for: '{workDirectory}'");
                try
                {
                    Directory.CreateDirectory(workDirectory);
                    IOUtil.ValidateExecutePermission(workDirectory);
                }
                catch (Exception ex)
                {
                    Trace.Error(ex);
                    jobContext.Error(ex);
                    return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed);
                }

                if (jobContext.Global.WriteDebug)
                {
                    jobContext.SetRunnerContext("debug", "1");
                }

                jobContext.SetRunnerContext("os", VarUtil.OS);
                jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture);

                var runnerSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
                jobContext.SetRunnerContext("name", runnerSettings.AgentName);

                string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
                Directory.CreateDirectory(toolsDirectory);
                jobContext.SetRunnerContext("tool_cache", toolsDirectory);

                // Setup TEMP directories
                _tempDirectoryManager = HostContext.GetService<ITempDirectoryManager>();
                _tempDirectoryManager.InitializeTempDirectory(jobContext);

                // Get the job extension.
                Trace.Info("Getting job extension.");
                IJobExtension jobExtension = HostContext.CreateService<IJobExtension>();
                List<IStep> jobSteps = null;
                try
                {
                    Trace.Info("Initialize job. Getting all job steps.");
                    jobSteps = await jobExtension.InitializeJob(jobContext, message);
                }
                catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested)
                {
                    // set the job to canceled
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Job is canceled during initialize.");
                    Trace.Error($"Caught exception: {ex}");
                    return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled);
                }
                catch (Exception ex)
                {
                    // set the job to failed.
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Job initialize failed.");
                    Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
                    return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed);
                }

                // trace out all steps
                Trace.Info($"Total job steps: {jobSteps.Count}.");
                Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
                HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");

                if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) &&
                    !string.IsNullOrEmpty(generateIdTokenUrl))
                {
                    // Server won't issue ID_TOKEN for non-inprogress job.
                    // If the job is trying to use OIDC feature, we want the job to be marked as in-progress before running any customer's steps as much as we can.
                    // Timeline record update background process runs every 500ms, so delay 1000ms is enough for most of the cases
                    Trace.Info($"Waiting for job to be marked as started.");
                    await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(1000));
                }

                // Run all job steps
                Trace.Info("Run all job steps.");
                var stepsRunner = HostContext.GetService<IStepsRunner>();
                try
                {
                    foreach (var step in jobSteps)
                    {
                        jobContext.JobSteps.Enqueue(step);
                    }

                    await stepsRunner.RunAsync(jobContext);
                }
                catch (Exception ex)
                {
                    // StepRunner should never throw exception out.
                    // End up here mean there is a bug in StepRunner
                    // Log the error and fail the job.
                    Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
                    jobContext.Error(ex);
                    return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed);
                }
                finally
                {
                    Trace.Info("Finalize job.");
                    jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
                }

                Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");

                Trace.Info("Completing the job execution context.");
                return await CompleteJobAsync(jobServer, jobContext, message);
            }
            finally
            {
                if (runnerShutdownRegistration != null)
                {
                    runnerShutdownRegistration.Value.Dispose();
                    runnerShutdownRegistration = null;
                }

                await ShutdownQueue(throwOnFailure: false);
            }
        }

private async Task<string> WaitForAnyGetHostIpTaskAsync(IEnumerable<Task<string>> tasks)
        {
            IList<Task<string>> customTasks = tasks.ToList();
            Task<string> completedTask;
            string ipAddress = null;

            do
            {
                completedTask = await Task.WhenAny(customTasks);
                ipAddress = completedTask.Result;
                customTasks.Remove(completedTask);

            } while (ipAddress == null && customTasks.Count > 0);

            return ipAddress;
        }

private static async Task RunTaskWithCompletionSource(Task originalTask, 
            Dictionary<CancellationToken, Task> completionSourceTokenTaskDict)
        {
            var any = await Task.WhenAny(completionSourceTokenTaskDict.Values.ToList().Append(originalTask));

            foreach (var completionSourceTokenTask in completionSourceTokenTaskDict)
            {
                if (any == completionSourceTokenTask.Value)
                {
                    completionSourceTokenTask.Key.ThrowIfCancellationRequested();
                }
            }
        }

public async Task InvokeAsync(TSender sender, TArgs e, AsyncEventExceptionMode exceptionMode = AsyncEventExceptionMode.Default)
        {
            var handlers = this._handlers;
            if (handlers.Length == 0)
                return;

            // Collect exceptions
            List<Exception> exceptions = null;
            if ((exceptionMode & AsyncEventExceptionMode.ThrowAll) != 0)
                exceptions = new List<Exception>(handlers.Length * 2 /* timeout + regular */);

            // If we have a timeout configured, start the timeout task
            var timeout = this.MaximumExecutionTime > TimeSpan.Zero ? Task.Delay(this.MaximumExecutionTime) : null;
            for (var i = 0; i < handlers.Length; i++)
            {
                var handler = handlers[i];
                try
                {
                    // Start the handler execution
                    var handlerTask = handler(sender, e);
                    if (handlerTask != null && timeout != null)
                    {
                        // If timeout is configured, wait for any task to finish
                        // If the timeout task finishes first, the handler is causing a timeout

                        var result = await Task.WhenAny(timeout, handlerTask).ConfigureAwait(false);
                        if (result == timeout)
                        {
                            timeout = null;
                            var timeoutEx = new AsyncEventTimeoutException<TSender, TArgs>(this, handler);

                            // Notify about the timeout and complete execution
                            if ((exceptionMode & AsyncEventExceptionMode.HandleNonFatal) == AsyncEventExceptionMode.HandleNonFatal)
                                this.HandleException(timeoutEx, handler, sender, e);

                            if ((exceptionMode & AsyncEventExceptionMode.ThrowNonFatal) == AsyncEventExceptionMode.ThrowNonFatal)
                                exceptions.Add(timeoutEx);

                            await handlerTask.ConfigureAwait(false);
                        }
                    }
                    else if (handlerTask != null)
                    {
                        // No timeout is configured, or timeout already expired, proceed as usual
                        await handlerTask.ConfigureAwait(false);
                    }

                    if (e.Handled)
                        break;
                }
                catch (Exception ex)
                {
                    e.Handled = false;

                    if ((exceptionMode & AsyncEventExceptionMode.HandleFatal) == AsyncEventExceptionMode.HandleFatal)
                        this.HandleException(ex, handler, sender, e);

                    if ((exceptionMode & AsyncEventExceptionMode.ThrowFatal) == AsyncEventExceptionMode.ThrowFatal)
                        exceptions.Add(ex);
                }
            }

            if ((exceptionMode & AsyncEventExceptionMode.ThrowAll) != 0 && exceptions.Count > 0)
                throw new AggregateException("Exceptions were thrown during execution of the event's handlers.", exceptions);
        }

private Task Discord_VoiceStateUpdated(DiscordClient client, VoiceStateUpdateEventArgs e)
        {
            var gld = e.Guild;
            if (gld == null)
                return Task.CompletedTask;

            if (e.User == null)
                return Task.CompletedTask;

            if (e.User.Id == this.Discord.CurrentUser.Id)
            {
                if (this._connectedGuilds.TryGetValue(e.Guild.Id, out var lvlgc))
                    lvlgc.VoiceStateUpdate = e;

                if (e.After.Channel == null && this.IsConnected && this._connectedGuilds.ContainsKey(gld.Id))
                {
                    _ = Task.Run(async () =>
                    {
                        var delayTask = Task.Delay(this.Configuration.WebSocketCloseTimeout);
                        var tcs = lvlgc.VoiceWsDisconnectTcs.Task;
                        _ = await Task.WhenAny(delayTask, tcs).ConfigureAwait(false);

                        await lvlgc.DisconnectInternalAsync(false, true).ConfigureAwait(false);
                        _ = this._connectedGuilds.TryRemove(gld.Id, out _);
                    });
                }

                if (!string.IsNullOrWhiteSpace(e.SessionId) && e.Channel != null && this.VoiceStateUpdates.TryRemove(gld.Id, out var xe))
                    xe.SetResult(e);
            }

            return Task.CompletedTask;
        }

public static async Task InvokeTasksByQueue(IEnumerable<Func<Task>> taskFactories, int maxDegreeOfParallelism)
        {
            var queue = new Queue<Func<Task>>(taskFactories);
            if (queue.Count == 0)
            {
                return;
            }
            var tasksInFlight = new List<Task>(maxDegreeOfParallelism);
            do
            {
                while (tasksInFlight.Count < maxDegreeOfParallelism && queue.Count != 0)
                {
                    var taskFactory = queue.Dequeue();
                    tasksInFlight.Add(taskFactory());
                }

                var completedTask = await Task.WhenAny(tasksInFlight).ConfigureAwait(false);
                await completedTask.ConfigureAwait(false);
                tasksInFlight.Remove(completedTask);
            }
            while (queue.Count != 0 || tasksInFlight.Count != 0);
        }

public async Task RunTasksInQueue(int maxDegreeOfParallelism = 8)
        {
            var tasksInFlight = new List<Task>(maxDegreeOfParallelism);
            while (_pendingTaskFactories.Any() || tasksInFlight.Any())
            {
                while (tasksInFlight.Count < maxDegreeOfParallelism && _pendingTaskFactories.Any())
                {
                    Func<Task> taskFactory = _pendingTaskFactories.Dequeue();
                    tasksInFlight.Add(taskFactory());
                    this._logger.LogDebug($"Engine selected one job to run. Currently there are still {_pendingTaskFactories.Count()} jobs remaining. {tasksInFlight.Count} jobs running.");
                }

                Task completedTask = await Task.WhenAny(tasksInFlight).ConfigureAwait(false);
                await completedTask.ConfigureAwait(false);
                this._logger.LogInformation($"Engine finished one job. Currently there are still {_pendingTaskFactories.Count()} jobs remaining. {tasksInFlight.Count} jobs running.");
                tasksInFlight.Remove(completedTask);
            }
        }

[TestMethod]
        public async Task TestRetry()
        {
            var connection = new RetryableWebSocketConnection<Book>("wss://aaaa.bbbb.ccccccc/aaaaaa/dddddd/eeeee/fff.ares");
            var retryCounts = 0;
            var startTime = DateTime.UtcNow;
            connection.PropertyChanged += (_, _) =>
            {
                Console.WriteLine($"{(DateTime.UtcNow - startTime).TotalSeconds} seconds preplaceded. Retried: {connection.AttemptCount} times.");
                retryCounts = connection.AttemptCount;
            };
            var pullTask = connection.PullAndMonitor(null, () => string.Empty, null, true);
            // 0 + 1 + 2 + 4     = 7
            // 0 + 1 + 2 + 4 + 8 = 15
            // When at 14, must tried 4 times.

            var waitTask = Task.Delay(14 * 1000);
            await Task.WhenAny(pullTask, waitTask);
            replacedert.AreEqual(4, retryCounts);
        }

[Fact]
        public async Task PlainSink_should_resume_on_deserialization_errors()
        {
            var callCount = 0;
            Directive Decider(Exception cause)
            {
                callCount++;
                switch (cause)
                {
                    case ProduceException<Null, string> ex when ex.Error.IsSerializationError():
                        return Directive.Resume;
                    default:
                        return Directive.Stop;
                }
            }

            var elementsCount = 10;
            var topic1 = CreateTopic(1);
            var group1 = CreateGroup(1);
            
            var producerSettings = ProducerSettings<Null, string>
                .Create(Sys, null, new FailingSerializer())
                .WithBootstrapServers(Fixture.KafkaServer);
            
            var sourceTask = Source
                .From(new []{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
                .Select(elem => new ProducerRecord<Null, string>(new TopicParreplacedion(topic1, 0), elem.ToString()))
                .RunWith(
                    KafkaProducer.PlainSink(producerSettings).WithAttributes(ActorAttributes.CreateSupervisionStrategy(Decider)), 
                    Materializer);
            
            var timeoutTask = Task.Delay(TimeSpan.FromSeconds(5));
            var completeTask = await Task.WhenAny(sourceTask, timeoutTask);
            if (completeTask == timeoutTask)
                throw new Exception("Producer timed out");

            var settings = CreateConsumerSettings<Null, string>(group1).WithValueDeserializer(new StringDeserializer());
            var probe = KafkaConsumer
                .PlainSource(settings, Subscriptions.replacedignment(new TopicParreplacedion(topic1, 0)))
                .Select(c => c.Value)
                .RunWith(this.SinkProbe<string>(), Materializer);

            probe.Request(elementsCount);
            for (var i = 0; i < 9; i++)
            {
                Log.Info($">>>>>>>>>>> {i}");
                probe.ExpectNext();
            }
            callCount.Should().Be(1);
            probe.Cancel();
        }

async Task<Task<TResult>> TaskWhenAny<TResult>(IEnumerable<Task<TResult>> tasks)
        {
            try
            {
                return await Task.WhenAny(tasks);
            }
            catch (TaskCanceledException e)
            {
                return (Task<TResult>)e.Task;
            }
        }

public async ValueTask<bool> MoveNextAsync()
        {
            if (_tasks.Count == 0)
                return false;
            
            var finishedSegmentTask = await Task.WhenAny(_tasks).ConfigureAwait(false);
            var finishedSegment = await finishedSegmentTask.ConfigureAwait(false);
            if (finishedSegment.Page.PaginationToken == null)
            {
                _tasks.Remove(finishedSegmentTask);
            }
            else
            {
                var finishedIndex = _tasks.IndexOf(finishedSegmentTask);
                _tasks[finishedIndex] = ScanSegmentAsync(_asyncEnumerable.TableName, new PaginationTokenNode(finishedSegment.Page.PaginationToken, _asyncEnumerable.TotalSegmentsNode),
                    finishedSegment.Segment,
                    _cts.Token);
            }

            Current = finishedSegment.Page.Items;
            return true;
        }

public static async Task<T> WithCancellation<T>(this Task<T> task, CancellationToken cancellationToken)
        {
            var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);

            using (cancellationToken.Register(state => ((TaskCompletionSource)state).TrySetResult(), tcs))
            {
                var resultTask = await Task.WhenAny(task, tcs.Task);
                if (resultTask == tcs.Task)
                {
                    throw new OperationCanceledException(cancellationToken);
                }

                return await task;
            }
        }

private static async Task<int> RunClientAsync(ILogger logger, CancellationToken cancellationToken)
		{
			Console.Write("Connection Type [1] TCP, [2] RS485: ");
			int cType = Convert.ToInt32(Console.ReadLine().Trim());

			IModbusClient client = null;
			try
			{
				switch (cType)
				{
					case 1:
						{
							Console.Write("Hostname: ");
							string host = Console.ReadLine().Trim();
							Console.Write("Port: ");
							int port = Convert.ToInt32(Console.ReadLine().Trim());

							client = new TcpClient(host, port, logger);
						}
						break;
					case 2:
						{
							Console.Write("Interface: ");
							string port = Console.ReadLine().Trim();

							Console.Write("Baud: ");
							int baud = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Stop-Bits [0|1|2|3=1.5]: ");
							int stopBits = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Parity [0] None [1] Odd [2] Even [3] Mark [4] Space: ");
							int parity = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Handshake [0] None [1] X-On/Off [2] RTS [3] RTS+X-On/Off: ");
							int handshake = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Timeout (ms): ");
							int timeout = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Set Driver to RS485 [0] No [1] Yes: ");
							int setDriver = Convert.ToInt32(Console.ReadLine().Trim());

							client = new SerialClient(port)
							{
								BaudRate = (BaudRate)baud,
								DataBits = 8,
								StopBits = (StopBits)stopBits,
								Parity = (Parity)parity,
								Handshake = (Handshake)handshake,
								SendTimeout = TimeSpan.FromMilliseconds(timeout),
								ReceiveTimeout = TimeSpan.FromMilliseconds(timeout)
							};

							if (setDriver == 1)
							{
								((SerialClient)client).DriverEnableRS485 = true;
							}
						}
						break;
					default:
						Console.Error.WriteLine($"Unknown type: {cType}");
						return 1;
				}

				await Task.WhenAny(client.Connect(), Task.Delay(Timeout.Infinite, cancellationToken));
				if (cancellationToken.IsCancellationRequested)
					return 0;

				while (!cancellationToken.IsCancellationRequested)
				{
					Console.Write("Device ID: ");
					byte id = Convert.ToByte(Console.ReadLine().Trim());

					Console.Write("Function [1] Read Register, [2] Device Info, [9] Write Register : ");
					int fn = Convert.ToInt32(Console.ReadLine().Trim());

					switch (fn)
					{
						case 1:
							{
								ushort address = 0;
								ushort count = 0;
								string type = "";

								Console.WriteLine();
								Console.Write("Address : ");
								address = Convert.ToUInt16(Console.ReadLine().Trim());
								Console.Write("DataType: ");
								type = Console.ReadLine().Trim();
								if (type == "string")
								{
									Console.Write("Register Count: ");
									count = Convert.ToUInt16(Console.ReadLine().Trim());
								}

								Console.WriteLine();
								Console.Write("Run as loop? [y/N]: ");
								string loop = Console.ReadLine().Trim().ToLower();
								int interval = 0;
								if (yesList.Contains(loop))
								{
									Console.Write("Loop interval (milliseconds): ");
									interval = Convert.ToInt32(Console.ReadLine().Trim());
								}

								Console.WriteLine();
								do
								{
									try
									{
										Console.Write("Result  : ");
										List<Register> result = null;
										switch (type.Trim().ToLower())
										{
											case "byte":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetByte());
												break;
											case "ushort":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetUInt16());
												break;
											case "uint":
												result = await client.ReadHoldingRegisters(id, address, 2);
												Console.WriteLine(result?.GetUInt32());
												break;
											case "ulong":
												result = await client.ReadHoldingRegisters(id, address, 4);
												Console.WriteLine(result?.GetUInt64());
												break;
											case "sbyte":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetSByte());
												break;
											case "short":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetInt16());
												break;
											case "int":
												result = await client.ReadHoldingRegisters(id, address, 2);
												Console.WriteLine(result?.GetInt32());
												break;
											case "long":
												result = await client.ReadHoldingRegisters(id, address, 4);
												Console.WriteLine(result?.GetInt64());
												break;
											case "float":
												result = await client.ReadHoldingRegisters(id, address, 2);
												Console.WriteLine(result?.GetSingle());
												break;
											case "double":
												result = await client.ReadHoldingRegisters(id, address, 4);
												Console.WriteLine(result?.GetDouble());
												break;
											case "string":
												result = await client.ReadHoldingRegisters(id, address, count);
												Console.WriteLine();
												Console.WriteLine("UTF8:             " + result?.GetString(count));
												Console.WriteLine("Unicode:          " + result?.GetString(count, 0, Encoding.Unicode));
												Console.WriteLine("BigEndianUnicode: " + result?.GetString(count, 0, Encoding.BigEndianUnicode));
												break;
											default:
												Console.Write("DataType unknown");
												break;
										}
									}
									catch
									{ }
									await Task.Delay(TimeSpan.FromMilliseconds(interval), cancellationToken);
								}
								while (interval > 0 && !cancellationToken.IsCancellationRequested);
							}
							break;
						case 2:
							{
								Console.Write("[1] Basic, [2] Regular, [3] Extended: ");
								int cat = Convert.ToInt32(Console.ReadLine().Trim());

								Dictionary<DeviceIDObject, string> info = null;
								switch (cat)
								{
									case 1:
										info = await client.ReadDeviceInformation(id, DeviceIDCategory.Basic);
										break;
									case 2:
										info = await client.ReadDeviceInformation(id, DeviceIDCategory.Regular);
										break;
									case 3:
										info = await client.ReadDeviceInformation(id, DeviceIDCategory.Extended);
										break;
								}
								if (info != null)
								{
									foreach (var kvp in info)
									{
										Console.WriteLine($"{kvp.Key}: {kvp.Value}");
									}
								}
							}
							break;
						case 9:
							{
								Console.Write("Address: ");
								ushort address = Convert.ToUInt16(Console.ReadLine().Trim());

								Console.Write("Bytes (HEX): ");
								string byteStr = Console.ReadLine().Trim();
								byteStr = byteStr.Replace(" ", "").ToLower();

								byte[] bytes = Enumerable.Range(0, byteStr.Length)
									.Where(i => i % 2 == 0)
									.Select(i => Convert.ToByte(byteStr.Substring(i, 2), 16))
									.ToArray();

								var registers = Enumerable.Range(0, bytes.Length)
									.Where(i => i % 2 == 0)
									.Select(i =>
									{
										return new Register
										{
											Type = ModbusObjectType.HoldingRegister,
											Address = address++,
											HiByte = bytes[i],
											LoByte = bytes[i + 1]
										};
									})
									.ToList();

								if (!await client.WriteRegisters(id, registers))
									throw new Exception($"Writing '{byteStr}' to address {address} failed");
							}
							break;
					}

					Console.Write("New Request? [y/N]: ");
					string again = Console.ReadLine().Trim().ToLower();
					if (!yesList.Contains(again))
						return 0;
				}

				return 0;
			}
			catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
			{
				return 0;
			}
			finally
			{
				Console.WriteLine("Disposing");
				client?.Dispose();
				Console.WriteLine("Disposed");
			}
		}

[Fact]
        public async Task TestAwaiting()
        {
            Event = null;

            // add handlers
            Event += (sender, args) => Task.CompletedTask;
            Event += (sender, args) => Task.Delay(100);
            Event += (sender, args) => Task.Delay(1000);

            var task = Event.InvokeAsync(this);

            // ensure that the task did not complete in 300 milliseconds
            replacedert.NotSame(task, await Task.WhenAny(Task.Delay(300), task));
        }

public static async Task YTDLMetadata(string id, IServerApplicationPaths appPaths, CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();
            var ytd = new YoutubeDL();
            ytd.Options.FilesystemOptions.WriteInfoJson = true;
            ytd.Options.VerbositySimulationOptions.SkipDownload = true;
            // Pulled from above, might want to abstract
            var dataPath = Path.Combine(appPaths.CachePath, "youtubemetadata", id, "ytvideo");
            ytd.Options.FilesystemOptions.Output = dataPath;
            var dlstring = "https://www.youtube.com/watch?v=" + id;
            List<string> ytdl_errs = new();
            ytd.StandardErrorEvent += (sender, error) => ytdl_errs.Add(error);
            var task = ytd.DownloadAsync(dlstring);
            if (await Task.WhenAny(task, Task.Delay(10000, cancellationToken)) == task)
            {
                await task;
            }
            else
            {
                throw new Exception(String.Format("Timeout error for video id: {0}, errors: {1}", id, String.Join(" ", ytdl_errs)));
            }

        }

public virtual async Task<IEnumerable<FtpEntry>> ListFilesRecursiveAsync(string startPath = null, Predicate<FtpEntry> skipFolder = null, CancellationToken ctk = default)
        {
            _logger.Trace("List files starting from path: {0}", startPath);

            if (skipFolder == null)
                skipFolder = x => false;

            Stack<FtpEntry> pendingFolders = new Stack<FtpEntry>();
            IEnumerable<FtpEntry> files = new List<FtpEntry>();
            List<Task<IEnumerable<FtpEntry>>> running = new List<Task<IEnumerable<FtpEntry>>>();

            async Task<IEnumerable<FtpEntry>> listFolderAsync(string path, CancellationToken ct)
            {
                var retrier = Policy
                    .Handle<Exception>()
                    .WaitAndRetryAsync(new[]
                    {
                        TimeSpan.FromSeconds(1),
                    }, (ex, ts) =>
                    {
                        _logger.Warn(ex, "Failed to list folder {0}. Try again soon ...", path);
                    });

                return await retrier.ExecuteAsync(async ct1 =>
                {
                    return await this.ListDirectoryAsync(path, ct1).ConfigureAwait(false);
                }, ct).ConfigureAwait(false);

                
            }

            void startListing(string path)
            {
                running.Add(Task.Run(() => listFolderAsync(path, ctk), ctk));
            }

            startListing(startPath);

            try
            {
                while (running.Count > 0)
                {
                    var t = await Task.WhenAny(running);

                    var list = await t;
                    running.Remove(t); // remove only if successful

                    foreach (var d in list.Where(x => x.IsDirectory && !x.Name.Equals(".") && !x.Name.Equals("..")))
                    {
                        if (skipFolder.Invoke(d))
                            _logger.Info("Skipping folder: {0}", d.FullPath);
                        else
                            pendingFolders.Push(d);
                    }

                    files = files.Concat(list.Where(x => !x.IsDirectory).ToList());

                    while (pendingFolders.Count > 0 && running.Count < this.MaxListingRecursiveParallelism)
                        startListing(pendingFolders.Pop().FullPath);                
                }
            }
            catch
            {
                await Task.WhenAll(running); // this still contains the failed one 
                throw;
            }


            return files;
        }

private async Task<string?> CatFile(string type, string id)
		{
			await catFileSemapreplaced.WaitAsync();
			try
			{
				bool isFirst = false;
				if (CatFileProcess is null)
				{
					isFirst = true;
					Log?.Verbatim($"{Root} $ git cat-file --batch");
					ProcessStartInfo info = new()
					{
						FileName = "git",
						Arguments = "cat-file --batch",
						WorkingDirectory = Root,
						RedirectStandardError = false,
						RedirectStandardOutput = true,
						RedirectStandardInput = true,
						UseShellExecute = false,
					};
					CatFileProcess = Process.Start(info);
				}

				var (cin, cout) = (CatFileProcess.StandardInput, CatFileProcess.StandardOutput);

				// if this git call is forwarded onto another shell script,
				// then it's possible to query git before it's ready, but once
				// it does respond, it's ready to be used.
				if (isFirst)
				{
					Log?.Verbatim($"First run: awaiting cat-file startup.");
					await cin.WriteLineAsync(" ");

					using var cts = new CancellationTokenSource();

					var timeout = Task.Delay(5000, cts.Token);
					var gotBack = cout.ReadLineAsync();

					var completedTask = await Task.WhenAny(timeout, gotBack);

					if (completedTask != timeout)
						cts.Cancel();
					else
						throw new UnknownGitException("The git cat-file process timed out.");

					var result = await gotBack;
					if (result != "  missing")
						throw new UnknownGitException($"The git cat-file process returned unexpected output: {result}");
				}

				await cin.WriteLineAsync(id);
				string line = await cout.ReadLineAsync();
				string[] response = line.Split(' ');

				Log?.Verbatim($"git cat-file < {id}");
				Log?.Verbatim($"git cat-file > {line}");

				if (response[0] != id)
					throw new UnknownGitException("The returned blob hash did not match.");
				else if (response[1] == "missing")
					return null;
				else if (response[1] != type)
					throw new UnknownGitException($"Blob for {id} expected {type} but was {response[1]}.");

				int length = int.Parse(response[2], CultureInfo.InvariantCulture);

				var buffer = new char[length];
				await cout.ReadBlockAsync(buffer, 0, length);
				await cout.ReadLineAsync(); // git appends a linefeed

				return new string(buffer);
			}
			catch (Exception ex)
			{
				throw new UnknownGitException($"Failed to communicate with the git cat-file process: {ex.Message}");
			}
			finally
			{
				catFileSemapreplaced.Release();
			}
		}

[SuppressMessage("ReSharper", "AccessToDisposedClosure", Justification = "This is safe because the cancellation token source's WaitHandle is not used.")]
        public async Task<SearchResult> Execute(WaitForCommand command)
        {
            using var cts = new CancellationTokenSource();

            var tasks = command.Elements.Select(async element => await this.WaitFor(element, command, cts.Token).ConfigureAwait(false));

            var firstTaskToFinish = await Task.WhenAny(tasks).ConfigureAwait(false);
            cts.Cancel();

            // Rethrow any exception
            if (firstTaskToFinish.IsFaulted)
            {
                await firstTaskToFinish.ConfigureAwait(false);
            }

            return await this.TrimRecognizerResultAndThrowIfRequired(command, firstTaskToFinish.Result).ConfigureAwait(false);
        }

private void OnStopping()
        {
            var shutdown = _cluster.ShutdownAsync();
            var timeout = Task.Delay(15000);
            Task.WhenAny(shutdown, timeout).GetAwaiter().GetResult();
            if (timeout.IsCompleted)
                _logger.LogError("Shut down cluster timed out...");
        }

private async ValueTask<(ResponseStatus Ok, T?)> TryRequestAsync<T>(
            ClusterIdenreplacedy clusterIdenreplacedy,
            object message,
            PID pid,
            PidSource source,
            ISenderContext context,
            IFuture future
        )
        {
            var t = Stopwatch.StartNew();

            try
            {
                context.Request(pid, message, future.Pid);
                var task = future.Task;
                await Task.WhenAny(task, _clock.CurrentBucket);

                if (task.IsCompleted)
                {
                    var res = task.Result;

                    return ToResult<T>(source, context, res);
                }

                if (!context.System.Shutdown.IsCancellationRequested)
                    Logger.LogDebug("TryRequestAsync timed out, PID from {Source}", source);
                _pidCache.RemoveByVal(clusterIdenreplacedy, pid);

                return (ResponseStatus.TimedOut, default);
            }
            catch (TimeoutException)
            {
                return (ResponseStatus.TimedOut, default);
            }
            catch (Exception x)
            {
                if (!context.System.Shutdown.IsCancellationRequested && _requestLogThrottle().IsOpen())
                    Logger.LogDebug(x, "TryRequestAsync failed with exception, PID from {Source}", source);
                _pidCache.RemoveByVal(clusterIdenreplacedy, pid);
                return (ResponseStatus.Exception, default);
            }
            finally
            {
                if (!context.System.Metrics.IsNoop)
                {
                    var elapsed = t.Elapsed;
                    context.System.Metrics.Get<ClusterMetrics>().ClusterRequestHistogram
                        .Observe(elapsed, new[]
                            {
                                context.System.Id, context.System.Address, clusterIdenreplacedy.Kind, message.GetType().Name,
                                source == PidSource.Cache ? "PidCache" : "IIdenreplacedyLookup"
                            }
                        );
                }
            }
        }

public async Task<T?> RequestAsync<T>(ClusterIdenreplacedy clusterIdenreplacedy, object message, ISenderContext context, CancellationToken ct)
        {
            var start = Stopwatch.StartNew();
            var i = 0;

            var future = context.GetFuture();
            PID? lastPid = null;

            try
            {
                while (!ct.IsCancellationRequested)
                {
                    if (context.System.Shutdown.IsCancellationRequested) return default;

                    i++;

                    var source = PidSource.Cache;
                    var pid = GetCachedPid(clusterIdenreplacedy);

                    if (pid is null)
                    {
                        source = PidSource.Lookup;
                        pid = await GetPidFromLookup(clusterIdenreplacedy, context, ct);
                    }

                    if (context.System.Shutdown.IsCancellationRequested) return default;

                    if (pid is null)
                    {
                        Logger.LogDebug("Requesting {ClusterIdenreplacedy} - Did not get PID from IdenreplacedyLookup", clusterIdenreplacedy);
                        await Task.Delay(i * 20, CancellationToken.None);
                        continue;
                    }

                    // Ensures that a future is not re-used against another actor.
                    if (lastPid is not null && !pid.Equals(lastPid)) RefreshFuture();

                    var t = Stopwatch.StartNew();

                    try
                    {
                        context.Request(pid, message, future.Pid);
                        var task = future.Task;

                        await Task.WhenAny(task, _clock.CurrentBucket);

                        if (task.IsCompleted)
                        {
                            var (status, result) = ToResult<T>(source, context, task.Result);

                            switch (status)
                            {
                                case ResponseStatus.Ok: return result;
                                case ResponseStatus.InvalidResponse:
                                    RefreshFuture();
                                    await RemoveFromSource(clusterIdenreplacedy, source, pid);
                                    break;
                                case ResponseStatus.DeadLetter:
                                    RefreshFuture();
                                    await RemoveFromSource(clusterIdenreplacedy, PidSource.Lookup, pid);
                                    break;
                            }
                        }
                        else
                        {
                            if (!context.System.Shutdown.IsCancellationRequested)
                                Logger.LogDebug("TryRequestAsync timed out, PID from {Source}", source);
                            _pidCache.RemoveByVal(clusterIdenreplacedy, pid);
                        }
                    }
                    catch (TimeoutException)
                    {
                        lastPid = pid;
                        await RemoveFromSource(clusterIdenreplacedy, PidSource.Cache, pid);
                        continue;
                    }
                    catch (Exception x)
                    {
                        if (!context.System.Shutdown.IsCancellationRequested && _requestLogThrottle().IsOpen())
                            Logger.LogDebug(x, "TryRequestAsync failed with exception, PID from {Source}", source);
                        _pidCache.RemoveByVal(clusterIdenreplacedy, pid);
                        RefreshFuture();
                        await RemoveFromSource(clusterIdenreplacedy, PidSource.Cache, pid);
                        await Task.Delay(i * 20, CancellationToken.None);
                        continue;
                    }
                    finally
                    {
                        if (!context.System.Metrics.IsNoop)
                        {
                            var elapsed = t.Elapsed;
                            context.System.Metrics.Get<ClusterMetrics>().ClusterRequestHistogram
                                .Observe(elapsed, new[]
                                    {
                                        context.System.Id, context.System.Address, clusterIdenreplacedy.Kind, message.GetType().Name,
                                        source == PidSource.Cache ? "PidCache" : "IIdenreplacedyLookup"
                                    }
                                );
                        }
                    }

                    if (!context.System.Metrics.IsNoop)
                    {
                        context.System.Metrics.Get<ClusterMetrics>().ClusterRequestRetryCount.Inc(new[]
                            {context.System.Id, context.System.Address, clusterIdenreplacedy.Kind, message.GetType().Name}
                        );
                    }
                }

                if (!context.System.Shutdown.IsCancellationRequested && _requestLogThrottle().IsOpen())
                {
                    Logger.LogWarning("RequestAsync retried but failed for {ClusterIdenreplacedy}, elapsed {Time}", clusterIdenreplacedy, start.Elapsed);
                }

                return default!;
            }
            finally
            {
                future.Dispose();
            }

            void RefreshFuture()
            {
                future.Dispose();
                future = context.GetFuture();
                lastPid = null;
            }
        }

[Fact]
        public async Task PoisonReturnsIfPidDoesNotExist()
        {
            var deadPid = PID.FromAddress(System.Address, "nowhere");
            var timeout = Task.Delay(2000);

            var poisonTask = Context.PoisonAsync(deadPid);

            await Task.WhenAny(timeout, poisonTask);

            poisonTask.IsCompleted.Should().BeTrue("Or we did not get a response when poisoning a missing pid");
        }

[Fact]
        public async Task PoisonTerminatesActor()
        {
            var pid = Context.Spawn(EchoProps);

            const string message = "hello";
            (await Context.RequestAsync<string>(pid, message)).Should().Be(message);

            var timeout = Task.Delay(5000);
            var poisonTask = Context.PoisonAsync(pid);
            await Task.WhenAny(timeout, poisonTask);

            poisonTask.IsCompleted.Should().BeTrue("Or we did not get a response when poisoning a live pid");

            Context.Invoking(ctx => ctx.RequestAsync<string>(pid, message)).Should().ThrowExactly<DeadLetterException>();
        }

public async Task<bool> TopologyConsensus(CancellationToken ct)
        {
            while (!ct.IsCancellationRequested)
            {
                var t = _topologyConsensus.Task;
                // ReSharper disable once MethodSupportsCancellation
                await Task.WhenAny(t, Task.Delay(500));
                if (t.IsCompleted)
                    return true;                
            }

            return false;
        }

[Fact]
        public async Task TopologiesShouldHaveConsensus()
        {
            var timeout = Task.Delay(20000);
        
            var consensus = Task.WhenAll(Members.Select(member => member.MemberList.TopologyConsensus(CancellationTokens.FromSeconds(20))));
        
            await Task.WhenAny(timeout, consensus);

            _testOutputHelper.WriteLine(LogStore.ToFormattedString());
            timeout.IsCompleted.Should().BeFalse();
        }

async Task LoadBatch(int quanreplacedy, NativeArray<byte> data, bool requireNormals = false, bool requireTangents = false) {

            var tasks = new List<Task<Mesh>>(quanreplacedy);
        
            for (var i = 0; i < quanreplacedy; i++)
            {
                DracoMeshLoader dracoLoader = new DracoMeshLoader();
                var task = dracoLoader.ConvertDracoMeshToUnity(data,requireNormals,requireTangents);
                tasks.Add(task);
            }

            while (tasks.Count > 0) {
                var task = await Task.WhenAny(tasks);
                tasks.Remove(task);
                var mesh = await task;
                if (mesh == null) {
                    Debug.LogError("Loading mesh failed");
                }
                else {
                    if (requireNormals) {
                        var normals = mesh.normals;
                        replacedert.Greater(normals.Length,0);
                    }
                    if (requireTangents) {
                        var tangents = mesh.tangents;
                        replacedert.Greater(tangents.Length,0);
                    }
                }
            }
            await Task.Yield();
        }

public async Task<int> RunAsync(Action<string> outputCallback = null, Action<string> errorCallback = null, TimeSpan? timeout = null)
        {
            var processInfo = new ProcessStartInfo
            {
                FileName = _exeName,
                Arguments = _arguments,
                CreateNoWindow = !_visibleProcess,
                UseShellExecute = _shareConsole,
                RedirectStandardError = _streamOutput,
                RedirectStandardInput = _streamOutput,
                RedirectStandardOutput = _streamOutput,
                WorkingDirectory = _workingDirectory ?? Environment.CurrentDirectory
            };

            try
            {
                Process = Process.Start(processInfo);
            }
            catch (Win32Exception ex)
            {
                if (ex.Message == "The system cannot find the file specified")
                {
                    throw new FileNotFoundException(ex.Message, ex);
                }
                throw ex;
            }

            if (_streamOutput)
            {
                Process.OutputDataReceived += (s, e) => outputCallback?.Invoke(e.Data);
                Process.BeginOutputReadLine();
                Process.ErrorDataReceived += (s, e) => errorCallback?.Invoke(e.Data);
                Process.BeginErrorReadLine();
                Process.EnableRaisingEvents = true;
            }

            var exitCodeTask = Process.WaitForExitAsync();

            if (timeout == null)
                return await exitCodeTask;
            else
            {
                await Task.WhenAny(exitCodeTask, Task.Delay(timeout.Value));

                if (exitCodeTask.IsCompleted)
                    return exitCodeTask.Result;
                else
                {
                    Process.Kill();
                    throw new Exception("Process didn't exit within specified timeout");
                }
            }
        }

public static async Task PrefetchInParallelAsync(
            IEnumerable<IPrefetcher> prefetchers,
            int maxConcurrency,
            ITrace trace,
            CancellationToken cancellationToken)
        {
            if (prefetchers == null)
            {
                throw new ArgumentNullException(nameof(prefetchers));
            }

            if (trace == null)
            {
                throw new ArgumentNullException(nameof(trace));
            }

            using (ITrace prefetchTrace = trace.StartChild(name: "Prefetching", TraceComponent.Pagination, TraceLevel.Info))
            {
                HashSet<Task> tasks = new HashSet<Task>();
                IEnumerator<IPrefetcher> prefetchersEnumerator = prefetchers.GetEnumerator();
                for (int i = 0; i < maxConcurrency; i++)
                {
                    if (!prefetchersEnumerator.MoveNext())
                    {
                        break;
                    }

                    IPrefetcher prefetcher = prefetchersEnumerator.Current;
                    tasks.Add(Task.Run(async () => await prefetcher.PrefetchAsync(prefetchTrace, cancellationToken)));
                }

                while (tasks.Count != 0)
                {
                    Task completedTask = await Task.WhenAny(tasks);
                    tasks.Remove(completedTask);
                    try
                    {
                        await completedTask;
                    }
                    catch
                    {
                        // Observe the remaining tasks
                        try
                        {
                            await Task.WhenAll(tasks);
                        }
                        catch
                        {
                        }

                        throw;
                    }

                    if (prefetchersEnumerator.MoveNext())
                    {
                        IPrefetcher bufferable = prefetchersEnumerator.Current;
                        tasks.Add(Task.Run(async () => await bufferable.PrefetchAsync(prefetchTrace, cancellationToken)));
                    }
                }
            }
        }

public async Task<AccountProperties> GetAccountPropertiesAsync()
            {
                // If there are no preferred regions then just wait for the global endpoint results
                if (this.Locations == null)
                {
                    return await this.GetOnlyGlobalEndpointAsync();
                }

                Task globalEndpointTask = this.GetAndUpdateAccountPropertiesAsync(this.DefaultEndpoint);

                // Start a timer to start secondary requests in parallel.
                Task timerTask = Task.Delay(TimeSpan.FromSeconds(5));
                await Task.WhenAny(globalEndpointTask, timerTask);
                if (this.AccountProperties != null)
                {
                    return this.AccountProperties;
                }

                if (this.NonRetriableException != null)
                {
                    ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw();
                }

                // Start 2 additional tasks to try to get the account information
                // from the preferred region list since global account has not succeed yet.
                HashSet<Task> tasksToWaitOn = new HashSet<Task>
                {
                    globalEndpointTask,
                    this.TryGetAccountPropertiesFromAllLocationsAsync(),
                    this.TryGetAccountPropertiesFromAllLocationsAsync()
                };

                while (tasksToWaitOn.Any())
                {
                    Task completedTask = await Task.WhenAny(tasksToWaitOn);
                    if (this.AccountProperties != null)
                    {
                        return this.AccountProperties;
                    }

                    if (this.NonRetriableException != null)
                    {
                        ExceptionDispatchInfo.Capture(this.NonRetriableException).Throw();
                    }

                    tasksToWaitOn.Remove(completedTask);
                }

                if (this.TransientExceptions.Count == 0)
                {
                    throw new ArgumentException("Account properties and NonRetriableException are null and there are no TransientExceptions.");
                }

                if (this.TransientExceptions.Count == 1)
                {
                    ExceptionDispatchInfo.Capture(this.TransientExceptions[0]).Throw();
                }

                throw new AggregateException(this.TransientExceptions);
            }

public static async Task Main(string[] args)
        {
            using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder
                        .AddConsole());
            
            ILogger logger = loggerFactory.CreateLogger<Program>();

            try
            {
                CTLConfig config = CTLConfig.From(args);

                SetEnvironmentVariables(config);
              
                if (config.OutputEventTraces)
                {
                    EnableTraceSourcesToConsole();
                }

                using CosmosClient client = config.CreateCosmosClient();

                string loggingContextIdentifier = $"{config.WorkloadType}{config.LogginContext}";
                using (logger.BeginScope(loggingContextIdentifier))
                {
                    IMetricsRoot metrics = ConfigureReporting(config, logger);

                    ICTLScenario scenario = CreateScenario(config.WorkloadType);

                    using CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();

                    await scenario.InitializeAsync(
                        config: config,
                        cosmosClient: client,
                        logger: logger);

                    logger.LogInformation("Initialization completed.");

                    if(client.ClientOptions.EnableClientTelemetry.GetValueOrDefault()) {
                        logger.LogInformation("Telemetry is enabled for CTL.");
                    } else {
                        logger.LogInformation("Telemetry is disabled for CTL.");
                    }

                    List<Task> tasks = new List<Task>
                    {
                        scenario.RunAsync(
                        config: config,
                        cosmosClient: client,
                        logger: logger,
                        metrics: metrics,
                        loggingContextIdentifier: loggingContextIdentifier,
                        cancellationToken: cancellationTokenSource.Token),

                        Task.Run(async () =>
                        { 
                            // Measure CPU/memory
                            Process process = Process.GetCurrentProcess();
                            
                            GaugeOptions processPhysicalMemoryGauge = new GaugeOptions
                            {
                                Name = "Process Working Set",
                                MeasurementUnit = Unit.Bytes,
                                Context = loggingContextIdentifier
                            };

                            GaugeOptions totalCpuGauge = new GaugeOptions
                            {
                                Name = "Total CPU",
                                MeasurementUnit = Unit.Percent,
                                Context = loggingContextIdentifier
                            };

                            GaugeOptions priviledgedCpuGauge = new GaugeOptions
                            {
                                Name = "Priviledged CPU",
                                MeasurementUnit = Unit.Percent,
                                Context = loggingContextIdentifier
                            };

                            GaugeOptions userCpuGauge = new GaugeOptions
                            {
                                Name = "User CPU",
                                MeasurementUnit = Unit.Percent,
                                Context = loggingContextIdentifier
                            };

                            DateTime lastTimeStamp = process.StartTime;
                            TimeSpan lastTotalProcessorTime = TimeSpan.Zero;
                            TimeSpan lastUserProcessorTime = TimeSpan.Zero;
                            TimeSpan lastPrivilegedProcessorTime = TimeSpan.Zero;

                            while (!cancellationTokenSource.Token.IsCancellationRequested)
                            {
                                await Task.Delay(TimeSpan.FromSeconds(config.ReportingIntervalInSeconds));
                                process.Refresh();

                                double totalCpuTimeUsed = process.TotalProcessorTime.TotalMilliseconds - lastTotalProcessorTime.TotalMilliseconds;
                                double privilegedCpuTimeUsed = process.PrivilegedProcessorTime.TotalMilliseconds - lastPrivilegedProcessorTime.TotalMilliseconds;
                                double userCpuTimeUsed = process.UserProcessorTime.TotalMilliseconds - lastUserProcessorTime.TotalMilliseconds;

                                lastTotalProcessorTime = process.TotalProcessorTime;
                                lastPrivilegedProcessorTime = process.PrivilegedProcessorTime;
                                lastUserProcessorTime = process.UserProcessorTime;

                                double cpuTimeElapsed = (DateTime.UtcNow - lastTimeStamp).TotalMilliseconds * Environment.ProcessorCount;
                                lastTimeStamp = DateTime.UtcNow;

                                metrics.Measure.Gauge.SetValue(totalCpuGauge, totalCpuTimeUsed * 100 / cpuTimeElapsed);
                                metrics.Measure.Gauge.SetValue(priviledgedCpuGauge, privilegedCpuTimeUsed * 100 / cpuTimeElapsed);
                                metrics.Measure.Gauge.SetValue(userCpuGauge, userCpuTimeUsed * 100 / cpuTimeElapsed);
                                metrics.Measure.Gauge.SetValue(processPhysicalMemoryGauge, process.WorkingSet64);

                                await Task.WhenAll(metrics.ReportRunner.RunAllAsync());
                            }
                        })
                    };

                    await Task.WhenAny(tasks);
                    cancellationTokenSource.Cancel();
                    // Final report
                    await Task.WhenAll(metrics.ReportRunner.RunAllAsync());

                    logger.LogInformation($"{nameof(CosmosCTL)} completed successfully.");
                }
            }
            catch (Exception ex)
            {
                Utils.LogError(logger, "Unhandled exception during execution", ex);
            }
        }

private Task InvokeAPIFromAction(AsyncAction action)
        {
            Task fireAndForgetTask = Task.CompletedTask;
            Task task = null;
            switch (action.ActionType)
            {
                case AsyncActionType.CallActivity:
                    task = this.context.CallActivityAsync(action.FunctionName, action.Input);
                    break;
                case AsyncActionType.CreateTimer:
                    DurableOrchestrationContext ctx = this.context as DurableOrchestrationContext;
                    using (var cts = new CancellationTokenSource())
                    {
                        if (ctx != null)
                        {
                            ctx.ThrowIfInvalidTimerLengthForStorageProvider(action.FireAt);
                        }

                        task = this.context.CreateTimer(action.FireAt, cts.Token);

                        if (action.IsCanceled)
                        {
                            cts.Cancel();
                        }
                    }

                    break;
                case AsyncActionType.CallActivityWithRetry:
                    task = this.context.CallActivityWithRetryAsync(action.FunctionName, action.RetryOptions, action.Input);
                    break;
                case AsyncActionType.CallSubOrchestrator:
                    task = this.context.CallSubOrchestratorAsync(action.FunctionName, action.InstanceId, action.Input);
                    break;
                case AsyncActionType.CallSubOrchestratorWithRetry:
                    task = this.context.CallSubOrchestratorWithRetryAsync(action.FunctionName, action.RetryOptions, action.InstanceId, action.Input);
                    break;
                case AsyncActionType.CallEnreplacedy:
                    {
                        var enreplacedyId = EnreplacedyId.GetEnreplacedyIdFromSchedulerId(action.InstanceId);
                        task = this.context.CallEnreplacedyAsync(enreplacedyId, action.EnreplacedyOperation, action.Input);
                        break;
                    }

                case AsyncActionType.SignalEnreplacedy:
                    {
                        // We do not add a task because this is 'fire and forget'
                        var enreplacedyId = EnreplacedyId.GetEnreplacedyIdFromSchedulerId(action.InstanceId);
                        this.context.SignalEnreplacedy(enreplacedyId, action.EnreplacedyOperation, action.Input);
                        task = fireAndForgetTask;
                        break;
                    }

                case AsyncActionType.ScheduledSignalEnreplacedy:
                    {
                        // We do not add a task because this is 'fire and forget'
                        var enreplacedyId = EnreplacedyId.GetEnreplacedyIdFromSchedulerId(action.InstanceId);
                        this.context.SignalEnreplacedy(enreplacedyId, action.FireAt, action.EnreplacedyOperation, action.Input);
                        task = fireAndForgetTask;
                        break;
                    }

                case AsyncActionType.ContinueAsNew:
                    this.context.ContinueAsNew(action.Input);
                    task = fireAndForgetTask;
                    break;
                case AsyncActionType.WaitForExternalEvent:
                    task = this.context.WaitForExternalEvent<object>(action.ExternalEventName);
                    break;
                case AsyncActionType.CallHttp:
                    task = this.context.CallHttpAsync(action.HttpRequest);
                    break;
                case AsyncActionType.WhenAll:
                    task = Task.WhenAll(action.CompoundActions.Select(x => this.InvokeAPIFromAction(x)));
                    break;
                case AsyncActionType.WhenAny:
                    task = Task.WhenAny(action.CompoundActions.Select(x => this.InvokeAPIFromAction(x)));
                    break;
                default:
                    throw new Exception($"Received an unexpected action type from the out-of-proc function: '${action.ActionType}'.");
            }

            return task;
        }

private async Task ProcessAsyncActionsV1(AsyncAction[][] actions)
        {
            if (actions == null)
            {
                throw new ArgumentNullException("Out-of-proc orchestrator schema must have a non-null actions property.");
            }

            // Each actionSet represents a particular execution of the orchestration.
            foreach (AsyncAction[] actionSet in actions)
            {
                var tasks = new List<Task>(actions.Length);
                DurableOrchestrationContext ctx = this.context as DurableOrchestrationContext;

                // An actionSet represents all actions that were scheduled within that execution.
                Task newTask;
                foreach (AsyncAction action in actionSet)
                {
                    newTask = this.InvokeAPIFromAction(action);
                    if (newTask != Task.CompletedTask)
                    {
                        tasks.Add(newTask);
                    }
                }

                if (tasks.Count > 0)
                {
                    await Task.WhenAny(tasks);
                }
            }
        }

[MemberData(nameof(FunctionUrls))]
        [SkipIfFunctionAbsent(Section)]
        public async Task RoutingTest(string key, string url)
        {
            var target = nameof(RoutingTest) + key;
            const int count = 2;
            var users = GenerateRandomUsers(count).ToArray();
            var completionSources = new ConcurrentDictionary<string, TaskCompletionSource>();
            var tasks = Enumerable.Range(0, count).Zip(users).Select(async (pair) =>
            {
                var (endpointId, user) = pair;
                var connectionInfo = await Client.Negotiate(user, url, endpointId);
                var connection = CreateHubConnection(connectionInfo.Url, connectionInfo.AccessToken);
                var taskCompleSource = new TaskCompletionSource();
                completionSources.TryAdd(user, taskCompleSource);
                connection.On(target, (LiteServiceEndpoint endpoint) =>
                {
                    var expectedHost = new Uri(connectionInfo.Url).Host;
                    var actualHost = new Uri(endpoint.Endpoint).Host;
                    if (expectedHost.Equals(actualHost) && endpoint.Online)
                    {
                        completionSources[user].SetResult();
                    }
                    else
                    {
                        completionSources[user].SetException(new Exception($"Expected host:{expectedHost}, Actual host:{actualHost}"));
                    }
                });
                await connection.StartAsync();
                return connection;
            }).ToArray();
            var connections = await Task.WhenAll(tasks);

            // send a message to endpoint[0]
            await Client.Send(url, 0, target);
            await Task.WhenAny(completionSources.Values.Select(s => s.Task.OrTimeout()));

            // received the correct messsage
            replacedert.True(completionSources[users[0]].Task.IsCompletedSuccessfully);
            // not received messages yet
            replacedert.False(completionSources[users[1]].Task.IsCompleted);

            //reset
            foreach (var user in users)
            {
                completionSources[user] = new TaskCompletionSource();
            }

            // send a message to endpoint[1]
            await Client.Send(url, 1, target);
            await Task.WhenAny(completionSources.Values.Select(s => s.Task.OrTimeout()));
            replacedert.True(completionSources[users[1]].Task.IsCompletedSuccessfully);
            replacedert.False(completionSources[users[0]].Task.IsCompleted);
        }

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            // note that we don't want to call this on each loop iteration,
            // or we will create a backlog of waits that must be satisfied by the controller.
            // we only want one outstanding at any time
            var wait = _trigger.WaitAsync();

            while (!stoppingToken.IsCancellationRequested)
            {
                try
                {
                    var requests = await _requestStore.List(stoppingToken);
                    await Task.WhenAll(requests.Select(HandleRequest));
                }
                catch (Exception ex) when (!(ex is TaskCanceledException))
                {
                    _logger.LogError(ex, $"Unexpected error in {nameof(ScaleUpProcessorHost)} main loop");
                }

                var completed = await Task.WhenAny(Task.Delay(ResizeCheckDelay, stoppingToken), wait);
                if (completed == wait)
                {
                    // if our wait was completed we need to start a new one
                    wait = _trigger.WaitAsync();
                }
            }
        }

public async Task ShutdownAsync()
        {
            var options = _options.GracefulShutdown;
            if (options.Mode == GracefulShutdownMode.Off)
            {
                return;
            }

            try
            {
                var source = new CancellationTokenSource(_options.GracefulShutdown.Timeout);

                Log.SettingServerOffline(_logger, _hubName);

                await Task.WhenAny(
                    _serviceConnectionManager.OfflineAsync(options.Mode),
                    Task.Delay(Timeout.InfiniteTimeSpan, source.Token)
                );

                Log.TriggeringShutdownHooks(_logger, _hubName);

                await Task.WhenAny(
                    options.OnShutdown(Context),
                    Task.Delay(Timeout.InfiniteTimeSpan, source.Token)
                );

                Log.WaitingClientConnectionsToClose(_logger, _hubName);

                await Task.WhenAny(
                    _clientConnectionManager.WhenAllCompleted(),
                    Task.Delay(Timeout.InfiniteTimeSpan, source.Token)
                );
            }
            catch (OperationCanceledException)
            {
                Log.GracefulShutdownTimeoutExceeded(_logger, _hubName, Convert.ToInt32(_options.GracefulShutdown.Timeout.TotalMilliseconds));
            }

            Log.StoppingServer(_logger, _hubName);
            await _serviceConnectionManager.StopAsync();
        }

private async Task PerformDisconnectAsyncCore(string connectionId)
        {
            var connection = RemoveClientConnection(connectionId);
            if (connection != null)
            {
                // In normal close, service already knows the client is closed, no need to be informed.
                connection.AbortOnClose = false;

                // We're done writing to the application output
                // Let the connection complete incoming
                connection.CompleteIncoming();

                // wait for the connection's lifetime task to end
                var lifetime = connection.LifetimeTask;

                // Wait on the application task to complete
                // We wait gracefully here to be consistent with self-host SignalR
                await Task.WhenAny(lifetime, Task.Delay(_closeTimeOutMilliseconds));

                if (!lifetime.IsCompleted)
                {
                    Log.DetectedLongRunningApplicationTask(Logger, connectionId);
                }

                await lifetime;
            }
        }

public override async Task<string> GenerateAccessTokenAsync(
            string audience,
            IEnumerable<Claim> claims,
            TimeSpan lifetime,
            AccessTokenAlgorithm algorithm,
            CancellationToken ctoken = default)
        {
            var task = await Task.WhenAny(InitializedTask, ctoken.AsTask());

            if (task == InitializedTask || InitializedTask.IsCompleted)
            {
                await task;
                if (Authorized)
                {
                    return await base.GenerateAccessTokenAsync(audience, claims, lifetime, algorithm);
                }
                else
                {
                    throw new AzureSignalRAccessTokenNotAuthorizedException("The given AzureAD idenreplacedy don't have the permission to generate access token.");
                }
            }
            else
            {
                throw new TaskCanceledException("Timeout reached when authorizing AzureAD idenreplacedy.");
            }
        }

private async Task AddHubServiceEndpointAsync(HubServiceEndpoint endpoint, CancellationToken cancellationToken)
        {
            Log.StartAddingEndpoint(_logger, endpoint.Endpoint, endpoint.Name);

            OnAdd?.Invoke(endpoint);

            // Wait for new endpoint turn Ready or timeout getting cancelled
            var task = await Task.WhenAny(endpoint.ScaleTask, cancellationToken.AsTask());

            if (task == endpoint.ScaleTask)
            {
                Log.SucceedAddingEndpoint(_logger, endpoint.ToString());
            }

            // Set complete
            endpoint.CompleteScale();
        }

private async Task RemoveHubServiceEndpointAsync(HubServiceEndpoint endpoint, CancellationToken cancellationToken)
        {
            Log.StartRemovingEndpoint(_logger, endpoint.Endpoint, endpoint.Name);

            OnRemove?.Invoke(endpoint);

            // Wait for endpoint turn offline or timeout getting cancelled
            var task = await Task.WhenAny(endpoint.ScaleTask, cancellationToken.AsTask());

            if (task == endpoint.ScaleTask)
            {
                Log.SucceedRemovingEndpoint(_logger, endpoint.ToString());
            }

            // Set complete
            endpoint.CompleteScale();
        }

protected async Task RemoveConnectionAsync(IServiceConnection c, GracefulShutdownMode mode)
        {
            var retry = 0;
            while (retry < MaxRetryRemoveSeverConnection)
            {
                using var source = new CancellationTokenSource();
                _ = WriteFinAsync(c, mode);

                var task = await Task.WhenAny(c.ConnectionOfflineTask, Task.Delay(Constants.Periods.RemoveFromServiceTimeout, source.Token));

                if (task == c.ConnectionOfflineTask)
                {
                    source.Cancel();
                    Log.ReceivedFinAckPing(Logger);
                    return;
                }
                retry++;
            }
            Log.TimeoutWaitingForFinAck(Logger, retry);
        }

private async Task WaitForApplicationTask(ClientConnectionContext clientContext, bool closeGracefully)
        {
            clientContext.Output.TryComplete();
            var app = clientContext.ApplicationTask;
            if (!app.IsCompleted)
            {
                try
                {
                    if (!closeGracefully)
                    {
                        clientContext.CancelPendingRead();
                    }

                    using (var delayCts = new CancellationTokenSource())
                    {
                        var resultTask =
                            await Task.WhenAny(app, Task.Delay(CloseApplicationTimeout, delayCts.Token));
                        if (resultTask != app)
                        {
                            // Application task timed out and it might never end writing to Transport.Output, cancel reading the pipe so that our ProcessOutgoing ends
                            clientContext.CancelPendingRead();
                            Log.ApplicationTaskTimedOut(Logger);
                        }
                        else
                        {
                            delayCts.Cancel();
                        }
                    }
                }
                catch (Exception ex)
                {
                    Log.ApplicationTaskFailed(Logger, ex);
                }
            }
        }

private static async Task<ConcurrentDictionary<MockServiceSideConnection, int>> DrainBroadcastMessages(
            int endpointCount, int msgNum, IMockService mockSvc, 
            ConcurrentDictionary<MockServiceSideConnection, int> counts = null)
        {
            counts ??= new ConcurrentDictionary<MockServiceSideConnection, int>();

            // Each endpoint will get the broadcast message so we should receive endpointCount * MsgNum messages
            for (int ep = 0; ep < endpointCount * msgNum; ep++)
            {
                // we go "peek then take" route because we don't know which secondary connection will receive the messages
                var connWithMessage = await Task.WhenAny(mockSvc.ServiceSideConnections.Select(async c =>
                {
                    bool moreData = await c.WaitToDequeueMessageAsync<BroadcastDataMessage>();
                    replacedert.True(moreData);
                    return (c, moreData);
                }));

                var conn = connWithMessage.Result.c;
                var newMsg = await conn.DequeueMessageAsync<BroadcastDataMessage>();

                int msgCount = counts.GetOrAdd(conn, 0);
                counts[conn] = ++msgCount;

                // parse each BroadcastDataMessage and verify this is the correct message
                var hubMessage = ParseBroadcastDataMessageJson(newMsg, mockSvc.CurrentInvocationBinder);
                replacedert.True(hubMessage is InvocationMessage);
                var invMsg = hubMessage as InvocationMessage;
                replacedert.Equal("Callback", invMsg.Target);

                // finally, get ready to verify the order of messages
                int actualCallbackNum = (int)invMsg.Arguments[0];

                // this check works for both primary and secondary connections
                replacedert.Equal(msgCount, actualCallbackNum);
            }
            // todo: verify we received no extra BroadcastDataMessage - need TryPeek method (async with timeout?)
            return counts;
        }

[Fact]
        // verifies that when original outboud service connection replacedociated with the hub call is closed:
        // - a new service connection for each endpoint is selected 
        // - all subsequent messages use newly selected service connections
        // Note 1: this test is for a scenario which already has much weaker message order guarantees
        // because switching from one connection to another may lead to out of order messages.
        // However there are valid scenarios when the outboud messages are only sent after the original connection is closed.
        // In this case all the messages will be sent over a new connection and the order will be preserved.
        // Note 2: there are multiple ways to close service connection. In this test we ask the mock service to close the pipe.
        // There are several other cases when SDK initiates disconnect which we didn't test here.
        // In either case we need to wait for the connection to be "fully closed" before sending messages.
        public async Task OutgoingMessagesSwitchOverToNewServiceConnection()
        {
            // step 0: create initial connections
            var builder = WebHost.CreateDefaultBuilder()
                .ConfigureServices((IServiceCollection services) => { })
                .ConfigureLogging(logging => logging.AddXunit(_output))
                .UseStartup<IntegrationTestStartup<MockServiceMessageOrderTestParams, SwitchOverToNewServiceConnectionHub>>();

            using var server = new AspNetTestServer(builder);
            var mockSvc = (server.Host.Services.GetRequiredService<ServiceHubDispatcher<SwitchOverToNewServiceConnectionHub>>() 
                as MockServiceHubDispatcher<SwitchOverToNewServiceConnectionHub>).MockService;
            mockSvc.CurrentInvocationBinder = new TestHubBroadcastNCallsInvocationBinder();
            await mockSvc.AllConnectionsEstablished();
            var allSvcConns = mockSvc.ServiceSideConnections;
            var primarySvc0 = allSvcConns.Where(i => i.Endpoint.EndpointType == EndpointType.Primary).FirstOrDefault();
            var client0 = await primarySvc0.ConnectClientAsync();

            // step 1: broadcast a message to figure out secondary connection selections
            // we already know the primary connection (primarySvc0) but not the secondary one(s)
            // which will only be selected at the first outgoing message to the service
            await client0.SendMessage("BroadcastNumCalls", new object[] { 1 });
            var epCount = MockServiceMessageOrderTestParams.ServiceEndpoints.Length;
            var connSelections = new ConcurrentBag<MockServiceSideConnection>();

            for (int ep = 0; ep < epCount; ep++)
            {
                var connReceivedMessage = await Task.WhenAny(allSvcConns.Select(async c =>
                {
                    await c.WaitToDequeueMessageAsync<BroadcastDataMessage>();
                    return c;
                }));
                connSelections.Add(connReceivedMessage.Result);
                await connReceivedMessage.Result.DequeueMessageAsync<BroadcastDataMessage>();
            }

            // sanity checks
            replacedert.Equal(primarySvc0, connSelections.Where(c => c.Endpoint.EndpointType == EndpointType.Primary).FirstOrDefault());
            var secondaryEpCount = MockServiceMessageOrderTestParams.ServiceEndpoints.Where(ep => ep.EndpointType == EndpointType.Secondary).Count();
            var secondaryReceivedCount = connSelections.Where(c => c.Endpoint.EndpointType == EndpointType.Secondary).Count();
            replacedert.Equal(secondaryEpCount, secondaryReceivedCount);

            // step 2: call hub and drop all the connections replacedociated with the current client
            const int MsgNum = 10;
            await client0.SendMessage("BroadcastNumCallsAfterDisconnected", new object[] { MsgNum });
            foreach (var secConnUsed in connSelections.Where(c => c.Endpoint.EndpointType == EndpointType.Secondary))
            {
                await secConnUsed.StopAsync();
            }
            await primarySvc0.StopAsync();

            // step 3: drain and count messages sent as the result of the call to BroadcastNumCallsAfterDisconnected
            await mockSvc.AllConnectionsEstablished(); 
            var counts = await DrainBroadcastMessages(epCount, MsgNum, mockSvc);

            // step 4: verify the connections that received messages
            var primary = counts.Where(c => c.Key.Endpoint.EndpointType == EndpointType.Primary);
            replacedert.Single(primary);
            // the primary is NOT the one we used to send client message
            replacedert.NotEqual(primary.FirstOrDefault().Key, primarySvc0);
            // and it received MsgNum messages
            replacedert.Equal(MsgNum, primary.FirstOrDefault().Value);

            // for every secondary verify that
            // - their number equals to the number of seconary endpoints
            // - each received MsgNum messages
            // - each of the secondary ones is not the same as the original selection
            var secondary = counts.Where(c => c.Key.Endpoint.EndpointType == EndpointType.Secondary);
            var secondaryEndpoints = MockServiceMessageOrderTestParams.ServiceEndpoints.Where(ep => ep.EndpointType == EndpointType.Secondary);
            replacedert.Equal(secondaryEndpoints.Count(), secondary.Count());
            foreach (var newSecCon in secondary)
            {
                // none of the new secondary connections are the same as the ones initially used
                foreach (var oldSecCon in connSelections.Where(c => c.Endpoint.EndpointType == EndpointType.Secondary))
                {
                    replacedert.NotEqual(newSecCon.Key, oldSecCon);
                }
                // each of the new secondary connections received MsgNum messages
                replacedert.Equal(MsgNum, newSecCon.Value);
            }
        }

See More Examples