System.Threading.Tasks.Task.Delay(System.TimeSpan, System.Threading.CancellationToken)

public static async Task<TResult> TimeoutAfter<TResult>(this Task<TResult> task, TimeSpan timeout, string message = "The operation has timed out.")
        {
            using (var timeoutCancellationTokenSource = new CancellationTokenSource())
            {
                var completedTask = await Task.WhenAny(task, Task.Delay(timeout, timeoutCancellationTokenSource.Token));
                if (completedTask == task)
                {
                    timeoutCancellationTokenSource.Cancel();
                    return await task;
                }
                else
                {
                    throw new TimeoutException(message);
                }
            }
        }

public static async Task TimeoutAfter(this Task task, TimeSpan timeout, string message = "The operation has timed out.")
        {
            using (var timeoutCancellationTokenSource = new CancellationTokenSource())
            {
                var completedTask = await Task.WhenAny(task, Task.Delay(timeout, timeoutCancellationTokenSource.Token));
                if (completedTask == task)
                {
                    timeoutCancellationTokenSource.Cancel();
                    await task;
                }
                else
                {
                    throw new TimeoutException(message);
                }
            }
        }

private async Task RunInternalAsync(CancellationToken token)
        {
            while (!token.IsCancellationRequested)
            {
                bool requiredRetry = false;
                try
                {
                    var tasks = _tasks.Values.ToArray();
                    foreach (var task in tasks)
                    {
                        await task.LoadFromLocalAsync();
                    }

                    var upgradeInfos = await _upgradeInfosGetter();

                    foreach (var task in tasks)
                    {
                        var info = upgradeInfos.FirstOrDefault(
                            item => item.Name.Equals(task.Name, StringComparison.InvariantCultureIgnoreCase));

                        if (info == null) continue;

                        await task.LoadFromRemoteAsync(info);
                    }
                }
                catch (Exception)
                {
                    requiredRetry = true;
                }

                try
                {
                    await Task.Delay(TimeSpan.FromMinutes(requiredRetry
                            ? RetryIntervalBaseMinute
                            : UpgradeIntervalBaseMinute),
                        token);
                }
                catch (TaskCanceledException)
                {
                    // Ignore
                }
            }
        }

public async Task Delay(TimeSpan delay, CancellationToken cancellationToken)
        {
            await Task.Delay(delay, cancellationToken);
        }

public async Task DownloadFromContainerAsync(
            RunnerActionPluginExecutionContext context,
            String destination,
            CancellationToken cancellationToken)
        {
            // Find out all container items need to be processed
            List<FileContainerItem> containerItems = new List<FileContainerItem>();
            int retryCount = 0;
            while (retryCount < 3)
            {
                try
                {
                    containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId,
                                                                                             _projectId,
                                                                                             _containerPath,
                                                                                             cancellationToken: cancellationToken);
                    break;
                }
                catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
                {
                    context.Debug($"Container query has been cancelled.");
                    throw;
                }
                catch (Exception ex) when (retryCount < 2)
                {
                    retryCount++;
                    context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}");
                    context.Debug(ex.ToString());
                }

                var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
                context.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
                await Task.Delay(backOff);
            }

            if (containerItems.Count == 0)
            {
                context.Output($"There is nothing under #/{_containerId}/{_containerPath}");
                return;
            }

            // container items will include both folders, files and even file with zero size
            // Create all required empty folders and emptry files, gather a list of files that we need to download from server.
            int foldersCreated = 0;
            int emptryFilesCreated = 0;
            List<DownloadInfo> downloadFiles = new List<DownloadInfo>();
            foreach (var item in containerItems.OrderBy(x => x.Path))
            {
                if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase))
                {
                    throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}");
                }

                var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/');
                var localPath = Path.Combine(destination, localRelativePath);

                if (item.ItemType == ContainerItemType.Folder)
                {
                    context.Debug($"Ensure folder exists: {localPath}");
                    Directory.CreateDirectory(localPath);
                    foldersCreated++;
                }
                else if (item.ItemType == ContainerItemType.File)
                {
                    if (item.FileLength == 0)
                    {
                        context.Debug($"Create empty file at: {localPath}");
                        var parentDirectory = Path.GetDirectoryName(localPath);
                        Directory.CreateDirectory(parentDirectory);
                        IOUtil.DeleteFile(localPath);
                        using (new FileStream(localPath, FileMode.Create))
                        {
                        }
                        emptryFilesCreated++;
                    }
                    else
                    {
                        context.Debug($"Prepare download {item.Path} to {localPath}");
                        downloadFiles.Add(new DownloadInfo(item.Path, localPath));
                    }
                }
                else
                {
                    throw new NotSupportedException(item.ItemType.ToString());
                }
            }

            if (foldersCreated > 0)
            {
                context.Output($"{foldersCreated} folders created.");
            }

            if (emptryFilesCreated > 0)
            {
                context.Output($"{emptryFilesCreated} empty files created.");
            }

            if (downloadFiles.Count == 0)
            {
                context.Output($"There is nothing to download");
                return;
            }

            // Start multi-task to download all files.
            using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                // try download all files for the first time.
                DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
                if (downloadResult.FailedFiles.Count == 0)
                {
                    // all files have been download succeed.
                    context.Output($"{downloadFiles.Count} files download succeed.");
                    return;
                }
                else
                {
                    context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute.");
                }

                // Delay 1 min then retry failed files.
                for (int timer = 60; timer > 0; timer -= 5)
                {
                    context.Output($"Retry file download after {timer} seconds.");
                    await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
                }

                // Retry download all failed files.
                context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload.");
                DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
                if (retryDownloadResult.FailedFiles.Count == 0)
                {
                    // all files have been download succeed after retry.
                    context.Output($"{downloadResult.FailedFiles} files download succeed after retry.");
                    return;
                }
                else
                {
                    throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry.");
                }
            }
        }

public async Task<long> CopyToContainerAsync(
            RunnerActionPluginExecutionContext context,
            String source,
            CancellationToken cancellationToken)
        {
            //set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
            int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
            //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");

            List<String> files;
            if (File.Exists(source))
            {
                files = new List<String>() { source };
                _sourceParentDirectory = Path.GetDirectoryName(source);
            }
            else
            {
                files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList();
                _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
            }

            context.Output($"Uploading {files.Count()} files");
            using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                // hook up reporting event from file container client.
                _fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived;
                _fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived;

                try
                {
                    // try upload all files for the first time.
                    UploadResult uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);

                    if (uploadResult.RetryFiles.Count == 0)
                    {
                        // all files have been upload succeed.
                        context.Output("File upload complete.");
                        return uploadResult.TotalFileSizeUploaded;
                    }
                    else
                    {
                        context.Output($"{uploadResult.RetryFiles.Count} files failed to upload, retry these files after a minute.");
                    }

                    // Delay 1 min then retry failed files.
                    for (int timer = 60; timer > 0; timer -= 5)
                    {
                        context.Output($"Retry file upload after {timer} seconds.");
                        await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
                    }

                    // Retry upload all failed files.
                    context.Output($"Start retry {uploadResult.RetryFiles.Count} failed files upload.");
                    UploadResult retryUploadResult = await ParallelUploadAsync(context, uploadResult.RetryFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);

                    if (retryUploadResult.RetryFiles.Count == 0)
                    {
                        // all files have been upload succeed after retry.
                        context.Output("File upload complete after retry.");
                        return uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded;
                    }
                    else
                    {
                        throw new Exception("File upload failed even after retry.");
                    }
                }
                finally
                {
                    _fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived;
                    _fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived;
                }
            }
        }

protected override async Task<HttpResponseMessage> SendAsync(
            HttpRequestMessage request,
            CancellationToken cancellationToken)
        {
            Int32 attempt = 1;
            HttpResponseMessage response = null;
            HttpRequestException exception = null;
            VssTraceActivity traceActivity = VssTraceActivity.Current;

            // Allow overriding default retry options per request
            VssHttpRetryOptions retryOptions = m_retryOptions;
            object retryOptionsObject;
            if (request.Properties.TryGetValue(HttpRetryOptionsKey, out retryOptionsObject)) // NETSTANDARD compliant, TryGetValue<T> is not
            {
                // Fallback to default options if object of unexpected type was preplaceded
                retryOptions = retryOptionsObject as VssHttpRetryOptions ?? m_retryOptions;
            }

            TimeSpan minBackoff = retryOptions.MinBackoff;
            Int32 maxAttempts = retryOptions.MaxRetries + 1;

            IVssHttpRetryInfo retryInfo = null;
            object retryInfoObject;
            if (request.Properties.TryGetValue(HttpRetryInfoKey, out retryInfoObject)) // NETSTANDARD compliant, TryGetValue<T> is not
            {
                retryInfo = retryInfoObject as IVssHttpRetryInfo;
            }

            if (IsLowPriority(request))
            {
                // Increase the backoff and retry count, low priority requests can be retried many times if the server is busy.
                minBackoff = TimeSpan.FromSeconds(minBackoff.TotalSeconds * 2);
                maxAttempts = maxAttempts * 10;
            }

            TimeSpan backoff = minBackoff;

            while (attempt <= maxAttempts)
            {
                // Reset the exception so we don't have a lingering variable
                exception = null;

                Boolean canRetry = false;
                SocketError? socketError = null;
                HttpStatusCode? statusCode = null;
                WebExceptionStatus? webExceptionStatus = null;
                WinHttpErrorCode? winHttpErrorCode = null;
                CurlErrorCode? curlErrorCode = null;
                string afdRefInfo = null;
                try
                {
                    if (attempt == 1)
                    {
                        retryInfo?.InitialAttempt(request);
                    }

                    response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);

                    if (attempt > 1)
                    {
                        TraceHttpRequestSucceededWithRetry(traceActivity, response, attempt);
                    }

                    // Verify the response is successful or the status code is one that may be retried. 
                    if (response.IsSuccessStatusCode)
                    {
                        break;
                    }
                    else
                    {
                        statusCode = response.StatusCode;
                        afdRefInfo = response.Headers.TryGetValues(HttpHeaders.AfdResponseRef, out var headers) ? headers.First() : null;
                        canRetry = m_retryOptions.IsRetryableResponse(response);
                    }
                }
                catch (HttpRequestException ex)
                {
                    exception = ex;
                    canRetry = VssNetworkHelper.IsTransientNetworkException(exception, m_retryOptions, out statusCode, out webExceptionStatus, out socketError, out winHttpErrorCode, out curlErrorCode);
                }
                catch (TimeoutException)
                {
                    throw;
                }

                if (attempt < maxAttempts && canRetry)
                {
                    backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, minBackoff, m_retryOptions.MaxBackoff, m_retryOptions.BackoffCoefficient);
                    retryInfo?.Retry(backoff);
                    TraceHttpRequestRetrying(traceActivity, request, attempt, backoff, statusCode, webExceptionStatus, socketError, winHttpErrorCode, curlErrorCode, afdRefInfo);
                }
                else
                {
                    if (attempt < maxAttempts)
                    {
                        if (exception == null)
                        {
                            TraceHttpRequestFailed(traceActivity, request, statusCode != null ? statusCode.Value : (HttpStatusCode)0, afdRefInfo);
                        }
                        else
                        {
                            TraceHttpRequestFailed(traceActivity, request, exception);
                        }
                    }
                    else
                    {
                        TraceHttpRequestFailedMaxAttempts(traceActivity, request, attempt, statusCode, webExceptionStatus, socketError, winHttpErrorCode, curlErrorCode, afdRefInfo);
                    }
                    break;
                }

                // Make sure to dispose of this so we don't keep the connection open
                if (response != null)
                {
                    response.Dispose();
                }

                attempt++;
                TraceRaw(request, 100011, TraceLevel.Error,
                    "{{ \"Client\":\"{0}\", \"Endpoint\":\"{1}\", \"Attempt\":{2}, \"MaxAttempts\":{3}, \"Backoff\":{4} }}",
                    m_clientName,
                    request.RequestUri.Host,
                    attempt,
                    maxAttempts,
                    backoff.TotalMilliseconds);
                await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
            }

            if (exception != null)
            {
                throw exception;
            }

            return response;
        }

[EditorBrowsable(EditorBrowsableState.Never)]
        public async Task<HttpResponseMessage> UploadFileAsync(
            Int64 containerId,
            String itemPath,
            Stream fileStream,
            byte[] contentId,
            Int64 fileLength,
            Boolean isGzipped,
            Guid scopeIdentifier,
            CancellationToken cancellationToken = default(CancellationToken),
            int chunkSize = c_defaultChunkSize,
            int chunkRetryTimes = c_defaultChunkRetryTimes,
            bool uploadFirstChunk = false,
            Object userState = null)
        {
            if (containerId < 1)
            {
                throw new ArgumentException(WebApiResources.ContainerIdMustBeGreaterThanZero(), "containerId");
            }

            if (chunkSize > c_maxChunkSize)
            {
                chunkSize = c_maxChunkSize;
            }

            // if a contentId is specified but the chunk size is not a 2mb multiple error
            if (contentId != null && (chunkSize % c_ContentChunkMultiple) != 0)
            {
                throw new ArgumentException(FileContainerResources.ChunksizeWrongWithContentId(c_ContentChunkMultiple), "chunkSize");
            }

            ArgumentUtility.CheckForNull(fileStream, "fileStream");

            ApiResourceVersion gzipSupportedVersion = new ApiResourceVersion(new Version(1, 0), 2);
            ApiResourceVersion requestVersion = await NegotiateRequestVersionAsync(FileContainerResourceIds.FileContainer, s_currentApiVersion, userState, cancellationToken).ConfigureAwait(false);

            if (isGzipped
                && (requestVersion.ApiVersion < gzipSupportedVersion.ApiVersion
                    || (requestVersion.ApiVersion == gzipSupportedVersion.ApiVersion && requestVersion.ResourceVersion < gzipSupportedVersion.ResourceVersion)))
            {
                throw new ArgumentException(FileContainerResources.GzipNotSupportedOnServer(), "isGzipped");
            }

            if (isGzipped && fileStream.Length >= fileLength)
            {
                throw new ArgumentException(FileContainerResources.BadCompression(), "fileLength");
            }

            HttpRequestMessage requestMessage = null;
            List<KeyValuePair<String, String>> query = AppendItemQueryString(itemPath, scopeIdentifier);

            if (fileStream.Length == 0)
            {
                // zero byte upload
                FileUploadTrace(itemPath, $"Upload zero byte file '{itemPath}'.");
                requestMessage = await CreateRequestMessageAsync(HttpMethod.Put, FileContainerResourceIds.FileContainer, routeValues: new { containerId = containerId }, version: s_currentApiVersion, queryParameters: query, userState: userState, cancellationToken: cancellationToken).ConfigureAwait(false);
                return await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false);
            }

            bool multiChunk = false;
            int totalChunks = 1;
            if (fileStream.Length > chunkSize)
            {
                totalChunks = (int)Math.Ceiling(fileStream.Length / (double)chunkSize);
                FileUploadTrace(itemPath, $"Begin chunking upload file '{itemPath}', chunk size '{chunkSize} Bytes', total chunks '{totalChunks}'.");
                multiChunk = true;
            }
            else
            {
                FileUploadTrace(itemPath, $"File '{itemPath}' will be uploaded in one chunk.");
                chunkSize = (int)fileStream.Length;
            }

            StreamParser streamParser = new StreamParser(fileStream, chunkSize);
            SubStream currentStream = streamParser.GetNextStream();
            HttpResponseMessage response = null;

            Byte[] dataToSend = new Byte[chunkSize];
            int currentChunk = 0;
            Stopwatch uploadTimer = new Stopwatch();
            while (currentStream.Length > 0 && !cancellationToken.IsCancellationRequested)
            {
                currentChunk++;

                for (int attempt = 1; attempt <= chunkRetryTimes && !cancellationToken.IsCancellationRequested; attempt++)
                {
                    if (attempt > 1)
                    {
                        TimeSpan backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10));
                        FileUploadTrace(itemPath, $"Backoff {backoff.TotalSeconds} seconds before attempt '{attempt}' chunk '{currentChunk}' of file '{itemPath}'.");
                        await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
                        currentStream.Seek(0, SeekOrigin.Begin);
                    }

                    FileUploadTrace(itemPath, $"Attempt '{attempt}' for uploading chunk '{currentChunk}' of file '{itemPath}'.");

                    // inorder for the upload to be retryable, we need the content to be re-readable
                    // to ensure this we copy the chunk into a byte array and send that
                    // chunk size ensures we can convert the length to an int
                    int bytesToCopy = (int)currentStream.Length;
                    using (MemoryStream ms = new MemoryStream(dataToSend))
                    {
                        await currentStream.CopyToAsync(ms, bytesToCopy, cancellationToken).ConfigureAwait(false);
                    }

                    // set the content and the Content-Range header
                    HttpContent byteArrayContent = new ByteArrayContent(dataToSend, 0, bytesToCopy);
                    byteArrayContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
                    byteArrayContent.Headers.ContentLength = currentStream.Length;
                    byteArrayContent.Headers.ContentRange = new System.Net.Http.Headers.ContentRangeHeaderValue(currentStream.StartingPostionOnOuterStream,
                                                                                                             currentStream.EndingPostionOnOuterStream,
                                                                                                             streamParser.Length);
                    FileUploadTrace(itemPath, $"Generate new HttpRequest for uploading file '{itemPath}', chunk '{currentChunk}' of '{totalChunks}'.");

                    try
                    {
                        if (requestMessage != null)
                        {
                            requestMessage.Dispose();
                            requestMessage = null;
                        }

                        requestMessage = await CreateRequestMessageAsync(
                            HttpMethod.Put,
                            FileContainerResourceIds.FileContainer,
                            routeValues: new { containerId = containerId },
                            version: s_currentApiVersion,
                            content: byteArrayContent,
                            queryParameters: query,
                            userState: userState,
                            cancellationToken: cancellationToken).ConfigureAwait(false);
                    }
                    catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
                    {
                        // stop re-try on cancellation.
                        throw;
                    }
                    catch (Exception ex) when (attempt < chunkRetryTimes) // not the last attempt
                    {
                        FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to create HttpRequest. Error: {ex.ToString()}.");
                        continue;
                    }

                    if (isGzipped)
                    {
                        //add gzip header info
                        byteArrayContent.Headers.ContentEncoding.Add("gzip");
                        byteArrayContent.Headers.Add("x-tfs-filelength", fileLength.ToString(System.Globalization.CultureInfo.InvariantCulture));
                    }

                    if (contentId != null)
                    {
                        byteArrayContent.Headers.Add("x-vso-contentId", Convert.ToBase64String(contentId)); // Base64FormattingOptions.None is default when not supplied
                    }

                    FileUploadTrace(itemPath, $"Start uploading file '{itemPath}' to server, chunk '{currentChunk}'.");
                    uploadTimer.Restart();

                    try
                    {
                        if (response != null)
                        {
                            response.Dispose();
                            response = null;
                        }

                        response = await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false);
                    }
                    catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
                    {
                        // stop re-try on cancellation.
                        throw;
                    }
                    catch (Exception ex) when (attempt < chunkRetryTimes) // not the last attempt
                    {
                        FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to send request to server. Error: {ex.ToString()}.");
                        continue;
                    }

                    uploadTimer.Stop();
                    FileUploadTrace(itemPath, $"Finished upload chunk '{currentChunk}' of file '{itemPath}', elapsed {uploadTimer.ElapsedMilliseconds} (ms), response code '{response.StatusCode}'.");

                    if (multiChunk)
                    {
                        FileUploadProgress(itemPath, currentChunk, (int)Math.Ceiling(fileStream.Length / (double)chunkSize));
                    }

                    if (response.IsSuccessStatusCode)
                    {
                        break;
                    }
                    else if (IsFastFailResponse(response))
                    {
                        FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request and cannot continue.");
                        break;
                    }
                    else
                    {
                        FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request.");
                        continue;
                    }
                }

                // if we don't have success then bail and return the failed response
                if (!response.IsSuccessStatusCode)
                {
                    break;
                }

                if (contentId != null && response.StatusCode == HttpStatusCode.Created)
                {
                    // no need to keep uploading since the server said it has all the content
                    FileUploadTrace(itemPath, $"Stop chunking upload the rest of the file '{itemPath}', since server already has all the content.");
                    break;
                }

                currentStream = streamParser.GetNextStream();
                if (uploadFirstChunk)
                {
                    break;
                }
            }

            cancellationToken.ThrowIfCancellationRequested();

            return response;
        }

private async Task BackgroundJob(CancellationToken cancellationToken)
        {
            using IServiceScope scope = _serviceProvider.CreateScope();

            var jobProcessor = scope.ServiceProvider
                                    .GetRequiredService<IJobDispatcher>();

            _logger.LogInformation("{OperationName} is triggered : {OperationTime}", nameof(IJobDispatcher.HandleNextJobAsync), DateTime.UtcNow);
            var jobHandled = false;
            try
            {
                jobHandled = await jobProcessor.HandleNextJobAsync(cancellationToken);
                _logger.LogInformation("{OperationName} is completed : {OperationTime}", nameof(IJobDispatcher.HandleNextJobAsync), DateTime.UtcNow);
            }
            catch (Exception e)
            {
                _logger.LogError(e, "{OperationName} is failed : {OperationTime}", nameof(IJobDispatcher.HandleNextJobAsync), DateTime.UtcNow);
            }

            await (jobHandled
                       ? Task.Delay(_waitAfterJobHandled, cancellationToken)
                       : Task.Delay(_waitAfterJobNotHandled, cancellationToken));
        }

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                await BackgroundJob(stoppingToken);
            }
            catch (Exception e)
            {
                _logger.LogError(e, "{OperationName} faced unexpected error", $"{nameof(JobRescuerHostedService)}.{nameof(BackgroundJob)}");
            }

            await Task.Delay(_executionPeriod, stoppingToken);
        }

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            try
            {
                await BackgroundJob(stoppingToken);
            }
            catch (Exception e)
            {
                _logger.LogError(e, "{OperationName} faced unexpected error", $"{nameof(JobRetrierHostedService)}.{nameof(BackgroundJob)}");
            }

            await Task.Delay(_executionPeriod, stoppingToken);
        }

public override async Task HandleAsync(LongProcessRequest payload, CancellationToken cancellationToken = default)
            {
                await Task.Delay(payload.TimeConsumption, cancellationToken);
            }

private async Task CleanupAsync(CancellationToken cancellationToken)
        {
            while (true)
            {
                try
                {
                    await Task.Delay(_interval, cancellationToken).ConfigureAwait(false);
                }
                catch (TaskCanceledException)
                {
                    _logger.LogDebug("TaskCanceledException. Exiting.");
                    break;
                }
                catch (Exception e)
                {
                    _logger.LogError(e, $"Task.Delay exception: {e.Message}. Exiting.");
                    break;
                }

                await RemoveExpiredTokensAsync(cancellationToken).ConfigureAwait(false);
            }
        }

private async Task CleanupBucketsAsync()
        {
            while (!this._bucketCleanerTokenSource.IsCancellationRequested)
            {
                try
                {
                    await Task.Delay(this._bucketCleanupDelay, this._bucketCleanerTokenSource.Token).ConfigureAwait(false);
                }
                catch { }

                if (this._disposed)
                    return;

                //Check and clean request queue first in case it wasn't removed properly during requests.
                foreach (var key in this.RequestQueue.Keys)
                {
                    var bucket = this.HashesToBuckets.Values.FirstOrDefault(x => x.RouteHashes.Contains(key));

                    if (bucket == null || (bucket != null && bucket.LastAttemptAt.AddSeconds(5) < DateTimeOffset.UtcNow))
                        _ = this.RequestQueue.TryRemove(key, out _);
                }

                var removedBuckets = 0;
                StringBuilder bucketIdStrBuilder = default;

                foreach (var kvp in this.HashesToBuckets)
                {
                    if (bucketIdStrBuilder == null)
                        bucketIdStrBuilder = new StringBuilder();

                    var key = kvp.Key;
                    var value = kvp.Value;

                    // Don't remove the bucket if it's currently being handled by the rest client, unless it's an unlimited bucket.
                    if (this.RequestQueue.ContainsKey(value.BucketId) && !value._isUnlimited)
                        continue;

                    var resetOffset = this.UseResetAfter ? value.ResetAfterOffset : value.Reset;

                    // Don't remove the bucket if it's reset date is less than now + the additional wait time, unless it's an unlimited bucket.
                    if (resetOffset != null && !value._isUnlimited && (resetOffset > DateTimeOffset.UtcNow || DateTimeOffset.UtcNow - resetOffset < this._bucketCleanupDelay))
                        continue;

                    _ = this.HashesToBuckets.TryRemove(key, out _);
                    removedBuckets++;
                    bucketIdStrBuilder.Append(value.BucketId + ", ");
                }

                if (removedBuckets > 0)
                    this.Logger.LogDebug(LoggerEvents.RestCleaner, "Removed {0} unused bucket{1}: [{2}]", removedBuckets, removedBuckets > 1 ? "s" : string.Empty, bucketIdStrBuilder.ToString().TrimEnd(',', ' '));

                if (this.HashesToBuckets.Count == 0)
                    break;
            }

            if (!this._bucketCleanerTokenSource.IsCancellationRequested)
                this._bucketCleanerTokenSource.Cancel();

            this._cleanerRunning = false;
            this.Logger.LogDebug(LoggerEvents.RestCleaner, "Bucket cleaner task stopped.");
        }

public async Task LockAsync()
        {
            await this.LockSemapreplaced.WaitAsync().ConfigureAwait(false);

            this.TimeoutCancelSource = new CancellationTokenSource();
            this.UnlockTask = Task.Delay(TimeSpan.FromSeconds(30), this.TimeoutCancel);
            _ = this.UnlockTask.ContinueWith(this.InternalUnlock, TaskContinuationOptions.NotOnCanceled);
        }

public void UnlockAfter(TimeSpan unlockDelay)
        {
            if (this.TimeoutCancelSource == null || this.LockSemapreplaced.CurrentCount > 0)
                return; // it's not unlockable because it's post-IDENTIFY or not locked

            try
            {
                this.TimeoutCancelSource.Cancel();
                this.TimeoutCancelSource.Dispose();
            }
            catch { }
            this.TimeoutCancelSource = null;

            this.UnlockTask = Task.Delay(unlockDelay, CancellationToken.None);
            _ = this.UnlockTask.ContinueWith(this.InternalUnlock);
        }

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            while (!stoppingToken.IsCancellationRequested)
            {
                await Task.Delay(_timeSpan, stoppingToken);
                await _backgroundJob.DoWork();
            }
        }

private void HandleMain(Task<bool> t)
            {
                if (t.IsCanceled)
                {
                    _error = new OperationCanceledException();
                    _done = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
                else if (t.IsFaulted)
                {
                    CancellationHelper.Cancel(ref _cts);
                    if (_emitLast)
                    {
                        var idx = _sourceIndex;
                        if (idx != 0)
                        {
                            SetLatest(_emitLasreplacedem, idx + 1);
                            _emitLasreplacedem = default;
                        }
                    }
                    _error = ExceptionHelper.Extract(t.Exception);
                    _done = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
                else if (t.Result)
                {
                    Volatile.Read(ref _cts)?.Cancel();

                    var v = _source.Current;
                    if (TryDispose())
                    {
                        if (_emitLast)
                        {
                            _emitLasreplacedem = v;
                        }
                        var idx = ++_sourceIndex;
                        var newCts = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);
                        if (CancellationHelper.Replace(ref _cts, newCts))
                        {
                            Task.Delay(_delay, newCts.Token)
                                .ContinueWith(tt => TimerHandler(tt, v, idx), newCts.Token);
                            MoveNext();
                        }
                    }
                }
                else
                {
                    CancellationHelper.Cancel(ref _cts);
                    if (_emitLast)
                    {
                        var idx = _sourceIndex;
                        if (idx != 0)
                        {
                            SetLatest(_emitLasreplacedem, idx + 1);
                            _emitLasreplacedem = default;
                        }
                    }
                    _done = true;
                    if (TryDispose())
                    {
                        ResumeHelper.Resume(ref _resume);
                    }
                }
            }

internal void StartFirst(TimeSpan initialDelay)
            {
                Task.Delay(initialDelay, _cts.Token)
                    .ContinueWith(NextAction, this);
            }

private void Next(Task t)
            {
                if (t.IsCanceled || _cts.IsCancellationRequested)
                {
                    ResumeHelper.Resume(ref _resume);
                    return;
                }
                var value = Interlocked.Increment(ref _available);
                ResumeHelper.Resume(ref _resume);

                if (value != _end)
                {
                    // FIXME compensate for drifts
                    Task.Delay(_period, _cts.Token)
                        .ContinueWith(NextAction, this);
                }
            }

internal void StartTimer()
            {
                Task.Delay(_period, _sourceCTS.Token)
                    .ContinueWith(HandleTimerAction, this, _sourceCTS.Token);
            }

public ValueTask<bool> MoveNextAsync()
            {
                var idx = Volatile.Read(ref _index);

                var result = new TaskCompletionSource<bool>();

                _token = CancellationTokenSource.CreateLinkedTokenSource(_sourceCTS.Token);

                Interlocked.Increment(ref _disposeWip);

                Task.Delay(_timeout, _token.Token)
                    .ContinueWith(t => Timeout(idx, result), _token.Token);

                _source.MoveNextAsync()
                    .AsTask().ContinueWith(t => Next(idx, t, result));

                return new ValueTask<bool>(result.Task);
            }

public async ValueTask<bool> MoveNextAsync()
            {
                if (_once)
                {
                    return false;
                }
                _once = true;

                await Task.Delay(_delay, _ct).ConfigureAwait(false);
                return true;
            }

public IDisposable Schedule(Action task, TimeSpan delay)
            {
                var dt = new DisposableTask(task, this);
                if (tasks.Add(dt))
                {
                    Task.Delay(delay, dt.cts.Token).ContinueWith(a => dt.Run(), dt.cts.Token);
                    return dt;
                }
                return EmptyDisposable.Instance;
            }

public IDisposable Schedule(Action task, TimeSpan initialDelay, TimeSpan period)
            {
                var dt = new DisposablePeriodicTask(task, this, (long)(Now + initialDelay.TotalMilliseconds), (long)period.TotalMilliseconds);
                if (tasks.Add(dt))
                {
                    Task.Delay(initialDelay, dt.cts.Token).ContinueWith(a => dt.Run(), dt.cts.Token);
                    return dt;
                }
                return EmptyDisposable.Instance;
            }

internal void Run()
                {
                    try
                    {
                        task();

                        long next = Math.Max(0L, start + (++count) * period - parent.Now);
                        Task.Delay(TimeSpan.FromMilliseconds(next), cts.Token).ContinueWith(a => Run(), cts.Token);
                    }
                    catch
                    {
                        Dispose();
                        throw;
                    }
                }

public IDisposable Schedule(Action task, TimeSpan delay)
            {
                if (Volatile.Read(ref cancelled) != 0)
                {
                    return EmptyDisposable.Instance;
                }
                var cts = new CancellationTokenSource();

                var t = new InterruptibleAction(task);
                cts.Token.Register(t.Dispose);

                Task.Delay(delay, cts.Token).ContinueWith(a =>
                {
                    queue.Offer(t);
                    Drain();
                }, cts.Token);

                return cts;
            }

internal static IDisposable ScheduleTask(Action task, TimeSpan delay)
        {
            var cts = new CancellationTokenSource();
            if (delay == TimeSpan.Zero)
            {
                Task.Run(task, cts.Token);
            }
            else
            {
                Task.Delay(delay, cts.Token).ContinueWith(a => task(), cts.Token);
            }
            return cts;
        }

internal static void ScheduleTask(Action task, TimeSpan initialDelay, TimeSpan period, CancellationTokenSource cts)
        {
            Action<Task> recursive = null;
            long now = NowUTC() + (long)initialDelay.TotalMilliseconds;
            long[] round = { 0 };
            recursive = t =>
            {
                task();
                long next = (long)(now + (++round[0]) * period.TotalMilliseconds - NowUTC());
                Task.Delay(TimeSpan.FromMilliseconds(Math.Max(0L, next)), cts.Token).ContinueWith(recursive, cts.Token);
            };

            Task.Delay(initialDelay, cts.Token)
                .ContinueWith(recursive, cts.Token);

        }

private static async Task GuardWithTimeoutAsync(Task asyncTask, TimeSpan timeout)
        {
            var cts = new CancellationTokenSource();
            try
            {
                var timeoutTask = Task.Delay(timeout, cts.Token);
                var completedTask = await Task.WhenAny(asyncTask, timeoutTask);
                if (completedTask == timeoutTask)
                    throw new TimeoutException($"Task exceeds timeout duration {timeout}");
                else
                    cts.Cancel();
            }
            finally
            {
                cts.Dispose();
            }
        }

public async Task<EventLogPosition> ReadEventLogPositionAsync(CancellationToken cancellationToken = default)
        {
            if (_client is null)
                await ConnectAsync(cancellationToken);

            while (true)
            {
                var response = await _client.SearchAsync<EventLogItem>(sd => sd
                        .Index($"{_eventLogItemsIndex}-*")
                        .Sort(ss =>
                            ss.Descending(c => c.Id))
                        .Size(1)
                    , cancellationToken);

                if (response.IsValid)
                {
                    var item = response.Doreplacedents.FirstOrDefault();

                    if (item is null)
                        return null;
                    return new EventLogPosition(item.FileName, item.EndPosition, item.LgfEndPosition, item.Id);
                }

                if (response.OriginalException is TaskCanceledException)
                    throw response.OriginalException;

                _logger?.LogError(
                    $"Failed to get last file's position ({_eventLogItemsIndex}): {response.OriginalException.Message}");

                var currentNodeHost = _currentNode.Host;

                await ConnectAsync(cancellationToken);

                // If it's the same node then wait while MaxRetryTimeout occurs, otherwise it'll be a too often request's loop
                if (_currentNode.Host.Equals(currentNodeHost))
                    await Task.Delay(_maxRetryTimeout, cancellationToken);
            }
        }

public static async Task WithTimeout(this Task task, TimeSpan timeout)
        {
            using (var cts = new CancellationTokenSource())
            {
                var timeoutTask = Task.Delay(timeout, cts.Token);
                var completed = await Task.WhenAny(task, timeoutTask);
                if (completed == timeoutTask)
                    throw new OperationCanceledException("Operation timed out");
                else
                    cts.Cancel();
            }
        }

async Task PublishRunAsync(CancellationToken cancellationToken)
        {
            while (cancellationToken.IsCancellationRequested == false)
            {
                try
                {
                    logger.LogInformation("Initiating periodic publish of values.");
                    await Task.WhenAll(entries.Select(i => PublishValueAsync(i.Key, i.Value.Value, cancellationToken)));
                }
                catch (Exception e)
                {
                    logger.LogError(e, "Unexpected exception occurred publishing values.");
                }

                await Task.Delay(frequency, cancellationToken);
            }
        }

async Task ExpireRunAsync(CancellationToken cancellationToken)
        {
            while (cancellationToken.IsCancellationRequested == false)
            {
                try
                {
                    using (slim.BeginUpgradableReadLock())
                    {
                        // continue while the first item is expired
                        while (
                            delQueue.Count > 0 &&
                            delQueue.FindMin() is Entry entry &&
                            entry.ExpireTime <= DateTime.UtcNow &&
                            entries.TryGetValue(entry.Key, out var record))
                        {
                            using (slim.BeginWriteLock())
                            {
                                logger.LogInformation("Removing expired key {Key}.", entry.Key);

                                // remove from del queue
                                if (record.DelQueueHandle != null && delQueue.Find(record.DelQueueHandle, out _))
                                    delQueue.Delete(record.DelQueueHandle);

                                // remove from rep queue
                                if (record.RepQueueHandle != null && repQueue.Find(record.RepQueueHandle, out _))
                                    repQueue.Delete(record.RepQueueHandle);

                                // remove entry
                                entries.Remove(entry.Key);
                            }

                            OnValueChanged(new KValueEventArgs<TNodeId>(entry.Key, null));
                        }
                    }
                }
                catch (Exception e)
                {
                    logger.LogError(e, "Unexpected exception occurred republishing stored values.");
                }

                await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
            }
        }

async Task PeriodicRefreshAsync(CancellationToken cancellationToken)
        {
            try
            {
                while (cancellationToken.IsCancellationRequested == false)
                {
                    await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
                    await connector.RefreshAsync(cancellationToken);
                    await Task.Delay(TimeSpan.FromMinutes(10), cancellationToken);
                }
            }
            catch (OperationCanceledException)
            {
                // ignore
            }
            catch (Exception e)
            {
                logger.LogError(e, "Exception occurred during periodic refresh.");
            }
        }

async Task ReplicateRunAsync(CancellationToken cancellationToken)
        {
            while (cancellationToken.IsCancellationRequested == false)
            {
                try
                {
                    List<Task> l = null;

                    using (slim.BeginUpgradableReadLock())
                    {
                        // continue while the first item is expired
                        while (
                            repQueue.Count > 0 &&
                            repQueue.FindMin() is Entry entry &&
                            entry.ReplicateTime != null &&
                            entry.ReplicateTime <= DateTime.UtcNow &&
                            entries.TryGetValue(entry.Key, out var record))
                        {
                            if (l == null)
                                l = new List<Task>();

                            using (slim.BeginWriteLock())
                            {
                                // schedule replication
                                l.Add(Task.Run(() => ReplicateAsync(entry, cancellationToken)));

                                // update to next time
                                entry.ReplicateTime = DateTime.UtcNow + frequency;

                                // remove existing queue entry
                                if (record.RepQueueHandle != null)
                                    repQueue.Delete(record.RepQueueHandle);

                                // add new queue entry
                                repQueue.Add(ref record.RepQueueHandle, entry);
                            }
                        }
                    }

                    // wait for all our replicate events to finish
                    if (l != null)
                        await Task.WhenAll(l);
                }
                catch (Exception e)
                {
                    logger.LogError(e, "Unexpected exception occurred republishing stored values.");
                }

                await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
            }
        }

public async ValueTask<HttpResponseMessage> SendAsync(DynamoDbContextConfig config, HttpContent httpContent, CancellationToken cancellationToken = default)
        {
            try
            {
                int internalServerErrorRetries = 0,
                    limitExceededRetries = 0,
                    provisionedThroughputExceededRetries = 0,
                    requestLimitExceededRetries = 0,
                    serviceUnavailableRetries = 0,
                    throttlingRetries = 0;
                while (true)
                {
                    TimeSpan delay;
                    try
                    {
                        using var request = new HttpRequestMessage(HttpMethod.Post, config.RegionEndpoint.RequestUri)
                        {
                            Content = httpContent
                        };

                        try
                        {
                            var httpClient = _httpClientFactory.CreateHttpClient();
                            var stream = await httpContent.ReadreplacedtreamAsync().ConfigureAwait(false);
                            var credentials = await config.CredentialsProvider.GetCredentialsAsync(cancellationToken).ConfigureAwait(false);
                            
                            var metadata = new SigningMetadata(config.RegionEndpoint, credentials, DateTime.UtcNow, httpClient.DefaultRequestHeaders,
                                httpClient.BaseAddress);
                            AwsRequestSigner.Sign(request, (RecyclableMemoryStream) stream, in metadata);

                            var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);

                            if (!response.IsSuccessStatusCode)
                                await ErrorHandler.ProcessErrorAsync(config.Metadata, response, cancellationToken).ConfigureAwait(false);

                            return response;
                        }
                        finally
                        {
                            request.Content = null;
                        }
                    }
                    catch (InternalServerErrorException)
                    {
                        if (!config.RetryStrategies.InternalServerErrorStrategy.TryGetRetryDelay(internalServerErrorRetries++, out delay))
                            throw;
                    }
                    catch (LimitExceededException)
                    {
                        if (!config.RetryStrategies.LimitExceededStrategy.TryGetRetryDelay(limitExceededRetries++, out delay))
                            throw;
                    }
                    catch (ProvisionedThroughputExceededException)
                    {
                        if (!config.RetryStrategies.ProvisionedThroughputExceededStrategy.TryGetRetryDelay(provisionedThroughputExceededRetries++, out delay))
                            throw;
                    }
                    catch (RequestLimitExceededException)
                    {
                        if (!config.RetryStrategies.RequestLimitExceededStrategy.TryGetRetryDelay(requestLimitExceededRetries++, out delay))
                            throw;
                    }
                    catch (ServiceUnavailableException)
                    {
                        if (!config.RetryStrategies.ServiceUnavailableStrategy.TryGetRetryDelay(serviceUnavailableRetries++, out delay))
                            throw;
                    }
                    catch (ThrottlingException)
                    {
                        if (!config.RetryStrategies.ThrottlingStrategy.TryGetRetryDelay(throttlingRetries++, out delay))
                            throw;
                    }

                    await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
                }
            }
            finally
            {
                httpContent.Dispose();
            }
        }

private async Task SendMessagesAsync(CancellationToken cancellationToken)
        {
            int messageCount = 0;

            while (!cancellationToken.IsCancellationRequested)
            {
                ++messageCount;
                var temperature = RandomGenerator.Next(20, 35);
                var humidity = RandomGenerator.Next(60, 80);
                string messagePayload = $"{{\"temperature\":{temperature},\"humidity\":{humidity}}}";
                var properties = new Dictionary<string, string>()
                {
                    {
                        "temperatureAlert", (temperature > TemperatureThreshold) ? "true" : "false"
                    }
                };

                while (true)
                {
                    var succeeded = await SendTelemetryAsync(messagePayload, messageCount.ToString(), cancellationToken, properties);
                    if (succeeded)
                        break;
                    // wait and retry
                    await Task.Delay(SleepDuration, cancellationToken);
                }
                await Task.Delay(SleepDuration, cancellationToken);
            }
        }

public static async Task<TResult> TimeoutAfter<TResult>(this Task<TResult> task, TimeSpan timeout)
        {
            using (var timeoutCancellationTokenSource = new CancellationTokenSource())
            {
                var completedTask = await Task.WhenAny(task, Task.Delay(timeout, timeoutCancellationTokenSource.Token));
                if (completedTask == task)
                {
                    timeoutCancellationTokenSource.Cancel();
                    return await task; // Very important in order to propagate exceptions
                }

                throw new TimeoutException("The operation has timed out.");
            }
        }

public static async Task TimeoutAfter(this Task task, TimeSpan timeout)
        {
            using (var timeoutCancellationTokenSource = new CancellationTokenSource())
            {
                var completedTask = await Task.WhenAny(task, Task.Delay(timeout, timeoutCancellationTokenSource.Token));
                if (completedTask == task)
                {
                    timeoutCancellationTokenSource.Cancel();
                    await task; // Very important in order to propagate exceptions
                    return;
                }

                throw new TimeoutException("The operation has timed out.");
            }
        }

public Task Delay(TimeSpan delay, CancellationToken cancellationToken = default)
        {
            return Task.Delay(delay, cancellationToken);
        }

private async Task Reconnect()
		{
			try
			{
				logger?.LogTrace("ModbusClient.Reconnect enter");
				lock (reconnectLock)
				{
					if (isReconnecting || stopCts.IsCancellationRequested)
						return;

					isReconnecting = true;
					IsConnected = false;
				}

				logger?.LogInformation($"{(wasConnected ? "Reconnect" : "Connect")} starting.");
				if (wasConnected)
					Task.Run(() => Disconnected?.Invoke(this, EventArgs.Empty)).Forget();

				sendCts?.Cancel();
				await sendTask;
				sendCts = null;

				int timeout = 2;
				int maxTimeout = 30;
				var startTime = DateTime.UtcNow;

				using (stopCts.Token.Register(() => reconnectTcs.TrySetCanceled()))
				{
					while (!stopCts.IsCancellationRequested)
					{
						try
						{
							serialPort?.Dispose();

							serialPort = new SerialPort(PortName)
							{
								BaudRate = (int)BaudRate,
								DataBits = DataBits,
								Parity = Parity,
								StopBits = StopBits,
								Handshake = Handshake,
								ReadTimeout = (int)ReceiveTimeout.TotalMilliseconds,
								WriteTimeout = (int)SendTimeout.TotalMilliseconds
							};

							if (bufferSize > 0)
							{
								serialPort.ReadBufferSize = bufferSize;
								serialPort.WriteBufferSize = bufferSize;
							}

							var task = Task.Run(() => serialPort.Open());
							if (await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(timeout), stopCts.Token)) == task)
							{
								if (serialPort.IsOpen)
								{
									lock (reconnectLock)
									{
										IsConnected = true;
										wasConnected = true;

										reconnectTcs?.TrySetResult(true);
										Task.Run(() => Connected?.Invoke(this, EventArgs.Empty)).Forget();
									}
									logger?.LogInformation($"{(wasConnected ? "Reconnect" : "Connect")}ed successfully.");

									sendCts = new CancellationTokenSource();
									sendTask = Task.Run(async () => await ProcessSendQueue());
									return;
								}
								else
								{
									logger?.LogError($"{(wasConnected ? "Reconnect" : "Connect")} failed: Could not open serial port {serialPort.PortName}.");
									reconnectTcs?.TrySetException((Exception)task.Exception ?? new IOException("Serial port not opened."));
									return;
								}
							}
							else if (stopCts.IsCancellationRequested)
							{
								logger?.LogInformation($"{(wasConnected ? "Reconnect" : "Connect")} cancelled.");
								return;
							}
							else
							{
								logger?.LogWarning($"{(wasConnected ? "Reconnect" : "Connect")} failed within {timeout} seconds.");
								timeout += 2;
								if (timeout > maxTimeout)
									timeout = maxTimeout;

								throw new IOException();
							}
						}
						catch (IOException) when (ReconnectTimeSpan == TimeSpan.MaxValue || DateTime.UtcNow <= startTime + ReconnectTimeSpan)
						{
							await Task.Delay(1000, stopCts.Token);
							continue;
						}
						catch (Exception ex)
						{
							logger?.LogError(ex, "ModbusClient.Reconnect failed");
							reconnectTcs?.TrySetException(ex);
							return;
						}
					}
				}
			}
			catch (OperationCanceledException) when (stopCts.IsCancellationRequested)
			{
				// Client shutting down
				return;
			}
			finally
			{
				lock (reconnectLock)
				{
					isReconnecting = false;
					reconnectTcs = null;
				}
				logger?.LogTrace("ModbusClient.Reconnect leave");
			}
		}

private async Task ProcessSendQueue()
		{
			try
			{
				logger?.LogTrace("ModbusClient.ProcessSendQueue enter");
				while (!sendCts.IsCancellationRequested)
				{
					RequestTask task = null;
					try
					{
						SpinWait.SpinUntil(() => sendCts.IsCancellationRequested || sendQueue.Any());
						if (sendCts.IsCancellationRequested)
							return;

						lock (queueLock)
						{
							task = sendQueue.First();
							sendQueue.Remove(task);
						}

						logger?.LogTrace(task.Request.ToString());

						// clear all data
						await serialPort.BaseStream.FlushAsync();
						serialPort.DiscardInBuffer();
						serialPort.DiscardOutBuffer();

						logger?.LogDebug($"Sending {task.Request}");
						byte[] bytes = task.Request.Serialize();
						await serialPort.WriteAsync(bytes, 0, bytes.Length, sendCts.Token);
						logger?.LogDebug("Request sent.");

						var responseBytes = new List<byte>
						{
							// Device/Slave ID
							await ReadByte(sendCts.Token)
						};

						// Function number
						byte fn = await ReadByte(sendCts.Token);
						responseBytes.Add(fn);

						byte expectedBytes = 0;
						var function = (FunctionCode)((fn & Consts.ErrorMask) > 0 ? fn ^ Consts.ErrorMask : fn);
						switch (function)
						{
							case FunctionCode.ReadCoils:
							case FunctionCode.ReadDiscreteInputs:
							case FunctionCode.ReadHoldingRegisters:
							case FunctionCode.ReadInputRegisters:
								expectedBytes = await ReadByte(sendCts.Token);
								responseBytes.Add(expectedBytes);
								break;
							case FunctionCode.WriteSingleCoil:
							case FunctionCode.WriteSingleRegister:
							case FunctionCode.WriteMultipleCoils:
							case FunctionCode.WriteMultipleRegisters:
								expectedBytes = 4;
								break;
							case FunctionCode.EncapsulatedInterface:
								responseBytes.AddRange(await ReadBytes(6, sendCts.Token));
								byte count = responseBytes.Last();
								for (int i = 0; i < count; i++)
								{
									// id
									responseBytes.Add(await ReadByte(sendCts.Token));
									// length
									expectedBytes = await ReadByte(sendCts.Token);
									responseBytes.Add(expectedBytes);
									// value
									responseBytes.AddRange(await ReadBytes(expectedBytes, sendCts.Token));
								}
								expectedBytes = 0;
								break;
							default:
								if ((fn & Consts.ErrorMask) == 0)
									throw new NotImplementedException();

								expectedBytes = 1;
								break;
						}

						expectedBytes += 2; // CRC Check

						responseBytes.AddRange(await ReadBytes(expectedBytes, sendCts.Token));
						logger?.LogDebug("Response received.");

						var response = new Response(responseBytes.ToArray());
						task.TaskCompletionSource.TrySetResult(response);
					}
					catch (OperationCanceledException) when (sendCts.IsCancellationRequested)
					{
						task?.TaskCompletionSource.TrySetResult(new Response(new byte[] { 0, 0, 0, 0, 0, 0 }));
					}
					catch (TimeoutException)
					{
						task?.TaskCompletionSource.TrySetResult(new Response(new byte[] { 0, 0, 0, 0, 0, 0 }));
					}
					catch (Exception ex)
					{
						logger?.LogError(ex, $"Unexpected error ({ex.GetType().Name}) on send: {ex.GetMessage()}");
						task?.TaskCompletionSource.TrySetResult(new Response(new byte[] { 0, 0, 0, 0, 0, 0 }));
					}
					finally
					{
						task?.Registration.Dispose();
						try
						{
							await Task.Delay(InterRequestDelay, sendCts.Token);
						}
						catch
						{ }
					}
				}
			}
			finally
			{
				logger?.LogTrace("ModbusClient.ProcessSendQueue leave");
			}
		}

public async Task Connect(CancellationToken cancellationToken = default)
		{
			var cancelTask = ReconnectTimeSpan == TimeSpan.MaxValue
				? Task.Delay(Timeout.Infinite, cancellationToken)
				: Task.Delay(ReconnectTimeSpan, cancellationToken);

			try
			{
				logger?.LogTrace("ModbusClient.Connect enter");
				CheckDisposed();

				if (isStarted)
				{
					await Task.WhenAny(ConnectingTask, cancelTask);
					return;
				}
				isStarted = true;
				stopCts = new CancellationTokenSource();

				logger?.LogInformation("Modbus client starting.");

				lock (reconnectLock)
				{
					transactionId = 0;
					IsConnected = false;
					wasConnected = false;
					ConnectingTask = GetReconnectTask(true);
				}

				logger?.LogInformation("Modbus client started.");
				await Task.WhenAny(ConnectingTask, cancelTask);
			}
			catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
			{ }
			finally
			{
				if (cancelTask.Status != TaskStatus.WaitingForActivation)
					cancelTask.Dispose();
				logger?.LogTrace("ModbusClient.Connect leave");
			}
		}

private async Task Reconnect()
		{
			try
			{
				logger?.LogTrace("ModbusClient.Reconnect enter");
				lock (reconnectLock)
				{
					if (isReconnecting || stopCts.IsCancellationRequested)
						return;

					isReconnecting = true;
					IsConnected = false;
				}

				logger?.LogInformation($"{(wasConnected ? "Rec" : "C")}onnect starting.");
				if (wasConnected)
				{
					receiveCts?.Cancel();
					await receiveTask;
					receiveCts = null;
					receiveTask = Task.CompletedTask;
					Task.Run(() => Disconnected?.Invoke(this, EventArgs.Empty)).Forget();
				}

				var timeout = TimeSpan.FromSeconds(2);
				var startTime = DateTime.UtcNow;

				var address = ResolveHost(Host);
				while (!stopCts.IsCancellationRequested)
				{
					try
					{
						stream?.Dispose();
						stream = null;

						tcpClient?.Dispose();
						tcpClient = new TcpClient(address.AddressFamily);

						var connectTask = tcpClient.ConnectAsync(address, Port);
						if (await Task.WhenAny(connectTask, Task.Delay(timeout, stopCts.Token)) == connectTask && tcpClient.Connected)
						{
							SetKeepAlive();
							stream = tcpClient.GetStream();

							receiveCts = new CancellationTokenSource();
							receiveTask = Task.Run(async () => await ReceiveLoop());

							lock (reconnectLock)
							{
								IsConnected = true;
								wasConnected = true;

								reconnectTcs?.TrySetResult(true);
								Task.Run(() => Connected?.Invoke(this, EventArgs.Empty)).Forget();
							}
							logger?.LogInformation($"{(wasConnected ? "Rec" : "C")}onnected successfully.");
							return;
						}
						else
						{
							if (timeout < MaxConnectTimeout)
							{
								logger?.LogWarning($"{(wasConnected ? "Rec" : "C")}onnect failed within {timeout}.");
								timeout = timeout.Add(TimeSpan.FromSeconds(2));
								if (timeout > MaxConnectTimeout)
									timeout = MaxConnectTimeout;
							}
							throw new SocketException((int)SocketError.TimedOut);
						}
					}
					catch (SocketException) when (ReconnectTimeSpan == TimeSpan.MaxValue || DateTime.UtcNow - startTime <= ReconnectTimeSpan)
					{
						await Task.Delay(1000, stopCts.Token);
						continue;
					}
					catch (OperationCanceledException) when (stopCts.IsCancellationRequested)
					{
						throw;
					}
					catch (Exception ex)
					{
						logger?.LogError(ex, $"{(wasConnected ? "Rec" : "C")}onnecting failed: {ex.GetMessage()}, trying again.");
					}
				}
			}
			catch (OperationCanceledException) when (stopCts.IsCancellationRequested)
			{
				reconnectTcs?.TrySetCanceled();
			}
			catch (Exception ex)
			{
				logger?.LogError(ex, $"{(wasConnected ? "Rec" : "C")}onnecting failed: {ex.GetMessage()}");
				reconnectTcs?.TrySetException(ex);
			}
			finally
			{
				lock (reconnectLock)
				{
					isReconnecting = false;
					reconnectTcs = null;
				}
				logger?.LogTrace("ModbusClient.Reconnect leave");
			}
		}

public async Task Connect(CancellationToken cancellationToken = default)
		{
			var cancelTask = ReconnectTimeSpan == TimeSpan.MaxValue
				? Task.Delay(Timeout.Infinite, cancellationToken)
				: Task.Delay(ReconnectTimeSpan, cancellationToken);

			try
			{
				logger?.LogTrace("ModbusClient.Connect enter");
				CheckDisposed();

				if (isStarted)
				{
					await Task.WhenAny(ConnectingTask, cancelTask);
					return;
				}
				isStarted = true;
				stopCts = new CancellationTokenSource();

				logger?.LogInformation("ModbusClient starting.");

				if (DriverEnableRS485 && RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
				{
					try
					{
						var rs485 = GetDriverState();
						serialDriverFlags = rs485.Flags;
						rs485.Flags |= RS485Flags.Enabled;
						rs485.Flags &= ~RS485Flags.RxDuringTx;
						SetDriverState(rs485);
						driverModified = true;
					}
					catch (Exception ex)
					{
						logger?.LogError(ex, $"Set driver state to RS485 failed: {ex.GetMessage()}");
						throw;
					}
				}

				lock (reconnectLock)
				{
					IsConnected = false;
					wasConnected = false;
					ConnectingTask = GetReconnectTask(true);
				}

				logger?.LogInformation("Modbus client started.");
				await Task.WhenAny(ConnectingTask, cancelTask);
			}
			finally
			{
				if (cancelTask.Status != TaskStatus.WaitingForActivation)
					cancelTask.Dispose();
				logger?.LogTrace("ModbusClient.Connect leave");
			}
		}

private static async Task<int> RunClientAsync(ILogger logger, CancellationToken cancellationToken)
		{
			Console.Write("Connection Type [1] TCP, [2] RS485: ");
			int cType = Convert.ToInt32(Console.ReadLine().Trim());

			IModbusClient client = null;
			try
			{
				switch (cType)
				{
					case 1:
						{
							Console.Write("Hostname: ");
							string host = Console.ReadLine().Trim();
							Console.Write("Port: ");
							int port = Convert.ToInt32(Console.ReadLine().Trim());

							client = new TcpClient(host, port, logger);
						}
						break;
					case 2:
						{
							Console.Write("Interface: ");
							string port = Console.ReadLine().Trim();

							Console.Write("Baud: ");
							int baud = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Stop-Bits [0|1|2|3=1.5]: ");
							int stopBits = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Parity [0] None [1] Odd [2] Even [3] Mark [4] Space: ");
							int parity = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Handshake [0] None [1] X-On/Off [2] RTS [3] RTS+X-On/Off: ");
							int handshake = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Timeout (ms): ");
							int timeout = Convert.ToInt32(Console.ReadLine().Trim());

							Console.Write("Set Driver to RS485 [0] No [1] Yes: ");
							int setDriver = Convert.ToInt32(Console.ReadLine().Trim());

							client = new SerialClient(port)
							{
								BaudRate = (BaudRate)baud,
								DataBits = 8,
								StopBits = (StopBits)stopBits,
								Parity = (Parity)parity,
								Handshake = (Handshake)handshake,
								SendTimeout = TimeSpan.FromMilliseconds(timeout),
								ReceiveTimeout = TimeSpan.FromMilliseconds(timeout)
							};

							if (setDriver == 1)
							{
								((SerialClient)client).DriverEnableRS485 = true;
							}
						}
						break;
					default:
						Console.Error.WriteLine($"Unknown type: {cType}");
						return 1;
				}

				await Task.WhenAny(client.Connect(), Task.Delay(Timeout.Infinite, cancellationToken));
				if (cancellationToken.IsCancellationRequested)
					return 0;

				while (!cancellationToken.IsCancellationRequested)
				{
					Console.Write("Device ID: ");
					byte id = Convert.ToByte(Console.ReadLine().Trim());

					Console.Write("Function [1] Read Register, [2] Device Info, [9] Write Register : ");
					int fn = Convert.ToInt32(Console.ReadLine().Trim());

					switch (fn)
					{
						case 1:
							{
								ushort address = 0;
								ushort count = 0;
								string type = "";

								Console.WriteLine();
								Console.Write("Address : ");
								address = Convert.ToUInt16(Console.ReadLine().Trim());
								Console.Write("DataType: ");
								type = Console.ReadLine().Trim();
								if (type == "string")
								{
									Console.Write("Register Count: ");
									count = Convert.ToUInt16(Console.ReadLine().Trim());
								}

								Console.WriteLine();
								Console.Write("Run as loop? [y/N]: ");
								string loop = Console.ReadLine().Trim().ToLower();
								int interval = 0;
								if (yesList.Contains(loop))
								{
									Console.Write("Loop interval (milliseconds): ");
									interval = Convert.ToInt32(Console.ReadLine().Trim());
								}

								Console.WriteLine();
								do
								{
									try
									{
										Console.Write("Result  : ");
										List<Register> result = null;
										switch (type.Trim().ToLower())
										{
											case "byte":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetByte());
												break;
											case "ushort":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetUInt16());
												break;
											case "uint":
												result = await client.ReadHoldingRegisters(id, address, 2);
												Console.WriteLine(result?.GetUInt32());
												break;
											case "ulong":
												result = await client.ReadHoldingRegisters(id, address, 4);
												Console.WriteLine(result?.GetUInt64());
												break;
											case "sbyte":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetSByte());
												break;
											case "short":
												result = await client.ReadHoldingRegisters(id, address, 1);
												Console.WriteLine(result?.First().GetInt16());
												break;
											case "int":
												result = await client.ReadHoldingRegisters(id, address, 2);
												Console.WriteLine(result?.GetInt32());
												break;
											case "long":
												result = await client.ReadHoldingRegisters(id, address, 4);
												Console.WriteLine(result?.GetInt64());
												break;
											case "float":
												result = await client.ReadHoldingRegisters(id, address, 2);
												Console.WriteLine(result?.GetSingle());
												break;
											case "double":
												result = await client.ReadHoldingRegisters(id, address, 4);
												Console.WriteLine(result?.GetDouble());
												break;
											case "string":
												result = await client.ReadHoldingRegisters(id, address, count);
												Console.WriteLine();
												Console.WriteLine("UTF8:             " + result?.GetString(count));
												Console.WriteLine("Unicode:          " + result?.GetString(count, 0, Encoding.Unicode));
												Console.WriteLine("BigEndianUnicode: " + result?.GetString(count, 0, Encoding.BigEndianUnicode));
												break;
											default:
												Console.Write("DataType unknown");
												break;
										}
									}
									catch
									{ }
									await Task.Delay(TimeSpan.FromMilliseconds(interval), cancellationToken);
								}
								while (interval > 0 && !cancellationToken.IsCancellationRequested);
							}
							break;
						case 2:
							{
								Console.Write("[1] Basic, [2] Regular, [3] Extended: ");
								int cat = Convert.ToInt32(Console.ReadLine().Trim());

								Dictionary<DeviceIDObject, string> info = null;
								switch (cat)
								{
									case 1:
										info = await client.ReadDeviceInformation(id, DeviceIDCategory.Basic);
										break;
									case 2:
										info = await client.ReadDeviceInformation(id, DeviceIDCategory.Regular);
										break;
									case 3:
										info = await client.ReadDeviceInformation(id, DeviceIDCategory.Extended);
										break;
								}
								if (info != null)
								{
									foreach (var kvp in info)
									{
										Console.WriteLine($"{kvp.Key}: {kvp.Value}");
									}
								}
							}
							break;
						case 9:
							{
								Console.Write("Address: ");
								ushort address = Convert.ToUInt16(Console.ReadLine().Trim());

								Console.Write("Bytes (HEX): ");
								string byteStr = Console.ReadLine().Trim();
								byteStr = byteStr.Replace(" ", "").ToLower();

								byte[] bytes = Enumerable.Range(0, byteStr.Length)
									.Where(i => i % 2 == 0)
									.Select(i => Convert.ToByte(byteStr.Substring(i, 2), 16))
									.ToArray();

								var registers = Enumerable.Range(0, bytes.Length)
									.Where(i => i % 2 == 0)
									.Select(i =>
									{
										return new Register
										{
											Type = ModbusObjectType.HoldingRegister,
											Address = address++,
											HiByte = bytes[i],
											LoByte = bytes[i + 1]
										};
									})
									.ToList();

								if (!await client.WriteRegisters(id, registers))
									throw new Exception($"Writing '{byteStr}' to address {address} failed");
							}
							break;
					}

					Console.Write("New Request? [y/N]: ");
					string again = Console.ReadLine().Trim().ToLower();
					if (!yesList.Contains(again))
						return 0;
				}

				return 0;
			}
			catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
			{
				return 0;
			}
			finally
			{
				Console.WriteLine("Disposing");
				client?.Dispose();
				Console.WriteLine("Disposed");
			}
		}

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            while (!stoppingToken.IsCancellationRequested)
            {
                using (var scope = _provider.CreateScope())
                {
                    _logger.LogInformation("Fetching latest rates");
                    var client = _provider.GetRequiredService<ExchangeRatesClient>();
                    var latestRates = await client.GetLatestRatesAsync();
                    var context = scope.ServiceProvider.GetRequiredService<AppDbContext>();
                    context.Add(latestRates);
                    await context.SaveChangesAsync();
                    _logger.LogInformation("Latest rates updated");
                }
                await Task.Delay(_refreshInterval, stoppingToken);
            }
        }

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            while (!stoppingToken.IsCancellationRequested)
            {
                _logger.LogInformation("Fetching latest rates");
                var client = _provider.GetRequiredService<ExchangeRatesClient>();
                var latest = await client.GetLatestRatesAsync();
                _cache.SetRates(latest);
                _logger.LogInformation("Latest rates updated");

                await Task.Delay(_refreshInterval, stoppingToken);
            }
        }

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
        {
            while (!stoppingToken.IsCancellationRequested)
            {
                using(var scope = _provider.CreateScope())
                {
                    _logger.LogInformation("Fetching latest rates");
                    var client = _provider.GetRequiredService<ExchangeRatesClient>();
                    var latestRates = await client.GetLatestRatesAsync();
                    var context = scope.ServiceProvider.GetRequiredService<AppDbContext>();
                    context.Add(latestRates);
                    await context.SaveChangesAsync();
                    _logger.LogInformation("Latest rates updated");
                }
                await Task.Delay(_refreshInterval, stoppingToken);
            }
        }