System.Diagnostics.Contracts.Contract.Requires(bool)

internal string GetSignResource(string unescapedPath, Dictionary<string, string[]>  unescapedQueries)
        {
            Contract.Requires(unescapedQueries != null);
            List<string> param = new List<string>();

            // http trigger special sign
            if (unescapedQueries != null)
            {
                foreach(var item in unescapedQueries)
                {
                    if(item.Value.Length > 0)
                    {
                        foreach(string v in item.Value)
                        {
                            param.Add(string.Format("{0}={1}", item.Key, v));
                        }
                    }
                    else
                    {
                        param.Add(item.Key);
                    }
                }
            }
            string resource = unescapedPath + "\n" + string.Join("\n", param.ToArray());
            return resource;
        }

public static T[] ConcatArrays<T>(T[] arr1, T[] arr2)
		{
			Contract.Requires(arr1 != null);
			Contract.Requires(arr2 != null);
			Contract.Ensures(Contract.Result<T[]>() != null);
			Contract.Ensures(Contract.Result<T[]>().Length == arr1.Length + arr2.Length);

			var result = new T[arr1.Length + arr2.Length];
			Buffer.BlockCopy(arr1, 0, result, 0, arr1.Length);
			Buffer.BlockCopy(arr2, 0, result, arr1.Length, arr2.Length);
			return result;
		}

public static byte[] VerifyAndRemoveCheckSum(byte[] data)
		{
			Contract.Requires<ArgumentNullException>(data != null);
			Contract.Ensures(Contract.Result<byte[]>() == null || Contract.Result<byte[]>().Length + CheckSumSizeInBytes == data.Length);
			byte[] result = ArrayHelpers.SubArray(data, 0, data.Length - CheckSumSizeInBytes);
			byte[] givenCheckSum = ArrayHelpers.SubArray(data, data.Length - CheckSumSizeInBytes);
			byte[] correctCheckSum = GetCheckSum(result);
			if (givenCheckSum.SequenceEqual(correctCheckSum))
				return result;
			else
				return null;
		}

private static byte[] GetCheckSum(byte[] data)
		{
			Contract.Requires<ArgumentNullException>(data != null);
			Contract.Ensures(Contract.Result<byte[]>() != null);

			SHA256 sha256 = new SHA256Managed();
			byte[] hash1 = sha256.ComputeHash(data);
			byte[] hash2 = sha256.ComputeHash(hash1);

			var result = new byte[CheckSumSizeInBytes];
			Buffer.BlockCopy(hash2, 0, result, 0, result.Length);

			return result;
		}

public static T[] ConcatArrays<T>(params T[][] arrays)
		{
			Contract.Requires(arrays != null);
			Contract.Requires(Contract.ForAll(arrays, (arr) => arr != null));
			Contract.Ensures(Contract.Result<T[]>() != null);
			Contract.Ensures(Contract.Result<T[]>().Length == arrays.Sum(arr => arr.Length));

			var result = new T[arrays.Sum(arr => arr.Length)];
			int offset = 0;
			for (int i = 0; i < arrays.Length; i++)
			{
				var arr = arrays[i];
				Buffer.BlockCopy(arr, 0, result, offset, arr.Length);
				offset += arr.Length;
			}
			return result;
		}

public static T[] SubArray<T>(T[] arr, int start, int length)
		{
			Contract.Requires(arr != null);
			Contract.Requires(start >= 0);
			Contract.Requires(length >= 0);
			Contract.Requires(start + length <= arr.Length);
			Contract.Ensures(Contract.Result<T[]>() != null);
			Contract.Ensures(Contract.Result<T[]>().Length == length);

			var result = new T[length];
			Buffer.BlockCopy(arr, start, result, 0, length);
			return result;
		}

public static T[] SubArray<T>(T[] arr, int start)
		{
			Contract.Requires(arr != null);
			Contract.Requires(start >= 0);
			Contract.Requires(start <= arr.Length);
			Contract.Ensures(Contract.Result<T[]>() != null);
			Contract.Ensures(Contract.Result<T[]>().Length == arr.Length - start);

			return SubArray(arr, start, arr.Length - start);
		}

public static byte[] AddCheckSum(byte[] data)
		{
			Contract.Requires<ArgumentNullException>(data != null);
			Contract.Ensures(Contract.Result<byte[]>().Length == data.Length + CheckSumSizeInBytes);
			byte[] checkSum = GetCheckSum(data);
			byte[] dataWithCheckSum = ArrayHelpers.ConcatArrays(data, checkSum);
			return dataWithCheckSum;
		}

public static string EncodeWithCheckSum(byte[] data)
		{
			Contract.Requires<ArgumentNullException>(data != null);
			Contract.Ensures(Contract.Result<string>() != null);
			return Encode(AddCheckSum(data));
		}

public static byte[] DecodeWithCheckSum(string s)
		{
			Contract.Requires<ArgumentNullException>(s != null);
			Contract.Ensures(Contract.Result<byte[]>() != null);
			var dataWithCheckSum = Decode(s);
			var dataWithoutCheckSum = VerifyAndRemoveCheckSum(dataWithCheckSum);
			if (dataWithoutCheckSum == null)
				throw new FormatException("Base58 checksum is invalid");
			return dataWithoutCheckSum;
		}

protected override StageResult Init()
        {
            if (TrainOp)
            {
                Contract.Requires(TrainingFile != null && TestFile == null);
                if (!TrainingFile.CheckExistsAndReportError(L))
                {
                    return StageResult.INPUT_ERROR;
                }
                if (!TestFile.CheckExistsAndReportError(L))
                {
                    return StageResult.INPUT_ERROR;
                }
                if (!ModelFileName.IsEmpty() && ModelFile.Exists && !OverwriteOutputFile)
                {
                    Error("The model file {0} exists but the overwrite option was not specified.", ModelFile.FullName);
                    return StageResult.INPUT_ERROR;
                }
            }
            return StageResult.SUCCESS;
        }

protected override StageResult Write()
        {
            Contract.Requires(OutputFile != null && OutputFile.Exists);
            Contract.Requires(ExtractedRecords != null);
            if (ExtractedRecords.Count == 0)
            {
                Warn("0 records extracted from file {0}. Not writing to output file.", InputFile.FullName);
                return StageResult.SUCCESS;
            }
            using (Operation writeOp = Begin("Writing {0} records to {file}", ExtractedRecords.Count, OutputFile.FullName))
            {
                JsonSerializer serializer = new JsonSerializer();
                serializer.Formatting = Formatting.Indented;
                if (!CompressOutputFile)
                {
                    using (StreamWriter sw = new StreamWriter(OutputFile.FullName, false, Encoding.UTF8))
                    {
                        serializer.Serialize(sw, ExtractedRecords);
                    }
                }
                else
                {
                    using (FileStream fs = new FileStream(OutputFile.FullName, FileMode.Create))
                    using (GZipStream gzs = new GZipStream(fs, CompressionMode.Compress))
                    using (StreamWriter sw = new StreamWriter(gzs, Encoding.UTF8))
                    {
                        serializer.Serialize(sw, ExtractedRecords);
                    }
                }
                writeOp.Complete();
                return StageResult.SUCCESS;
            }
        }

protected override StageResult Init()
        {
            Contract.Requires(InputFile != null && TrainingFile != null && TestFile != null);
            
            if (!InputFile.CheckExistsAndReportError(L))
            {
                return StageResult.INPUT_ERROR;
            }

            if (TrainingFile.Exists && !OverwriteOutputFile)
            {
                Error("The training data file {0} exists but the overwrite option was not specified.", TrainingFile.FullName);
                return StageResult.OUTPUT_ERROR;
            }
            else if (TrainingFile.Exists)
            {
                Warn("Training data file {0} exists and will be overwritten.", TrainingFile.FullName);
            }

            if (TestFile.Exists && !OverwriteOutputFile)
            {
                Error("The test data file {0} exists but the overwrite option was not specified.", TestFile.FullName);
                return StageResult.OUTPUT_ERROR;
            }
            else if (TestFile.Exists)
            {
                Warn("Test data file {0} exists and will be overwritten.", TestFile.FullName);
            }

            return StageResult.SUCCESS;
        }

protected override StageResult Write()
        {
            Contract.Requires(TrainingRecords.Count > 0);
            Contract.Requires(TestRecords.Count > 0);
            StageResult r = StageResult.OUTPUT_ERROR;
            if ((r = Save(TrainingFile, TrainingRecords)) != StageResult.SUCCESS)
            {
                return StageResult.OUTPUT_ERROR;
            }
            Info("Wrote {0} training records to file {1}.", TrainingRecords.Count, TrainingFile.FullName);
            if ((r = Save(TestFile, TestRecords)) != StageResult.SUCCESS)
            {
                return StageResult.OUTPUT_ERROR;
            }
            Info("Wrote {0} test records to file {1}.", TestRecords.Count, TestFile.FullName);
            return StageResult.SUCCESS;
        }

protected override StageResult Extract(int? recordBatchSize = null, int? recordLimit = null, Dictionary<string, string> options = null)
        {
            Contract.Requires(InputFileUri != null);
            if (InputFileUri.Segments.Any(s => s.EndsWith(".zip")))
            {
                InputFileName += ".zip";
            }
            else if (InputFileUri.Segments.Any(s => s.EndsWith(".gz")))
            {
                InputFileName += ".gz";
            }
            else if (InputFileUri.Segments.Any(s => s.EndsWith(".tar.gz")))
            {
                InputFileName += ".tar.gz";
            }
            FileDownload = new HttpFileDownload(InputFileUri.ToString(), TempFile);
            FileDownloadTask = FileDownload.StartTask();
            try
            {
                FileDownloadTask.Wait();
            }
            catch (Exception e)
            {
                Error(e, "Exception throw attempting to download file from Url {0} to {1}.", InputFileUri.ToString(), TempFile);
                return StageResult.INPUT_ERROR;
            }

            if (!FileDownload.CompletedSuccessfully)
            {
                Error("Failed to download file from Url {0} to {1}.", InputFileUri.ToString(), TempFile);
                return StageResult.INPUT_ERROR;
            }
            return base.Extract();  
        }

static void Benchmark<T>() where T : struct, IEquatable<T>, IComparable<T>, IConvertible
        {
            Contract.Requires(BenchmarkOptions.ContainsKey("Category"));
            Contract.Requires(BenchmarkOptions.ContainsKey("Operation"));
            Contract.Requires(BenchmarkOptions.ContainsKey("Sizes"));
            IConfig config = ManualConfig
                         .Create(DefaultConfig.Instance);
            JemBenchmarkAttribute.CurrentConfig = config;
            JemBenchmarkAttribute.Category = (Category)BenchmarkOptions["Category"];
            JemBenchmarkAttribute.Operation = (Operation)BenchmarkOptions["Operation"];
            try
            {
                switch ((Category)BenchmarkOptions["Category"])
                {
                    case Category.MALLOC:
                        MallocVsArrayBenchmark<T>.Debug = (bool)BenchmarkOptions["Debug"];
                        MallocVsArrayBenchmark<T>.Category = JemBenchmarkAttribute.Category;
                        MallocVsArrayBenchmark<T>.Operation = JemBenchmarkAttribute.Operation;
                        MallocVsArrayBenchmark<T>.BenchmarkParameters = ((IEnumerable<int>)BenchmarkOptions["Sizes"]).ToList();
                        switch ((Operation)BenchmarkOptions["Operation"])
                        {
                            case Operation.CREATE:
                                config = config.With(new NameFilter(name => name.Contains("Create")));
                                L.Information("Starting {num} create benchmarks for data type {t} with array sizes: {s}", JemBenchmark<T, int>.GetBenchmarkMethodCount<MallocVsArrayBenchmark<T>>(),
                                    typeof(T).Name, MallocVsArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                break;

                            case Operation.FILL:
                                config = config.With(new NameFilter(name => name.Contains("Fill")));
                                L.Information("Starting {num} fill benchmarks for data type {t} with array sizes: {s}", JemBenchmark<T, int>.GetBenchmarkMethodCount<MallocVsArrayBenchmark<T>>(),
                                    typeof(T).Name, MallocVsArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");                        
                                break;

                            case Operation.FRAGMENT:
                                config = config.With(new NameFilter(name => name.Contains("Fragment")));
                                L.Information("Starting {num} fragment benchmarks for data type {t} with array sizes: {s}", JemBenchmark<T, int>.GetBenchmarkMethodCount<MallocVsArrayBenchmark<T>>(),
                                    typeof(T).Name, MallocVsArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                break;

                            default:
                                throw new InvalidOperationException($"Unknown operation: {(Operation)BenchmarkOptions["Operation"]} for category {(Category)BenchmarkOptions["Category"]}.");
                        }
                        BenchmarkSummary = BenchmarkRunner.Run<MallocVsArrayBenchmark<T>>(config);
                        break;

                    case Category.BUFFER:
                        FixedBufferVsManagedArrayBenchmark<T>.BenchmarkParameters = ((IEnumerable<int>)BenchmarkOptions["Sizes"]).ToList();
                        FixedBufferVsManagedArrayBenchmark<T>.Debug = (bool)BenchmarkOptions["Debug"];
                        FixedBufferVsManagedArrayBenchmark<T>.Category = JemBenchmarkAttribute.Category;
                        FixedBufferVsManagedArrayBenchmark<T>.Operation = JemBenchmarkAttribute.Operation;
                        switch ((Operation)BenchmarkOptions["Operation"])
                        {
                            case Operation.CREATE:
                                L.Information("Starting {num} create array benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, int>.GetBenchmarkMethodCount<FixedBufferVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name,
                                    FixedBufferVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                config = config
                                    .With(new NameFilter(name => name.StartsWith("Create")));
                                break;

                            case Operation.FILL:
                                L.Information("Starting {num} array fill benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, int>.GetBenchmarkMethodCount<FixedBufferVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name, FixedBufferVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                config = config
                                    .With(new NameFilter(name => name.StartsWith("Fill")));
                                break;

                            case Operation.MATH:
                                L.Information("Starting {num} array math benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, int>.GetBenchmarkMethodCount<FixedBufferVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name, FixedBufferVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                config = config
                                    .With(new NameFilter(name => name.StartsWith("Arith")));
                                break;

                            default:
                                throw new InvalidOperationException($"Unknown operation: {(Operation)BenchmarkOptions["Operation"]} for category {(Category)BenchmarkOptions["Category"]}.");
                        }
                        BenchmarkSummary = BenchmarkRunner.Run<FixedBufferVsManagedArrayBenchmark<T>>(config);
                        break;

                    case Category.NARRAY:
                        SafeVsManagedArrayBenchmark<T>.BenchmarkParameters = ((IEnumerable<int>)BenchmarkOptions["Sizes"]).ToList();
                        SafeVsManagedArrayBenchmark<T>.Debug = (bool)BenchmarkOptions["Debug"];
                        SafeVsManagedArrayBenchmark<T>.Category = JemBenchmarkAttribute.Category;
                        SafeVsManagedArrayBenchmark<T>.Operation = JemBenchmarkAttribute.Operation;
                        switch ((Operation)BenchmarkOptions["Operation"])
                        {
                            case Operation.CREATE:
                                L.Information("Starting {num} create array benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, int>.GetBenchmarkMethodCount<SafeVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name,
                                    SafeVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                config = config
                                    .With(new NameFilter(name => name.StartsWith("Create")));
                                break;

                            case Operation.FILL:
                                L.Information("Starting {num} array fill benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, int>.GetBenchmarkMethodCount<SafeVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name, SafeVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                config = config
                                    .With(new NameFilter(name => name.StartsWith("Fill")));
                                break;

                            case Operation.MATH:
                                L.Information("Starting {num} array math benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, int>.GetBenchmarkMethodCount<SafeVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name, SafeVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("Please allow some time for the pilot and warmup phases of the benchmark.");
                                config = config
                                    .With(new NameFilter(name => name.StartsWith("Arith")));
                                break;

                            default:
                                throw new InvalidOperationException($"Unknown operation: {(Operation)BenchmarkOptions["Operation"]} for category {(Category)BenchmarkOptions["Category"]}.");
                        }
                        BenchmarkSummary = BenchmarkRunner.Run<SafeVsManagedArrayBenchmark<T>>(config);
                        break;

                    case Category.HUGEARRAY:
                        HugeNativeVsManagedArrayBenchmark<T>.BenchmarkParameters = ((IEnumerable<ulong>)BenchmarkOptions["Sizes"]).ToList();
                        HugeNativeVsManagedArrayBenchmark<T>.Debug = (bool)BenchmarkOptions["Debug"];
                        HugeNativeVsManagedArrayBenchmark<T>.Category = JemBenchmarkAttribute.Category;
                        HugeNativeVsManagedArrayBenchmark<T>.Operation = JemBenchmarkAttribute.Operation;
                        switch ((Operation)BenchmarkOptions["Operation"])
                        {
                            case Operation.CREATE:
                                config = config.With(new NameFilter(name => name.Contains("Create")));
                                L.Information("Starting {num} huge array create benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, ulong>.GetBenchmarkMethodCount<HugeNativeVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name, HugeNativeVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("This benchmark does not have a warmup phase but can still take a while (10-15 minutes.)");
                                break;

                            case Operation.FILL:
                                config = config.With(new NameFilter(name => name.Contains("Fill")));
                                L.Information("Starting {num} huge array fill benchmarks for data type {t} with array sizes: {s}",
                                    JemBenchmark<T, ulong>.GetBenchmarkMethodCount<HugeNativeVsManagedArrayBenchmark<T>>(),
                                    typeof(T).Name, HugeNativeVsManagedArrayBenchmark<T>.BenchmarkParameters);
                                L.Information("This benchmark does not have a warmup phase but can still take a while (10-15 minutes.)");
                                break;

                            default:
                                throw new InvalidOperationException($"Unknown operation: {(Operation)BenchmarkOptions["Operation"]} for category {(Category)BenchmarkOptions["Category"]}.");
                        }
                        BenchmarkSummary = BenchmarkRunner.Run<HugeNativeVsManagedArrayBenchmark<T>>(config);
                        break;

                    case Category.VECTOR:
                        VectorBenchmark<T>.BenchmarkParameters = ((IEnumerable<int>)BenchmarkOptions["Scales"]).ToList();
                        VectorBenchmark<T>.Debug = (bool)BenchmarkOptions["Debug"];
                        VectorBenchmark<T>.Category = JemBenchmarkAttribute.Category;
                        VectorBenchmark<T>.Operation = JemBenchmarkAttribute.Operation;
                        
                        switch ((Operation)BenchmarkOptions["Operation"])
                        {
                            case Operation.MANDELBROT:
                                config = config.With(BenchmarkStatisticColumn.ThreadCycles);
                                config = config.With(BenchmarkStatisticColumn.ISPCResult);
                                config = config.With(BenchmarkStatisticColumn.ISPCResult2);
                                config = config.With(new NameFilter(name => name.StartsWith("Mandelbrot")));
                                L.Information("Starting vector Mandelbrot benchmarks with scale: {s}", VectorBenchmark<T>.BenchmarkParameters);
                                break;

                            case Operation.FILL:
                                config = config.With(new NameFilter(name => name.StartsWith("Fill")));
                                L.Information("Starting vector fill benchmarks with array sizes: {s}", VectorBenchmark<T>.BenchmarkParameters);
                                break;

                            case Operation.TEST:
                                config = config.With(new NameFilter(name => name.StartsWith("Test")));
                                L.Information("Starting vector logical comparison and test benchmarks with array sizes: {s}", VectorBenchmark<T>.BenchmarkParameters);
                                break;

                            default:
                                throw new InvalidOperationException($"Unknown operation: {(Operation)BenchmarkOptions["Operation"]} for category {(Category)BenchmarkOptions["Category"]}.");
                        }
                        BenchmarkSummary = BenchmarkRunner.Run<VectorBenchmark<T>>(config);
                        break;

                    default:
                        throw new InvalidOperationException($"Unknown category: {(Category)BenchmarkOptions["Category"]}.");
                }
            }
            catch (Exception e)
            {
                Log.Error(e, "Exception thrown during benchmark.");
                Exit(ExitResult.UNHANDLED_EXCEPTION);
            }
        }

protected virtual StageResult Load(int? recordBatchSize = null, int? recordLimit = null, Dictionary<string, string> options = null)
        {
            Contract.Requires(InputRecords.Count > 0);
            float s = TrainingTestSplit;
            float c = InputRecords.Count;
            int splitCount = (int)((1f / s) * c);
            for (int i = 0; i < splitCount; i++)
            {
                TestRecords.Add(InputRecords[i]);
            }
            for (int i = splitCount; i < InputRecords.Count; i++)
            {
                TrainingRecords.Add(InputRecords[i]);
            }
            return StageResult.SUCCESS;
        }

protected virtual StageResult Save(FileInfo file, List<TRecord> records)
        {
            Contract.Requires(records != null);
            if (records.Count == 0)
            {
                Error("0 records found to write to file {0}.", file.FullName);
                return StageResult.INPUT_ERROR;
            }
            StageResult r = StageResult.OUTPUT_ERROR;
            if (!CompressOutputFile)
            {
                using (FileStream fs = new FileStream(file.FullName, FileMode.Create))
                using (StreamWriter sw = new StreamWriter(fs))
                {
                    r = WriteFileStream(this, sw, records, WriterOptions);
                }
            }
            else
            {
                using (FileStream fs = new FileStream(file.FullName, FileMode.Create))
                using (GZipStream gzs = new GZipStream(fs, CompressionMode.Compress))
                using (StreamWriter sw = new StreamWriter(gzs))
                {
                    r = WriteFileStream(this, sw, records, WriterOptions);
                }

            }
            return r;
        }

protected override StageResult Init()
        {
            Contract.Requires(ClreplacedStatisticsFile != null && ClreplacedStatisticsFile == null);
            Contract.Requires(ClreplacedifierResultsFile != null && ClreplacedifierResultsFile == null);
            if (!ClreplacedStatisticsFile.CheckExistsAndReportError(L))
            {
                return StageResult.INPUT_ERROR;
            }
            if (!ClreplacedifierResultsFile.CheckExistsAndReportError(L))
            {
                return StageResult.INPUT_ERROR;
            }
            if (OutputFile.Exists && !OverwriteOutputFile)
            {
                Error("The output file {0} exists but the overwrite option was not specified.", OutputFile.FullName);
                return StageResult.OUTPUT_ERROR;
            }
            else if (OutputFile.Exists)
            {
                Warn("Output file {0} exists and will be overwritten.", OutputFile.FullName);
            }
            return StageResult.SUCCESS;
        }

protected override StageResult Init()
        {
            Contract.Requires(InputFile != null && OutputFile == null);
            if (!InputFile.CheckExistsAndReportError(L))
            {
                return StageResult.INPUT_ERROR;
            }
            if (OutputFile.Exists && !OverwriteOutputFile)
            {
                Error("The output file {0} exists but the overwrite option was not specified.", OutputFile.FullName);
                return StageResult.OUTPUT_ERROR;
            }
            else if (OutputFile.Exists)
            {
                Warn("Output file {0} exists and will be overwritten.", OutputFile.FullName);
            }

            return StageResult.SUCCESS;
        }

protected override StageResult Write()
        {
            Contract.Requires(OutputRecords != null);
            if (OutputRecords.Count == 0)
            {
                Warn("0 records transformed from file {0}. Not writing to output file.", InputFile.FullName);
                return StageResult.SUCCESS;
            }
            JsonSerializer serializer = new JsonSerializer();
            serializer.Formatting = Formatting.Indented;
            if (!CompressOutputFile)
            {
                using (FileStream fs = new FileStream(OutputFile.FullName, FileMode.Create))
                using (StreamWriter sw = new StreamWriter(fs, Encoding.UTF8))
                {
                    serializer.Serialize(sw, OutputRecords);
                }
            }
            else
            {
                using (FileStream fs = new FileStream(OutputFile.FullName, FileMode.Create))
                using (GZipStream gzs = new GZipStream(fs, CompressionMode.Compress))
                using (StreamWriter sw = new StreamWriter(gzs, Encoding.UTF8))
                {
                    serializer.Serialize(sw, OutputRecords);
                }
            }
            if (!CompressOutputFile)
            {
                Info("Wrote {0} output records to {1}.", OutputRecords.Count, OutputFileName);
            }
            else
            {
                Info("Wrote {0} output records to gzip-compressed {1}.", OutputRecords.Count, OutputFileName);
            }
            return StageResult.SUCCESS;
        }

static void Benchmark(Options o)
        {
            Contract.Requires(BenchmarkOptions.ContainsKey("Operation"));
            Contract.Requires(BenchmarkOptions.ContainsKey("Category"));
            Contract.Requires(BenchmarkOptions.ContainsKey("Sizes") || BenchmarkOptions.ContainsKey("Scales"));
            if (o.Int8 && o.Unsigned)
            {
                Benchmark<Byte>();
            }
            else if (o.Int8)
            {
                Benchmark<SByte>();
            }
            else if (o.Int16 && o.Unsigned)
            {
                Benchmark<UInt16>();
            }
            else if (o.Int16)
            {
                Benchmark<Int16>();
            }
            else if (o.Int32 && o.Unsigned)
            {
                Benchmark<UInt32>();
            }
            else if (o.Int32)
            {
                Benchmark<Int32>();
            }
            else if (o.Int64 && o.Unsigned)
            {
                Benchmark<UInt64>();
            }
            else if (o.Int64)
            {
                Benchmark<Int64>();
            }
            else if (o.Float)
            {
                Benchmark<Single>();
            }
            else if (o.Double)
            {
                Benchmark<Double>();
            }
            else if (o.Udt)
            {
                Benchmark<TestUDT>();
            }
            else
            {
                L.Error("You must select a data type to benchmark with: -b, -i, -h, -l, -d, -s, and -u.");
                Exit(ExitResult.INVALID_OPTIONS);
            }
        }

public void AddVertex(T vertex)
        {
            Contract.Requires<ArgumentNullException>(vertex != null);
            
            if(dict.ContainsKey(vertex))
            {
                return;
            }
            dict[vertex] = new HashSet<T>();
        }

public void AddEdge(T vertex1, T vertex2) 
        {
            Contract.Requires<ArgumentNullException>(vertex1 != null);
            Contract.Requires<ArgumentNullException>(vertex2 != null);

            if (!dict.ContainsKey(vertex1))
            {
                dict[vertex1] = new HashSet<T>();
            }
            if (!dict.ContainsKey(vertex2))
            {
                dict[vertex2] = new HashSet<T>();
            }
            dict[vertex1].Add(vertex2);
            dict[vertex2].Add(vertex1);
        }

public bool IsNeighbourOf(T vertex, T neighbour) 
        {
            Contract.Requires<ArgumentNullException>(vertex != null);
            Contract.Requires<ArgumentNullException>(neighbour != null);

            return dict.ContainsKey(vertex) && dict[vertex].Contains(neighbour);
        }

public IList<T> GetNeighbours(T vertex) 
        {
            Contract.Requires<ArgumentNullException>(vertex != null);
            
            return dict.ContainsKey(vertex)? dict[vertex].ToList(): new List<T>();
        }

public static int CommonPrefixLength(this string str1, string str2)
        {
            Contract.Requires<ArgumentNullException>(str2 != null);
            Contract.Ensures(Contract.Result<int>() >= 0);

            int count = 0;
            for (int i = 0; i < str1.Length && i < str2.Length; i++, count++)
            {
                if (str1[i] != str2[i])
                {
                    break;
                }
            }
            return count;
        }

public AuthorizationPolicyBuilder Requirereplacedertion(Func<AuthorizationContext, bool> handler)
        {
            Contract.Requires(handler != null);

            _requirements.Add(new replacedertionRequirement(handler));
            return this;
        }

public AuthorizationPolicyBuilder Requirereplacedertion(Func<AuthorizationContext, Task<bool>> handler)
        {
            Contract.Requires(handler != null);

            _requirements.Add(new replacedertionRequirement(handler));
            return this;
        }

public AuthorizationPolicyBuilder Combine(IAuthorizationPolicy policy)
        {
            Contract.Requires(policy != null);

            AddRequirements(policy.Requirements.ToArray());
            return this;
        }

public AuthorizationPolicyBuilder RequireClaim(string claimType, params string[] requiredValues)
        {
            Contract.Requires(claimType != null);

            return RequireClaim(claimType, (IEnumerable<string>)requiredValues);
        }

public AuthorizationPolicyBuilder RequireClaim(string claimType, IEnumerable<string> requiredValues)
        {
            Contract.Requires(claimType != null);

            _requirements.Add(new ClaimsAuthorizationRequirement(claimType, requiredValues));
            return this;
        }

public AuthorizationPolicyBuilder RequireClaim(string claimType)
        {
            Contract.Requires(claimType != null);

            _requirements.Add(new ClaimsAuthorizationRequirement(claimType, allowedValues: null));
            return this;
        }

public AuthorizationPolicyBuilder RequireRole(params string[] roles)
        {
            Contract.Requires(roles != null);

            return RequireRole((IEnumerable<string>)roles);
        }

public AuthorizationPolicyBuilder RequireRole(IEnumerable<string> roles)
        {
            Contract.Requires(roles != null);

            _requirements.Add(new RolesAuthorizationRequirement(roles));
            return this;
        }

internal void Abort(Exception innerException)
        {
            Contract.Requires(innerException != null);
            _aborted = true;
            _abortException = innerException;
            Dispose();
        }

internal void End()
        {
            int priorState = Interlocked.Exchange(ref _requestState, Completed);

            if (priorState == RequestInProgress)
            {
                // Premature ending, must be an error.
                // If the response has not started yet then we can send an error response before closing it.
                _context.Response.StatusCode = 500;
                _context.Response.ContentLength64 = 0;
                _context.Response.Headers.Clear();
                try
                {
                    _context.Response.Close();
                }
                catch (HttpListenerException)
                {
                    // TODO: Trace
                }
            }
            else if (priorState == ResponseInProgress)
            {
                _context.Response.Abort();
            }
            else
            {
                Contract.Requires(priorState == Completed);

                // Clean up after exceptions in the shutdown process. No-op if Response.Close() succeeded.
                _context.Response.Abort();
            }
        }

public static IDisposable BeginWriteLock(this ReaderWriterLockSlim slimLock)
        {
            Contract.Requires(slimLock != null);
            slimLock.EnterWriteLock();
            return new ActionOnDispose(slimLock.ExitWriteLock);
        }

public static string StringFormat(this string source, params object[] args)
    {
        Contract.Requires(source != null);
        Contract.Requires(args != null);
        return string.Format(source, args);
    }

public static IDisposable BeginReadLock(this ReaderWriterLockSlim slimLock)
        {
            Contract.Requires(slimLock != null);
            slimLock.EnterReadLock();
            return new ActionOnDispose(slimLock.ExitReadLock);
        }

public static bool NextBool(this Random rnd)
    {
        Contract.Requires(rnd != null);
        return rnd.Next() % 2 == 0;
    }

public static float NextFloat(this Random rnd, float min = 0, float max = 1)
    {
        Contract.Requires(rnd != null);
        Contract.Requires(max >= min);
        var delta = max - min;
        return (float)rnd.NextDouble() * delta + min;
    }

public static string StringFormat(this string source, params object[] args)
        {
            Contract.Requires(source != null);
            Contract.Requires(args != null);
            return string.Format(source, args);
        }

public AuthenticationProperties GetAuthenticationProperties(string redirectPath)
        {
            this.logger.LogDebug("Getting Authentication properties with redirectPath={0}", redirectPath);
            Contract.Requires(redirectPath != null);

            AuthenticationProperties authProps = new AuthenticationProperties()
            {
                RedirectUri = redirectPath,
            };

            return authProps;
        }

protected override void OnModelCreating(ModelBuilder modelBuilder)
        {
            modelBuilder.HasDefaultSchema("gateway");

            Contract.Requires(modelBuilder != null);
            modelBuilder.Hreplacedequence<long>(Sequence.PharmanetTrace)
            .StartsAt(1)
            .IncrementsBy(1)
            .HasMin(1)
            .HasMax(999999)
            .IsCyclic(true);

            // Create the unique index for the SHA256 hash
            modelBuilder!.Enreplacedy<FileDownload>()
                    .HasIndex(f => f.Hash)
                    .IsUnique();

            modelBuilder.Enreplacedy<FileDownload>()
                    .HasIndex(i => i.ProgramCode)
                    .IsUnique();

            // Create Foreign keys for Email
            modelBuilder.Enreplacedy<Email>()
                .HasOne<EmailFormatCode>()
                .WithMany()
                .HasPrincipalKey(k => k.FormatCode)
                .HasForeignKey(k => k.FormatCode);

            modelBuilder.Enreplacedy<Email>()
                .HasOne<EmailStatusCode>()
                .WithMany()
                .HasPrincipalKey(k => k.StatusCode)
                .HasForeignKey(k => k.EmailStatusCode);

            var emailFormatCodeConvertor = new ValueConverter<EmailFormat, string>(
                    v => EnumUtility.ToEnumString<EmailFormat>(v, false),
                    v => EnumUtility.ToEnum<EmailFormat>(v, false));

            modelBuilder.Enreplacedy<Email>()
                .Property(e => e.FormatCode)
                .HasConversion(emailFormatCodeConvertor);

            modelBuilder.Enreplacedy<EmailTemplate>()
                .HasOne<EmailFormatCode>()
                .WithMany()
                .HasPrincipalKey(k => k.FormatCode)
                .HasForeignKey(k => k.FormatCode);

            modelBuilder.Enreplacedy<EmailTemplate>()
                .Property(e => e.FormatCode)
                .HasConversion(emailFormatCodeConvertor);

            modelBuilder.Enreplacedy<EmailFormatCode>()
                .Property(e => e.FormatCode)
                .HasConversion(emailFormatCodeConvertor);

            var emailStatusCodeConvertor = new ValueConverter<EmailStatus, string>(
                    v => EnumUtility.ToEnumString<EmailStatus>(v, false),
                    v => EnumUtility.ToEnum<EmailStatus>(v, false));

            modelBuilder.Enreplacedy<Email>()
                .Property(e => e.EmailStatusCode)
                .HasConversion(emailStatusCodeConvertor);

            modelBuilder.Enreplacedy<EmailStatusCode>()
                .Property(e => e.StatusCode)
                .HasConversion(emailStatusCodeConvertor);

            // Create Foreign keys for Audit
            modelBuilder.Enreplacedy<AuditEvent>()
                    .HasOne<ProgramTypeCode>()
                    .WithMany()
                    .HasPrincipalKey(k => k.ProgramCode)
                    .HasForeignKey(k => k.ApplicationType);

            modelBuilder.Enreplacedy<AuditEvent>()
                    .HasOne<AuditTransactionResultCode>()
                    .WithMany()
                    .HasPrincipalKey(k => k.ResultCode)
                    .HasForeignKey(k => k.TransactionResultCode)
                    .OnDelete(DeleteBehavior.Restrict);

            var auditTransactionResultConvertor = new ValueConverter<AuditTransactionResult, string>(
                    v => EnumUtility.ToEnumString<AuditTransactionResult>(v, true),
                    v => EnumUtility.ToEnum<AuditTransactionResult>(v, true));

            modelBuilder.Enreplacedy<AuditEvent>()
                    .Property(e => e.TransactionResultCode)
                    .HasConversion(auditTransactionResultConvertor);

            modelBuilder.Enreplacedy<AuditTransactionResultCode>()
                    .Property(e => e.ResultCode)
                    .HasConversion(auditTransactionResultConvertor);

            // Create Foreign keys for FileDownload
            modelBuilder.Enreplacedy<FileDownload>()
                    .HasOne<ProgramTypeCode>()
                    .WithMany()
                    .HasPrincipalKey(k => k.ProgramCode)
                    .HasForeignKey(k => k.ProgramCode);

            // Create Foreign keys for Legal Agreements
            modelBuilder.Enreplacedy<LegalAgreement>()
                .HasOne<LegalAgreementTypeCode>()
                .WithMany()
                .HasPrincipalKey(k => k.LegalAgreementCode)
                .HasForeignKey(k => k.LegalAgreementCode)
                .OnDelete(DeleteBehavior.Restrict);

            // Create Foreign keys for Application Settings
            modelBuilder.Enreplacedy<ApplicationSetting>()
                    .HasOne<ProgramTypeCode>()
                    .WithMany()
                    .HasPrincipalKey(k => k.ProgramCode)
                    .HasForeignKey(k => k.Application)
                    .OnDelete(DeleteBehavior.Restrict);

            // Create unique index for app/component/key on app settings
            modelBuilder.Enreplacedy<ApplicationSetting>()
                    .HasIndex(i => new { i.Application, i.Component, i.Key })
                    .IsUnique();

            // Create Composite Key for User Notes
            modelBuilder.Enreplacedy<Note>().HasKey(k => new { k.Id, k.HdId });

            // Create Foreign keys for User Notes
            modelBuilder.Enreplacedy<Note>()
                    .HasOne<UserProfile>()
                    .WithMany()
                    .HasPrincipalKey(k => k.HdId)
                    .HasForeignKey(k => k.HdId);

            // Create Foreign keys for Messaging Verifications
            modelBuilder.Enreplacedy<MessagingVerification>()
                .HasOne<MessagingVerificationTypeCode>()
                .WithMany()
                .HasPrincipalKey(k => k.MessagingVerificationCode)
                .HasForeignKey(k => k.VerificationType)
                .OnDelete(DeleteBehavior.Restrict);

            modelBuilder.Enreplacedy<UserPreference>()
                .HasKey(c => new { c.HdId, c.Preference });

            modelBuilder.Enreplacedy<Communication>()
                .HasOne<CommunicationTypeCode>()
                .WithMany()
                .HasPrincipalKey(k => k.StatusCode)
                .HasForeignKey(k => k.CommunicationTypeCode);

            modelBuilder.Enreplacedy<Communication>()
                .HasOne<CommunicationStatusCode>()
                .WithMany()
                .HasPrincipalKey(k => k.StatusCode)
                .HasForeignKey(k => k.CommunicationStatusCode);

            var communicationStatusCodeConverter = new ValueConverter<CommunicationStatus, string>(
                    v => EnumUtility.ToEnumString<CommunicationStatus>(v, false),
                    v => EnumUtility.ToEnum<CommunicationStatus>(v, false));

            modelBuilder.Enreplacedy<Communication>()
                .Property(e => e.CommunicationStatusCode)
                .HasConversion(communicationStatusCodeConverter);

            modelBuilder.Enreplacedy<CommunicationStatusCode>()
                .Property(e => e.StatusCode)
                .HasConversion(communicationStatusCodeConverter);

            var communicationTypeCodeConverter = new ValueConverter<CommunicationType, string>(
                    v => EnumUtility.ToEnumString<CommunicationType>(v, false),
                    v => EnumUtility.ToEnum<CommunicationType>(v, false));

            modelBuilder.Enreplacedy<Communication>()
                .Property(e => e.CommunicationTypeCode)
                .HasConversion(communicationTypeCodeConverter);

            modelBuilder.Enreplacedy<CommunicationTypeCode>()
                .Property(e => e.StatusCode)
                .HasConversion(communicationTypeCodeConverter);

            modelBuilder.Enreplacedy<LegalAgreementTypeCode>()
                .Property(e => e.LegalAgreementCode)
                .HasConversion(new ValueConverter<LegalAgreementType, string>(
                    v => EnumUtility.ToEnumString<LegalAgreementType>(v, true),
                    v => EnumUtility.ToEnum<LegalAgreementType>(v, true)));

            // Resource Delegate Models

            // Create ResourceDelegate
            modelBuilder.Enreplacedy<ResourceDelegate>()
                .HasKey(resourceDelegate => new { resourceDelegate.ResourceOwnerHdid, resourceDelegate.ProfileHdid, resourceDelegate.ReasonCode });

            // Create FK keys
            modelBuilder.Enreplacedy<ResourceDelegate>()
                .HasOne<ResourceDelegateReasonCode>()
                .WithMany()
                .HasPrincipalKey(k => k.ReasonTypeCode)
                .HasForeignKey(k => k.ReasonCode);

            modelBuilder.Enreplacedy<ResourceDelegate>()
                    .HasOne<UserProfile>()
                    .WithMany()
                    .HasPrincipalKey(k => k.HdId)
                    .HasForeignKey(k => k.ProfileHdid);

            var resourceDelegateReasonCodeConverter = new ValueConverter<ResourceDelegateReason, string>(
                v => EnumUtility.ToEnumString<ResourceDelegateReason>(v, false),
                v => EnumUtility.ToEnum<ResourceDelegateReason>(v, false));

            modelBuilder.Enreplacedy<ResourceDelegate>()
                .Property(e => e.ReasonCode)
                .HasConversion(resourceDelegateReasonCodeConverter);

            modelBuilder.Enreplacedy<ResourceDelegateReasonCode>()
                .Property(e => e.ReasonTypeCode)
                .HasConversion(resourceDelegateReasonCodeConverter);

            modelBuilder.Enreplacedy<ResourceDelegateHistory>()
                .Property(e => e.ReasonCode)
                .HasConversion(resourceDelegateReasonCodeConverter);

            // Create HDID index on GenericCache
            modelBuilder!.Enreplacedy<GenericCache>()
                    .HasIndex(i => new { i.HdId, i.Domain })
                    .IsUnique();

            // Create Foreign keys for Comment
            modelBuilder.Enreplacedy<Comment>()
                .HasOne<CommentEntryTypeCode>()
                .WithMany()
                .HasPrincipalKey(k => k.CommentEntryCode)
                .HasForeignKey(k => k.EntryTypeCode)
                .OnDelete(DeleteBehavior.Restrict);

            // Create unique keys for UserFeedbackTag
            modelBuilder.Enreplacedy<UserFeedbackTag>()
                .HasIndex(p => new { p.AdminTagId, p.UserFeedbackId })
                .IsUnique(true);

            // Create unique keys for AdminTag
            modelBuilder.Enreplacedy<AdminTag>()
                .HasIndex(p => p.Name)
                .IsUnique(true);

            // Initial seed data
            this.SeedProgramTypes(modelBuilder);
            this.SeedEmail(modelBuilder);
            this.SeedAuditTransactionResults(modelBuilder);
            this.SeedLegalAgreements(modelBuilder);
            this.SeedApplicationSettings(modelBuilder);
            this.SeedMessagingVerifications(modelBuilder);
            this.SeedCommunication(modelBuilder);
            this.SeedResourceDelegateReason(modelBuilder);
            this.SeedCommentEntryTypeCode(modelBuilder);
        }

public static void ScheduleDrugLoadJob<T>(IConfiguration cfg, string key)
            where T : IDrugApp
        {
            Contract.Requires(cfg != null);
            JobConfiguration jc = GetJobConfiguration(cfg!, key);
            ScheduleJob<T>(jc, GetLocalTimeZone(cfg!), j => j.Process(key!));
        }

public static void ScheduleJob<T>(IConfiguration cfg, string key, Expression<Action<T>> methodCall)
        {
            Contract.Requires(cfg != null);
            JobConfiguration jc = GetJobConfiguration(cfg!, key);
            ScheduleJob<T>(jc, GetLocalTimeZone(cfg!), methodCall);
        }

public static void ScheduleJob<T>(JobConfiguration cfg, TimeZoneInfo tz, Expression<Action<T>> methodCall)
        {
            Contract.Requires(cfg != null);
            RecurringJob.AddOrUpdate<T>(cfg!.Id, methodCall, cfg!.Schedule, tz);
            if (cfg.Immediate)
            {
                BackgroundJob.Schedule<T>(methodCall, TimeSpan.FromSeconds(cfg.Delay));
            }
        }

public static Delegate Convert<replacedem>(Func<replacedem, object> func, Type argType, Type resultType)
        {
            // If we need more versions of func then consider using params Type as we can abstract some of the
            // conversion then.

            Contract.Requires(func != null);
            Contract.Requires(resultType != null);

            ParameterExpression? param = Expression.Parameter(argType);
            Expression[]? convertedParam = new Expression[] { Expression.Convert(param, typeof(replacedem)) };

            // This is gnarly... If a func contains a closure, then even though its static, its first
            // param is used to carry the closure, so its as if it is not a static method, so we need
            // to check for that param and call the func with it if it has one...
            Expression call;
            call = Expression.Convert(
                func!.Target == null
                ? Expression.Call(func!.Method, convertedParam)
                : Expression.Call(Expression.Constant(func.Target), func.Method, convertedParam), resultType!);

            Type? delegateType = typeof(Func<,>).MakeGenericType(argType, resultType!);
            return Expression.Lambda(delegateType, call, param).Compile();
        }

static internal Delegate CreateMapDelegate(Type inType, Type outType)
        {
            Contract.Requires(outType != null);
            Contract.Requires(inType != null);
            Contract.Ensures(Contract.Result<Delegate>() != null);

            var result = Mapping.CreateFromSource(inType, outType, inType.Name);
            if (result.Mapped.Count == 0)
                throw new InvalidOperationException("No fields or properties were mapped");
            LambdaExpression lambdaExpression = CreateMappingLambda(inType, outType, result.Mapped);
            return lambdaExpression.Compile();
        }