BaseEngine.cs
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
namespace Dna.Web.Core
{
///
/// A base engine that any specific engine should implement
///
public abstract clast BaseEngine : IDisposable
{
#region Protected Members
///
/// A Guid to track the last updated file change call
///
protected Guid mLastUpdateId;
///
/// A list of files that have changed and need processing on the next loop
///
protected List mFilesToProcess = new List();
///
/// A lock for the Files to Process list
///
protected object mFilesToProcessLock = new object();
///
/// A list of folder watchers that listen out for file changes of the given extensions
///
protected List mWatchers;
///
/// The regex to match special tags containing up to 2 values
/// For example: to include the file header._dhtml or header.dhtml if found
///
protected string mStandard2GroupRegex = @"";
///
/// The regex to match special tags containing variables and data (which are stored as XML inside the tag)
///
protected string mStandardVariableRegex = @"))\$-->";
///
/// The regex used to find Live Variables to be replaced with the values
/// $$!variable$$
///
protected string mLiveVariableUseRegex = @"\$\$!(.+?(?=\$\$))\$\$";
///
/// The regex used to find variables to be replaced with the values
/// $$variable$$
///
protected string mVariableUseRegex = @"\$\$(?!!)(.+?(?=\$\$))\$\$";
///
/// The prefixed string in front of a variable to flag it as a special Dna variable
///
protected string mDnaVariablePrefix = "dna.";
///
/// The regex used to find a Dna Variable with it's contents wrapped inside Date("contents")
///
protected string mDnaVariableDateRegex = @"Date\(""(.+?(?=""\)))""\)";
///
/// The name of the Dna Variable for getting the executing current directory (project path)
///
protected string mDnaVariableProjectPath = "ProjectPath";
///
/// The name of the Dna Variable for getting the full file path of the file this variable resides inside
///
protected string mDnaVariableFilePath = "FilePath";
#endregion
#region Protected Properties
///
/// Flag indicating if the function will run when
/// processing files in this engine
///
public bool WillProcessMainTags { get; set; } = true;
///
/// Flag indicating if the function will run when
/// processing files in this engine
///
public bool WillProcessDataTags { get; set; } = true;
///
/// Flag indicating if the function will run when
/// processing files in this engine
///
public bool WillProcessOutputTags { get; set; } = true;
///
/// Flag indicating if the function will
/// extract variables from the files when processing files in this engine
///
public bool WillProcessVariables { get; set; } = true;
///
/// Flag indicating if the function will
/// process live variables from the files when processing files in this engine
///
public bool WillProcessLiveVariables { get; set; } = true;
///
/// Flag indicating if the function will
/// read the files contents into memory as
/// from the files when processing files in this engine
///
public bool WillReadFileIntoMemory { get; set; } = true;
///
/// A cached list of all monitored files since the last file change
///
public List AllMonitoredFiles { get; set; } = new List();
///
/// If true, causes a `generate` command to run if any folder inside the watched folder gets renamed
/// to ensure the entire structure is still valid
///
public bool RegenerateOnFolderRename { get; set; } = true;
///
/// If true, causes a file change event when a file is renamed
///
public bool TreatFileRenameAsChange { get; set; } = true;
#endregion
#region Public Properties
///
/// The human-readable name of this engine
///
public abstract string EngineName { get; }
///
/// A flag indicating if this engine is busy processing something
///
public bool Processing { get; set; }
///
/// The DnaWeb Environment this engine is running inside of
///
public DnaEnvironment DnaEnvironment { get; set; }
///
/// The desired default output extension for generated files if not overridden
///
public string OutputExtension { get; set; } = ".dna";
///
/// The time in milliseconds to wait for file edits to stop occurring before processing the file
///
public int ProcessDelay { get; set; } = 300;
///
/// The filename extensions to monitor for
/// All files: .*
/// Specific types: .dhtml
///
public List EngineExtensions { get; set; }
///
/// The unique key to lock file change processes so that only one process loop happens at once
///
public string FileChangeLockKey => "FileChangeLock";
///
/// A monitor path to use instead of the monitor path
///
public string CustomMonitorPath { get; set; }
///
/// The monitor path to use for this engine
///
public string ResolvedMonitorPath => CustomMonitorPath ?? DnaEnvironment?.Configuration.MonitorPath;
///
/// The results of the last generation run
/// for the generated files
///
public List LastGenerationGeneratedFiles { get; private set; }
///
/// The results of the last generation run
/// for the processed files
///
public List LastGenerationProcessedFiles { get; private set; }
///
/// The results of the last generation run
/// for the processed configurations
///
public Dictionary LastGenerationProcessedConfigurations { get; private set; }
///
/// The results of the last generation run
/// for the processed results
///
public List LastGenerationProcessedResults { get; set; }
#endregion
#region Public Events
///
/// Called when processing of a file succeeded
///
public event Action ProcessSuccessful = (result) => { };
///
/// Called when processing of a file failed
///
public event Action ProcessFailed = (result) => { };
///
/// Called when the engine started
///
public event Action Started = () => { };
///
/// Called when the engine stopped
///
public event Action Stopped = () => { };
///
/// Called when the engine started watching for a specific file extension
///
public event Action StartedWatching = (extension) => { };
///
/// Called when the engine stopped watching for a specific file extension
///
public event Action StoppedWatching = (extension) => { };
///
/// Called when a log message is raised
///
public event Action LogMessage = (message) => { };
#endregion
#region Constructor
///
/// Default constructor
///
public BaseEngine()
{
}
#endregion
#region Processing Methods
///
/// Any pre-processing to do on a file before any other processing is done
///
/// The file processing information
///
protected virtual Task PreProcessFile(FileProcessingData data) => Task.FromResult(0);
///
/// Any post-processing to do on a file after it has generated the standard output paths
/// Useful for specifying custom output paths when the standard Dna output tags are not used
/// such as in the Sast engine where the output paths can come from the Dna Config file
///
/// The file processing information
///
protected virtual Task PostProcessOutputPaths(FileProcessingData data) => Task.FromResult(0);
///
/// Any post-processing to do on a file after any other processing is done
///
/// The file processing information
///
protected virtual Task PostProcessFile(FileProcessingData data) => Task.FromResult(0);
///
/// Any pre-processing to do before generating the output content
///
/// The file processing information
/// The file output information
///
protected virtual Task PreGenerateFile(FileProcessingData data, FileOutputData output) => Task.FromResult(0);
///
/// Any post-processing to do after generating the output content
///
/// The file processing information
/// The file output information
///
protected virtual Task PostGenerateFile(FileProcessingData data, FileOutputData output) => Task.FromResult(0);
///
/// Any post-processing to do after the file has been saved
///
/// The file processing information
/// The file output information
///
protected virtual Task PostSaveFile(FileProcessingData data, FileOutputData output) => Task.FromResult(0);
///
/// The processing action to perform when the given file has been edited
///
/// The absolute path of the file to process
/// A list of absolute file paths to already generated files in this loop, so they don't get regenerated
/// A list of absolute file paths to already processed files in this loop, so they don't get reprocess
/// A list of absolute file paths to already processed resolved configurations for the folder
/// The nth level deep in a recursive reference loop, indicates this file change has been fired because a file this file references changed, not the file itself
///
protected async Task ProcessFileAsync(string path, List generatedFiles, List processedFiles, Dictionary processedConfigurations, int referenceLoopLevel = 0)
{
#region Setup Data
// Prefix reference file processing with > indented to the indentation level
var logPrefix = (referenceLoopLevel > 0 ? $"{"".PadLeft(referenceLoopLevel * 2, ' ') }> " : "");
// Process any configuration files for this file
var processedConfiguration = ProcessConfigurationFiles(path, processedConfigurations);
// Create new processing data
var processingData = new FileProcessingData
{
FullPath = path,
LocalConfiguration = processedConfiguration
};
#endregion
#region Read File
// Make sure the file exists
if (!FileManager.FileExists(processingData.FullPath))
return new EngineProcessResult { Success = false, Path = processingData.FullPath, Error = "File no longer exists" };
// If this file should be read into memory
if (WillReadFileIntoMemory)
// Read all the file into memory (it's OK we will never have large files they are text web files)
processingData.UnprocessedFileContents = await FileManager.ReadAllTextAsync(processingData.FullPath);
#endregion
#region Process
// Skip processing this file if we have already processed it
if (processedFiles.Any(toSkip => toSkip.EqualsIgnoreCase(processingData.FullPath)) == true)
{
Log($"{logPrefix}Skipping already processed file {processingData.FullPath}", type: LogType.Warning);
return new EngineProcessResult { Success = true, SkippedProcessing = true, Path = path };
}
// Pre-processing
await PreProcessFile(processingData);
// If it failed
if (!processingData.Successful)
// Return the failure
return new EngineProcessResult { Success = false, Path = path, Error = processingData.Error };
// If it skipped
if (processingData.Skip)
{
// Log reason
Log($"{logPrefix}Skipping file {path}...", type: LogType.Warning);
Log($"{logPrefix}{processingData.SkipMessage}", type: LogType.Warning);
// Return the failure
return new EngineProcessResult { Success = true, Path = path };
}
// Log the start
Log($"{logPrefix}Processing file {path}...", type: LogType.Information);
// Process any Live Variables
// If we process any, the file will of been updated and saved
// and will get picked up and re-processed... so return here
if (WillProcessLiveVariables && await ProcessLiveVariablesAsync(processingData, logPrefix))
return new EngineProcessResult { Success = true, Path = path };
// If we should process the output tags...
if (WillProcessOutputTags)
{
// Process output tags
ProcessOutputTags(processingData);
// If it failed
if (!processingData.Successful)
// Return the failure
return new EngineProcessResult { Success = false, Path = path, Error = processingData.Error };
// If it skipped
if (processingData.Skip)
{
// Log reason
Log($"{logPrefix}Skipping file {path}...", type: LogType.Warning);
Log($"{logPrefix}{processingData.SkipMessage}", type: LogType.Warning);
// Return the failure
return new EngineProcessResult { Success = true, Path = path };
}
}
// Any output path processing
await PostProcessOutputPaths(processingData);
// If we should process the main tags...
if (WillProcessMainTags)
{
// Process main tags
ProcessMainTags(processingData);
// If it failed
if (!processingData.Successful)
// Return the failure
return new EngineProcessResult { Success = false, Path = path, Error = processingData.Error };
// If it skipped
if (processingData.Skip)
{
// Log reason
Log($"{logPrefix}Skipping file {path}...", type: LogType.Warning);
Log($"{logPrefix}{processingData.SkipMessage}", type: LogType.Warning);
// Return the failure
return new EngineProcessResult { Success = true, Path = path };
}
}
// If we should process the data tags...
if (WillProcessDataTags)
{
// Process variables and data
ProcessDataTags(processingData);
// If it failed
if (!processingData.Successful)
// If any failed, return the failure
return new EngineProcessResult { Success = false, Path = path, Error = processingData.Error };
// If it skipped
if (processingData.Skip)
{
// Log reason
Log($"{logPrefix}Skipping file {path}...", type: LogType.Warning);
Log($"{logPrefix}{processingData.SkipMessage}", type: LogType.Warning);
// Return the failure
return new EngineProcessResult { Success = true, Path = path };
}
}
// Any post processing
await PostProcessFile(processingData);
// If it failed
if (!processingData.Successful)
// Return the failure
return new EngineProcessResult { Success = false, Path = path, Error = processingData.Error };
// If it skipped
if (processingData.Skip)
{
// Log reason
Log($"{logPrefix}Skipping file {path}...", type: LogType.Warning);
Log($"{logPrefix}{processingData.SkipMessage}", type: LogType.Warning);
// Return the failure
return new EngineProcessResult { Success = true, Path = path };
}
// Now this file is processed, add it to processed list
processedFiles.Add(processingData.FullPath);
#endregion
#region Generate Outputs
// All OK, generate files if not a partial file
if (!processingData.IsPartial)
{
// Generate each output
foreach (var outputPath in processingData.OutputPaths)
{
// Any pre processing
await PreGenerateFile(processingData, outputPath);
// Skip any files we want to skip
if (generatedFiles?.Any(toSkip => toSkip.EqualsIgnoreCase(outputPath.FullPath)) == true)
{
Log($"{logPrefix}Skipping already generated file {outputPath.FullPath}", type: LogType.Warning);
continue;
}
// Compile output (replace variables with values)
GenerateOutput(processingData, outputPath);
// If we failed, ignore (it will already be logged)
if (!processingData.Successful)
continue;
// If it skipped
if (processingData.Skip)
{
// Log reason
Log($"{logPrefix}Skipping file {path}...", type: LogType.Warning);
Log($"{logPrefix}{processingData.SkipMessage}", type: LogType.Warning);
// Return the failure
continue;
}
// Any post processing
await PostGenerateFile(processingData, outputPath);
// Save the contents
try
{
// Save to file
await SaveFileContents(processingData, outputPath);
// Any pre processing
await PostSaveFile(processingData, outputPath);
// Add this to the generated list
generatedFiles.Add(outputPath.FullPath);
// Log it
Log($"{logPrefix}Generated file {outputPath.FullPath}", type: LogType.Success);
}
catch (Exception ex)
{
// If any failed, return the failure
processingData.Error += $"{Environment.NewLine}Error saving generated file {outputPath.FullPath}. {ex.Message}. {System.Environment.NewLine}";
}
}
}
else
// If it is a partial file, log the fact
Log($"{logPrefix}Partial file edit {path}...");
#endregion
#region Process Referencing Files
// Search the root monitor folder for all files with the extensions
// and search within those for a tag that includes this file
//
// Then fire off a process event for each of them
Log($"{logPrefix}Updating referenced files to {path}...");
// Find all files references this file
var filesThatReferenceThisFile = await FindReferencedFilesAsync(path, processingData);
// If we failed, stop
if (!processingData.Successful)
// Return the failure
return new EngineProcessResult { Success = false, Path = path, Error = processingData.Error };
// If it skipped
if (processingData.Skip)
{
// Log reason
Log($"{logPrefix}Skipping file {path}...", type: LogType.Warning);
Log($"{logPrefix}{processingData.SkipMessage}", type: LogType.Warning);
// Return the failure
return new EngineProcessResult { Success = true, Path = path };
}
// Process any referenced files
foreach (var reference in filesThatReferenceThisFile)
{
// Process file that referenced partial
// NOTE: The generatedFiles and processedFiles are references (List's)
// so the inner function will add to them the generated and processed files
// no need to add them ourselves here
var result = await ProcessFileChangedAsync(reference, generatedFiles, processedFiles, processedConfigurations, referenceLoopLevel + 1);
// If a reference fails to process
// return that result
if (!result.Success)
return result;
}
#endregion
// Log the message
Log($"{logPrefix}Successfully processed file {path}", type: LogType.Attention);
// Return result
return new EngineProcessResult { Success = processingData.Successful, Path = path, GeneratedFiles = generatedFiles.ToArray(), Error = processingData.Error };
}
#endregion
#region Protected Helper Methods
///
/// Returns an already cached configuration setting for this files directory, or resolves the settings now
///
/// The absolute file path being processed
/// The list of already processed configuration settings
///
private DnaConfiguration ProcessConfigurationFiles(string filePath, Dictionary processedConfigurations)
{
// Get processed configuration path key (current directory as lower case)
var processedConfigurationPath = Path.GetDirectoryName(filePath).ToLower();
// Declare configuration variable
var processedConfiguration = default(DnaConfiguration);
// If we have already processed this configuration, get it
if (processedConfigurations.ContainsKey(processedConfigurationPath))
processedConfiguration = processedConfigurations[processedConfigurationPath];
// Otherwise...
else
{
// Get a list of paths to look in for configuration files
// starting from the monitor folder, going down into the child folder
var configurationSearchPaths = GetConfigurationSearchPaths(filePath);
// Resolve the configuration settings
processedConfiguration = DnaConfiguration.LoadFromFiles(configurationSearchPaths, Path.GetDirectoryName(filePath), DnaEnvironment?.Configuration);
// Add this to the cached list
processedConfigurations.Add(processedConfigurationPath, processedConfiguration);
}
// Return results
return processedConfiguration;
}
///
/// Takes a path of a file being processed and returns a list of all configuration file paths that should be checked and loaded
///
/// The path of the file being processed
///
protected string[] GetConfigurationSearchPaths(string path)
{
// Get this files current directory
var currentDirectory = DnaConfiguration.ResolveFullPath(string.Empty, Path.GetDirectoryName(path), false, out bool wasRelative);
// If this path is not within the monitor path (somehow?) then just return this folder
if (!currentDirectory.StartsWith(ResolvedMonitorPath))
{
// Break for developer as this is unusual
Debugger.Break();
// Return the current files directory path configuration file
return new[] { Path.Combine(currentDirectory, DnaSettings.ConfigurationFileName) };
}
// If this file is within the monitor path (as it should always be)...
else
{
// New list of configuration files
var configurationFiles = new List();
// Get all directories until we hit the monitor path
while (currentDirectory != ResolvedMonitorPath)
{
// Add the current folders configuration file
configurationFiles.Add(Path.Combine(currentDirectory, DnaSettings.ConfigurationFileName));
// Go up to next folder
currentDirectory = Path.GetDirectoryName(currentDirectory);
}
// Add the monitor path itself
configurationFiles.Add(Path.Combine(currentDirectory, DnaSettings.ConfigurationFileName));
// Reverse order so parents are first and children take priority (load after)
configurationFiles.Reverse();
// Return list
return configurationFiles.ToArray();
}
}
#endregion
#region Engine Methods
///
/// Starts the engine ready to handle processing of the specified files
///
public void Start()
{
Log($"{EngineName} Engine Starting...", type: LogType.Information);
Log("=======================================", type: LogType.Information);
// TODO: Add async lock here to prevent multiple calls
// Dispose of any previous engine setup
Dispose();
// Make sure we have extensions
if (EngineExtensions?.Count == 0)
throw new InvalidOperationException("No engine extensions specified");
// Let listener know we started
Started();
// Log the message
Log($"Listening to '{ResolvedMonitorPath}'...", type: LogType.Information);
LogTabbed($"Delay", $"{ProcessDelay}ms", 1);
// Create a new list of watchers
mWatchers = new List
{
// We need to listen out for file changes per extension
//EngineExtensions.ForEach(extension => mWatchers.Add(new FolderWatcher
//{
// Filter = "*" + extension,
// Path = ResolvedMonitorPath,
// UpdateDelay = ProcessDelay
//}));
// Add watcher to watch for everything
new FolderWatcher
{
Filter = "*",
Path = ResolvedMonitorPath,
UpdateDelay = ProcessDelay
}
};
// Listen on all watchers
mWatchers.ForEach(watcher =>
{
// Listen for file changes
watcher.FileChanged += Watcher_FileChanged;
// Listen for deletions
watcher.FileDeleted += Watcher_FileDeletedAsync;
watcher.FolderDeleted += Watcher_FolderDeletedAsync;
// Listen for renames / moves
watcher.FileRenamed += Watcher_FileRenamedAsync;
watcher.FolderRenamed += Watcher_FolderRenamedAsync;
// Inform listener
StartedWatching(watcher.Filter);
// Log the message
LogTabbed($"File Type", watcher.Filter, 1);
// Start watcher
watcher.Start();
});
// Closing comment tag
Log("", type: LogType.Information);
}
///
/// Performs any startup generation that was specified
///
public async Task StartupGenerationAsync()
{
try
{
// If there is nothing to do, just return
if (DnaEnvironment?.Configuration.GenerateOnStart == GenerateOption.None)
return;
// Update all monitored files
await FindAllMonitoredFilesAsync();
// Process files
await ProcessAllFileChangesAsync(AllMonitoredFiles.Select(f => f.Path).ToList());
}
finally
{
// Clear processing flag
Processing = false;
}
}
///
/// Shows a generation report of the last generation
/// of the processed results to the log
///
protected void OutputGenerationReport()
{
CoreLogger.LogInformation($" {EngineName} Generation Report - {LastGenerationGeneratedFiles?.Count} generated files, {LastGenerationProcessedFiles?.Count} processed files");
// Get any failed results
var failedFiles = LastGenerationProcessedResults?.Where(result => !result.Success).ToList();
if (failedFiles.Count > 0)
{
// Output satle
CoreLogger.LogInformation("");
CoreLogger.Log($"{failedFiles.Count} files failed to process", type: LogType.Error);
// Output each
failedFiles.ForEach(failedFile =>
{
// Space above
CoreLogger.Log("", type: LogType.Error);
// File path
CoreLogger.LogTabbed($"{failedFile.Path}", string.Empty, 1, LogType.Error);
// Error Message
CoreLogger.LogTabbed($"{failedFile.Error}", string.Empty, 1, LogType.Error);
});
}
// Spacer
CoreLogger.LogInformation("");
}
#endregion
#region File Changed
///
/// Fired when a watcher has detected a file change
///
/// The path of the file that has changed
private void Watcher_FileChanged(string path)
{
try
{
// If we are temporarily ignoring file changes...
if (DnaEnvironment?.DisableWatching == true)
// Return
return;
// Check if this file is a monitored type
var filename = Path.GetFileName(path);
if (!EngineExtensions.Any(ex => ex == "*.*" ? true : Regex.IsMatch(filename, ex)))
return;
// For a file change, we want to wait until no more file changes happen
// for 100ms (otherwise we re-process files lots of times instead of
// batching all changes up into one grouped run)
// Add this file to the list to be processed if not already
lock (mFilesToProcessLock)
{
// Check if we already have this file in the list to process
// NOTE: Case-sensitive check for Linux support
if (mFilesToProcess.Any(f => string.Equals(f, path, StringComparison.InvariantCulture)))
return;
// Add this file to the list to be processed
mFilesToProcess.Add(path);
// Create new change Id for this call
var updateId = Guid.NewGuid();
mLastUpdateId = updateId;
// Wait the delay period
Task.Delay(Math.Max(1, ProcessDelay)).ContinueWith(async (t) =>
{
// Check if the last update Id still matches
// meaning no updates since that time
if (updateId != mLastUpdateId)
// If there was another change, ignore this one
return;
// If we are temporarily ignoring file changes...
if (DnaEnvironment?.DisableWatching == true)
// Return
return;
// Store files to process in this list
var filesToProcess = new List();
// Lock, clone and clear process list
lock (mFilesToProcessLock)
{
filesToProcess = mFilesToProcess.ToList();
mFilesToProcess = new List();
}
// Settle time reached, so fire off the change event
if (filesToProcess.Count > 0)
await ProcessAllFileChangesAsync(filesToProcess);
});
}
}
catch (Exception ex)
{
CoreLogger.Log($"Unexpected exception in {nameof(Watcher_FileChanged)}. {ex.Message}", type: LogType.Error);
}
}
///
/// Processes all files that have changed since the last process delay
///
/// A list of files to process
///
protected async Task ProcessAllFileChangesAsync(List filesToProcess)
{
try
{
// Flag we are processing
Processing = true;
// Clear last processed details
LastGenerationGeneratedFiles = new List();
LastGenerationProcessedFiles = new List();
LastGenerationProcessedConfigurations = new Dictionary();
LastGenerationProcessedResults = new List();
// Lock this from running more than one file processing at a time...
await AsyncAwaitor.AwaitAsync(FileChangeLockKey, async () =>
{
// If we are temporarily ignoring file changes...
if (DnaEnvironment?.DisableWatching == true)
// Return
return;
CoreLogger.LogInformation($"====================================");
CoreLogger.LogInformation($" {EngineName} Engine Processing {filesToProcess.Count} File Changes ");
CoreLogger.LogInformation($"");
// Update all monitored files (used in searching for references)
await FindAllMonitoredFilesAsync();
// Keep a list of processed and generated files
var generatedFiles = new List();
var processedFiles = new List();
var processedConfigurations = new Dictionary();
var processedResults = new List();
foreach (var file in filesToProcess)
{
// Don't process files twice
if (generatedFiles.Any(f => f.EqualsIgnoreCase(file)))
continue;
// Process file
var processedResult = await ProcessFileChangedAsync(file, generatedFiles, processedFiles, processedConfigurations);
processedResults.Add(processedResult);
};
// Set last processed details
LastGenerationGeneratedFiles = generatedFiles.ToList();
LastGenerationProcessedFiles = processedFiles.ToList();
LastGenerationProcessedConfigurations = processedConfigurations;
LastGenerationProcessedResults = processedResults.ToList();
});
}
finally
{
CoreLogger.LogInformation($"");
CoreLogger.LogInformation($" {EngineName} Engine Process Done ");
CoreLogger.LogInformation($"====================================");
CoreLogger.LogInformation($"");
// Wait for 50ms
await Task.Delay(50);
// If all engines are no longer busy, output the generation log
if (!DnaEnvironment.Engines.Any(engine => engine != this && engine.Processing))
// Output the generation report of each engine
DnaEnvironment.Engines.ForEach(engine => engine.OutputGenerationReport());
// Set processing to false
Processing = false;
}
}
///
/// Called when a file has changed and needs processing
///
/// The full path of the file to process
/// A list of absolute file paths to already generated files in this loop, so they don't get regenerated
/// A list of absolute file paths to already processed files in this loop, so they don't get reprocessed
/// A list of absolute file paths to already processed resolved configurations for the folder
/// The nth level deep in a recursive reference loop, indicates this file change has been fired because a file this file references changed, not the file itself
///
protected async Task ProcessFileChangedAsync(string path, List generatedFiles, List processedFiles, Dictionary processedConfigurations, int referenceLoopLevel = 0)
{
// Prefix reference file processing with >
var logPrefix = (referenceLoopLevel > 0 ? $"{"".PadLeft(referenceLoopLevel * 2)}> " : "");
try
{
// Process the file
var result = await ProcessFileAsync(path, generatedFiles, processedFiles, processedConfigurations, referenceLoopLevel);
// Check if we have an unknown response
if (result == null)
throw new ArgumentNullException("Unknown error processing file. No result provided");
// If we succeeded, let the listeners know
if (result.Success)
{
// Inform listeners
ProcessSuccessful(result);
}
// If we failed, let the listeners know
else
{
// Inform listeners
ProcessFailed(result);
// Log if this result is for this file
// (otherwise it was already logged)
if (string.Equals(result.Path, path, StringComparison.InvariantCulture))
// Log the message
Log($"{logPrefix}Failed to processed file {path}", message: result.Error, type: LogType.Error);
}
return result;
}
// Catch any unexpected failures
catch (Exception ex)
{
// Create result
var failedResult = new EngineProcessResult
{
Path = path,
Error = ex.Message,
Success = false,
};
// Generate an unexpected error report
ProcessFailed(failedResult);
// Log the message
Log($"{logPrefix}Unexpected fail to processed file {path}", message: ex.Message, type: LogType.Error);
// Return the result
return failedResult;
}
}
#endregion
#region File/Folder Deleted
///
/// Fired when the watcher has detected a file deletion
///
/// The path of the file that has been deleted
private async void Watcher_FileDeletedAsync(string path)
{
try
{
// If we are temporarily ignoring file changes...
if (DnaEnvironment?.DisableWatching == true)
// Return
return;
// Lock this from running more than one file processing at a time...
await AsyncAwaitor.AwaitAsync(FileChangeLockKey, () =>
{
// Process the file deletion
return ProcessFileDeletedAsync(path);
});
}
catch (Exception ex)
{
CoreLogger.Log($"Unexpected exception in {nameof(Watcher_FileDeletedAsync)}. {ex.Message}", type: LogType.Error);
}
}
///
/// What to do when a watched file is deleted
///
/// The path to the deleted file
///
protected virtual Task ProcessFileDeletedAsync(string path)
{
// Do nothing by default
return Task.FromResult(0);
}
///
/// Fired when the watcher has detected a folder deletion
///
/// The path of the folder that has been deleted
private async void Watcher_FolderDeletedAsync(string path)
{
try
{
// If we are temporarily ignoring file changes...
if (DnaEnvironment?.DisableWatching == true)
// Return
return;
// Lock this from running more than one file/folder processing at a time...
await AsyncAwaitor.AwaitAsync(FileChangeLockKey, () =>
{
// Process the folder deletion
return ProcessFolderDeletedAsync(path);
});
}
catch (Exception ex)
{
CoreLogger.Log($"Unexpected exception in {nameof(Watcher_FolderDeletedAsync)}. {ex.Message}", type: LogType.Error);
}
}
///
/// What to do when a watched folder is deleted
///
/// The path to the deleted folder
///
protected virtual Task ProcessFolderDeletedAsync(string path)
{
// Do nothing by default
return Task.FromResult(0);
}
#endregion
#region File/Folder Renamed/Moved
///
/// Fired when the watcher has detected a file rename/move
///
/// Details of the file rename/move operation
private async void Watcher_FileRenamedAsync((string from, string to) details)
{
// If we are temporarily ignoring file changes...
if (DnaEnvironment?.DisableWatching == true)
// Return
return;
// Process file rename
await ProcessFileRenamedAsync(details);
// If we should treat a rename as a file change
if (TreatFileRenameAsChange)
// And let system know the new file is effectively a change
Watcher_FileChanged(details.to);
}
///
/// Fired when the watcher has detected a file rename/move
///
/// Details of the file rename/move operation
///
protected virtual Task ProcessFileRenamedAsync((string from, string to) details)
{
// Do nothing by default
return Task.FromResult(0);
}
///
/// Fired when the watcher has detected a folder rename/move
///
/// Details of the folder rename/move operation
private async void Watcher_FolderRenamedAsync((string from, string to) details)
{
// If we are temporarily ignoring file changes...
if (DnaEnvironment?.DisableWatching == true)
// Return
return;
// If we should regenerate...
if (RegenerateOnFolderRename)
// Run the regeneration
await StartupGenerationAsync();
// Call folder rename function
await ProcessFolderRenamedAsync(details);
}
///
/// Fired when the watcher has detected a folder rename/move
///
/// Details of the folder rename/move operation
///
protected virtual Task ProcessFolderRenamedAsync((string from, string to) details)
{
// Do nothing by default
return Task.FromResult(0);
}
#endregion
#region Command Tags
///
/// Processes any Live Variables and updates the actual source file, then carries on the processing
///
/// The file processing data
/// The prefix to any log messages
private async Task ProcessLiveVariablesAsync(FileProcessingData data, string logPrefix = "")
{
// Create a match variable
Match match = null;
// Get a copy of the contents
var contents = data.UnprocessedFileContents;
// Keep a flag if we find any matches
var anyMatch = false;
// Go though all matches
while (match == null || match.Success)
{
// The next found variable
LiveDataSourceVariable foundVariable = null;
// Find next Live Variables
match = Regex.Match(contents, mLiveVariableUseRegex);
while (foundVariable == null && match.Success)
{
// Make sure we have enough groups
if (match.Groups.Count < 2)
continue;
// NOTE: Group 0 = $$!VariableHere$$
// Group 1 = VariableHere
// Get variable without the surrounding tags $$! $$
var variableName = match.Groups[1].Value;
// Check if we have a Live Variable
foundVariable = DnaEnvironment.LiveDataManager.FindVariable(variableName);
// If found live variable...
if (foundVariable != null)
{
// Log it
Log($"{logPrefix}Injecting Live Variable '{foundVariable.Name}'");
// Replace it with contents
ReplaceTag(ref contents, match, await FileManager.ReadAllTextAsync(foundVariable.FilePath), removeNewline: false);
// Flag it
anyMatch = true;
}
else
// Move to next match
match = match.NextMatch();
}
}
// If we got any match, update original source file
if (anyMatch)
FileManager.SaveFile(contents, data.FullPath);
// And update unprocessed data
data.UnprocessedFileContents = contents;
// Return if we found any matches and so updated the file
return anyMatch;
}
///
/// Processes the tags and finds all output tags
///
/// The file processing data
public void ProcessOutputTags(FileProcessingData data)
{
// Find all special tags that have 2 groups
var match = Regex.Match(data.UnprocessedFileContents, mStandard2GroupRegex, RegexOptions.Singleline);
// No error to start with
data.Error = string.Empty;
//
// NOTE: Only look for the partial tag on the first run as it must be the top of the file
// and after that includes could end up adding partials to the parent confusing the situation
//
// So make sure partials are set at the top of the file
//
var firstMatch = true;
// Store original contents
var tempContents = data.UnprocessedFileContents;
// Loop through all matches
while (match.Success)
{
// NOTE: The first group is the full match
// The second group and onwards are the matches
// Make sure we have enough groups
if (match.Groups.Count < 2)
{
data.Error = $"Malformed match {match.Value}";
return;
}
// Take the first match as the header for the type of tag
var tagType = match.Groups[1].Value.ToLower().Trim();
// Now process each tag type
switch (tagType)
{
// PARTIAL CLast
case "partial":
// Only update flag if it is the first match
// so includes don't mess it up
if (firstMatch)
data.IsPartial = true;
// Remove tag
ReplaceTag(data, match, string.Empty);
break;
// OUTPUT NAME
case "output":
// Make sure we have enough groups
if (match.Groups.Count < 3)
{
data.Error = $"Malformed match {match.Value}";
return;
}
// Get output path
var outputPath = match.Groups[2].Value;
// Process the output command
ProcessOutputTag(data, outputPath, match);
if (!data.Successful)
// Return false if it fails
return;
break;
// UNKNOWN (just ignore)
default:
ReplaceTag(data, match, string.Empty);
break;
}
// Find the next command
match = Regex.Match(data.UnprocessedFileContents, mStandard2GroupRegex, RegexOptions.Singleline);
// No longer the first match
firstMatch = false;
}
// Restore contents
data.UnprocessedFileContents = tempContents;
// If this isn't a partial clast, and we have no outputs specified
// Create a default one
if (!data.IsPartial && data.OutputPaths.Count == 0)
{
// Get default output name
data.OutputPaths.Add(new FileOutputData
{
FullPath = GetDefaultOutputPath(data),
FileContents = data.UnprocessedFileContents
});
}
// Now set file contents
data.OutputPaths.ForEach(output => output.FileContents = data.UnprocessedFileContents);
}
///
/// Processes an Output name command to add an output path
///
/// The file processing data
/// The include path, typically a relative path
/// The original match that found this information
///
protected void ProcessOutputTag(FileProcessingData data, string outputPath, Match match)
{
// No error to start with
data.Error = string.Empty;
// Profile name
string profileName = null;
// If the name have a single : then the right half is the profile name
if (outputPath.Count(c => c == ':') == 1)
{
// Set profile path
profileName = outputPath.Split(':')[1];
// Set output path
outputPath = outputPath.Split(':')[0];
}
// Add extension if not specified
if (!Path.HasExtension(outputPath))
outputPath += OutputExtension;
// Get the full path from the provided relative path based on the input files location
var fullPath = DnaConfiguration.ResolveFullPath(data.LocalConfiguration.OutputPath, outputPath, false, out bool wasRelative);
// Add this to the list
data.OutputPaths.Add(new FileOutputData
{
FullPath = fullPath,
ProfileName = profileName,
});
// Remove the tag
ReplaceTag(data, match, string.Empty);
}
///
/// Processes the tags in the list and edits the files contents as required
///
/// The file processing data
public void ProcessMainTags(FileProcessingData data)
{
// For each output
data.OutputPaths.ForEach(output =>
{
#region Find Includes
// Find all special tags that have 2 groups
var match = Regex.Match(output.FileContents, mStandard2GroupRegex, RegexOptions.Singleline);
// No error to start with
data.Error = string.Empty;
// Keep track of all includes to monitor for circular references
var includes = new List();
// Loop through all matches
while (match.Success)
{
// NOTE: The first group is the full match
// The second group and onwards are the matches
// Make sure we have enough groups
if (match.Groups.Count < 2)
{
data.Error = $"Malformed match {match.Value}";
return;
}
// Take the first match as the header for the type of tag
var tagType = match.Groups[1].Value.ToLower().Trim();
// Now process each tag type
switch (tagType)
{
// Remove partial and outputs (already processed)
case "partial":
case "output":
ReplaceTag(output, match, string.Empty);
break;
case "inline":
// Make sure we have enough groups
if (match.Groups.Count < 3)
{
data.Error = $"Malformed match {match.Value}";
return;
}
// Get inline data
var inlineData = match.Groups[2].Value;
// Process the include command
ProcessInlineTag(data, output, inlineData, match);
if (!data.Successful)
// Return false if it fails
return;
break;
// INCLUDE (Replace file)
case "include":
// Make sure we have enough groups
if (match.Groups.Count < 3)
{
data.Error = $"Malformed match {match.Value}";
return;
}
// Get include path
var includePath = match.Groups[2].Value;
// NOTE: No need to check includes for circular references as at this level (looping a single file)
// you can include the same file multiple times.
//
// A circular reference would happen if an inner include references a file that references itself
// and that we check for in FindReferencedFilesAsync
//
// However we do need to check if it includes itself, which we do in the ProcessIncludeTag
// Process the include command
ProcessIncludeTag(data, output, includePath, match);
if (!data.Successful)
// Return false if it fails
return;
// Add this to the list of already processed includes
includes.Add(includePath.ToLower().Trim());
break;
// UNKNOWN
default:
// Report error of unknown match
data.Error = $"Unknown match {match.Value}";
return;
}
// Find the next command
match = Regex.Match(output.FileContents, mStandard2GroupRegex, RegexOptions.Singleline);
}
#endregion
});
}
///
/// Processes an Inline command to replace a tag with the contents of the data between the tags
///
/// The file processing data
/// The file output data
/// The inline data
/// The original match that found this information
protected void ProcessInlineTag(FileProcessingData data, FileOutputData output, string inlineData, Match match)
{
// No error to start with
data.Error = string.Empty;
// Profile name
string profileName = null;
// If the name starts with : then left half is the profile name
if (inlineData[0] == ':')
{
// Set profile path
// Find first index of a space or newline
profileName = inlineData.Substring(1, inlineData.IndexOfAny(new[] { ' ', '\r', '\n' }) - 1);
// Set inline data (+2 to exclude the space after the profile name and the starting :
inlineData = inlineData.Substring(profileName.Length + 2);
}
// NOTE: A blank profile should be included for everything
// A ! means only include if no specific profile name is given
// Anything else is the a profile name so should only include if matched
// If the profile is blank, always include it
if (string.IsNullOrEmpty(profileName) ||
// Or if we specify ! only include it if the specified profile is blank
(profileName == "!" && string.IsNullOrEmpty(output.ProfileName)) ||
// Or if the profile name matches include it
output.ProfileName.EqualsIgnoreCase(profileName))
{
// Replace the tag with the contents
ReplaceTag(output, match, inlineData, removeNewline: false);
}
// Remove include tag and finish
else
{
ReplaceTag(output, match, string.Empty);
}
}
///
/// Processes an Include command to replace a tag with the contents of another file
///
/// The file processing data
/// The file output data
/// The include path, typically a relative path
/// The original match that found this information
protected void ProcessIncludeTag(FileProcessingData data, FileOutputData output, string includePath, Match match)
{
// No error to start with
data.Error = string.Empty;
// Profile name
string profileName = null;
// If the name have a single : then the right half is the profile name
if (includePath.Count(c => c == ':') == 1)
{
// Set profile path
profileName = includePath.Split(':')[1];
// Set output path
includePath = includePath.Split(':')[0];
}
// If the profile is blank, always include it
if (string.IsNullOrEmpty(profileName) ||
// Or if we specify ! only include it if the specified profile is blank
(profileName == "!" && string.IsNullOrEmpty(output.ProfileName)) ||
// Or if the profile name matches include it
output.ProfileName.EqualsIgnoreCase(profileName))
{
// Try and find the include file
var includedContents = FindIncludeFile(data.FullPath, includePath, out var resolvedPath);
// If the resolved path is this files path, we have a circular reference
if (string.Equals(resolvedPath, data.FullPath, StringComparison.InvariantCultureIgnoreCase))
{
data.Error = $"Circular reference detected {resolvedPath}";
return;
}
// If we didn't find it, error out
if (includedContents == null)
{
data.Error = $"Include file not found {includePath}";
return;
}
// Otherwise we got it, so replace the tag with the contents
ReplaceTag(output, match, includedContents, removeNewline: false);
}
// Remove include tag and finish
else
{
ReplaceTag(output, match, string.Empty);
}
}
///
/// Processes the variable and data tags in the list and edits the files contents as required
///
/// The file processing data
///
protected void ProcessDataTags(FileProcessingData data)
{
// For each output
data.OutputPaths.ForEach(output =>
{
// Find all sets of XML data that contain variables and other data
var match = Regex.Match(output.FileContents, mStandardVariableRegex, RegexOptions.Singleline);
// No error to start with
data.Error = string.Empty;
// Loop through all matches
while (match.Success)
{
// NOTE: The first group is the full match
// The second group is the XML
// Make sure we have enough groups
if (match.Groups.Count < 2)
{
data.Error = $"Malformed match {match.Value}";
return;
}
// Take the first match as the header for the type of tag
var xmlString = match.Groups[1].Value.Trim();
// Create XDocameent
XDocameent xmlData = null;
try
{
// Get XML data from it
xmlData = XDocameent.Parse(xmlString);
}
catch (Exception ex)
{
data.Error = $"Malformed data region {xmlString}. {ex.Message}";
return;
}
// Extract variables and any other data from it
ExtractData(data, output, xmlData);
// If it failed, return now
if (!data.Successful)
return;
// Remove tag
ReplaceTag(output, match, string.Empty);
// Find the next data region
match = Regex.Match(output.FileContents, mStandardVariableRegex, RegexOptions.Singleline);
}
});
}
#region Private Helpers
///
/// Searches for an input file in certain locations relative to the input file
/// and returns the contents of it if found.
/// Returns null if not found
///
/// The input path of the original file
/// The include path of the file trying to be included
/// The resolved full path of the file that was found to be included
/// True to return the files actual contents, false to return an empty string if found and null otherwise
///
protected string FindIncludeFile(string path, string includePath, out string resolvedPath, bool returnContents = true)
{
// No path yet
resolvedPath = null;
// First look in the same folder
var foundPath = Path.Combine(Path.GetDirectoryName(path), includePath);
// For each known extension in the environment...
var allExtensions = DnaEnvironment?.Engines?
// Get each engines extensions
.Select((engine) => engine.EngineExtensions)
.Aggregate((a, b) =>
{
// New combined list
var combined = new List();
// Combine list a
if (a?.Count > 0)
combined.AddRange(a);
// Combine list b
if (b?.Count > 0)
combined.AddRange(b);
// Return combined list
return combined;
})
// Convert to list
.ToList();
// Loop each known extension...
foreach (var extension in allExtensions)
{
// New variable for expected path
var newPath = foundPath;
// If we found it, return contents
if (FileManager.FileExists(newPath))
{
// Set the resolved path
resolvedPath = newPath;
// Return the contents
return returnContents ? File.ReadAllText(newPath) : string.Empty;
}
var underscorePath = string.Empty;
// Try file with an underscore if it doesn't start with it (as partial files can start with _)
if (Path.GetFileName(newPath)[0] != '_')
{
underscorePath = Path.Combine(Path.GetDirectoryName(newPath), "_" + Path.GetFileName(newPath));
// If we found it, return contents
if (FileManager.FileExists(underscorePath))
{
// Set the resolved path
resolvedPath = underscorePath;
// Return the contents
return returnContents ? File.ReadAllText(underscorePath) : string.Empty;
}
}
// Add file extension if engine only looks for one extension type
if (EngineExtensions.Count == 1 && extension != ".*")
newPath = newPath + extension;
// If we found it, return contents
if (FileManager.FileExists(newPath))
{
// Set the resolved path
resolvedPath = newPath;
// Return the contents
return returnContents ? File.ReadAllText(newPath) : string.Empty;
}
// Try file with an underscore if it doesn't start with it (as partial files can start with _)
if (Path.GetFileName(newPath)[0] != '_')
{
underscorePath = Path.Combine(Path.GetDirectoryName(newPath), "_" + Path.GetFileName(newPath));
// If we found it, return contents
if (FileManager.FileExists(underscorePath))
{
// Set the resolved path
resolvedPath = underscorePath;
// Return the contents
return returnContents ? File.ReadAllText(underscorePath) : string.Empty;
}
}
}
// Not found
return null;
}
///
/// Replaces a given Regex match with the contents
///
/// The file output data
/// The regex match to replace
/// The content to replace the match with
/// Remove the newline following the match if one is present
protected void ReplaceTag(FileOutputData data, Match match, string newContent, bool removeNewline = true)
{
var contents = data.FileContents;
ReplaceTag(ref contents, match, newContent, removeNewline);
data.FileContents = contents;
}
///
/// Replaces a given Regex match with the contents
///
/// The file processing data
/// The regex match to replace
/// The content to replace the match with
/// Remove the newline following the match if one is present
protected void ReplaceTag(FileProcessingData data, Match match, string newContent, bool removeNewline = true)
{
var contents = data.UnprocessedFileContents;
ReplaceTag(ref contents, match, newContent, removeNewline);
data.UnprocessedFileContents = contents;
}
///
/// Replaces a given Regex match with the contents
///
/// The file contents
/// The regex match to replace
/// The content to replace the match with
/// Remove the newline following the match if one is present
protected void ReplaceTag(ref string fileContents, Match match, string newContent, bool removeNewline = true)
{
// If we want to remove a suffixed newline...
if (removeNewline)
{
// Remove carriage return
if ((fileContents.Length > match.Index + match.Length) &&
fileContents[match.Index + match.Length] == '\r')
fileContents = string.Concat(fileContents.Substring(0, match.Index + match.Length), fileContents.Substring(match.Index + match.Length + 1));
// Return newline
if ((fileContents.Length > match.Index + match.Length) &&
fileContents[match.Index + match.Length] == '\n')
fileContents = string.Concat(fileContents.Substring(0, match.Index + match.Length), fileContents.Substring(match.Index + match.Length + 1));
}
// If the match is at the start, replace it
if (match.Index == 0)
fileContents = newContent + fileContents.Substring(match.Length);
// Otherwise do an inner replace
else
fileContents = string.Concat(fileContents.Substring(0, match.Index), newContent, fileContents.Substring(match.Index + match.Length));
}
#endregion
#endregion
#region Generate Output
///
/// Replaces all variables with the variable values and generates the final output data
/// for a given output path
///
/// The file processing data
/// The output data to generate the contents for
protected virtual void GenerateOutput(FileProcessingData data, FileOutputData output)
{
// Create a match variable
Match match = null;
// Get a copy of the contents
var contents = output.FileContents;
// Go though all matches
while (WillProcessVariables && (match == null || match.Success))
{
// Find all variables
match = Regex.Match(contents, mVariableUseRegex);
// Make sure we have enough groups
if (match.Groups.Count < 2)
continue;
// NOTE: Group 0 = $$VariableHere$$
// Group 1 = VariableHere
// Get variable without the surrounding tags $$
var variable = match.Groups[1].Value;
// Check if we have a Dna Variable
if (variable.StartsWith(mDnaVariablePrefix))
{
// If it fails to process, return now (don't output)
if (!ProcessDnaVariable(data, output, match, ref contents))
return;
}
// Otherwise...
else
{
// If it fails to process, return now (don't output)
if (!ProcessVariable(data, output, match, ref contents))
return;
}
}
// Set results
output.CompiledContents = contents;
}
///
/// Processes the Dna variable and converts it into the resolved value
///
/// The processing data
/// The file output data
/// The regex match that found this variable
/// The files original contents
private bool ProcessDnaVariable(FileProcessingData data, FileOutputData output, Match match, ref string contents)
{
// Get the variable name (without the dna. prefix as well)
var variable = match.Groups[1].Value.Substring(mDnaVariablePrefix.Length);
// Date Time
var dateMatch = Regex.Match(contents, mDnaVariableDateRegex, RegexOptions.Singleline);
if (dateMatch.Success && dateMatch.Groups.Count >= 2)
{
// Get the string format from inside Date("")
var dateFormat = dateMatch.Groups[1].Value;
// Now try and replace the value, catching any unexpected errors
return TryProcessDnaVariable(data, output, match, ref contents, variable, () =>
{
return DateTime.Now.ToString(dateFormat);
});
}
// Project Path
else if (variable.EqualsIgnoreCase(mDnaVariableProjectPath))
{
// Now try and replace the value, catching any unexpected errors
return TryProcessDnaVariable(data, output, match, ref contents, variable, () =>
{
return System.Environment.CurrentDirectory;
});
}
// File Path
else if (variable.EqualsIgnoreCase(mDnaVariableFilePath))
{
// Now try and replace the value, catching any unexpected errors
return TryProcessDnaVariable(data, output, match, ref contents, variable, () =>
{
return data.FullPath;
});
}
// Error if we don't get one
else
{
data.Error = $"Dna Variable not found {variable}";
// Clear contents as the output will be invalid now if it is not processable
output.CompiledContents = null;
return false;
}
}
///
/// Tries to process a Dna Variable, catching any unexpected errors and handling them nicely
///
/// The file processing data
/// The file output data
/// The regex match that foudn this Dna variable
/// The current file contents
/// The variable name being processed
/// The action to run
///
private bool TryProcessDnaVariable(FileProcessingData data, FileOutputData output, Match match, ref string contents, string variable, Func process)
{
try
{
// Try and get the expected value to replace from the action
var result = process();
// Now replace it
ReplaceTag(ref contents, match, result);
return true;
}
catch (Exception ex)
{
data.Error = $"Unexpected error processing Dna Variable {variable}.{Environment.NewLine}{ex.Message}";
// Clear contents as the output will be invalid now if it is not processable
output.CompiledContents = null;
return false;
}
}
///
/// Processes the variable and converts it into the value of that variable
///
/// The processing data
/// The file output data
/// The regex match that found this variable
/// The files original contents
private bool ProcessVariable(FileProcessingData data, FileOutputData output, Match match, ref string contents)
{
// Get the variable name
var variable = match.Groups[1].Value;
// Resolve the name to a variable value stored in the output variables
var variableValue = output.Variables.FirstOrDefault(v =>
v.Name.EqualsIgnoreCase(variable) &&
v.ProfileName.EqualsIgnoreCase(output.ProfileName));
// If this was a profile-specific variable, fallback to the standard variable
if (variableValue == null && !string.IsNullOrEmpty(output.ProfileName))
variableValue = output.Variables.FirstOrDefault(v =>
v.Name.EqualsIgnoreCase(variable) &&
string.IsNullOrEmpty(v.ProfileName));
// Error if we don't get one
if (variableValue == null)
{
data.Error = $"Variable not found {variable} for profile '{output.ProfileName}'";
// Clear contents as the output will be invalid now if it is not processable
output.CompiledContents = null;
return false;
}
// Replace with the value
ReplaceTag(ref contents, match, variableValue?.Value);
return true;
}
///
/// Changes the file extension to the default output file extension
///
/// The file processing data
///
protected string GetDefaultOutputPath(FileProcessingData data)
{
return Path.Combine(data.LocalConfiguration.OutputPath, OutputExtension == null ? Path.GetFileName(data.FullPath) : Path.GetFileNameWithoutExtension(data.FullPath) + OutputExtension);
}
#endregion
#region Save Output
///
/// Saves the files compiled contents to the output location
///
/// The processing data
/// The output path information
///
public virtual Task SaveFileContents(FileProcessingData processingData, FileOutputData outputPath)
{
return SafeTask.Run(() =>
{
// Save the contents
FileManager.SaveFile(outputPath.CompiledContents, outputPath.FullPath);
});
}
#endregion
#region Extract Data Regions
///
/// Extracts the data from an and stores the variables and data in the engine
///
/// The file processing data
/// The file output data
/// The data to extract
protected void ExtractData(FileProcessingData data, FileOutputData output, XDocameent xmlData)
{
// Find any variables
ExtractVariables(data, output, xmlData.Root);
// Profiles
foreach (var profileElement in xmlData.Root.Elements("Profile"))
{
// Find any variables
ExtractVariables(data, output, profileElement, profileName: profileElement.Attribute("Name")?.Value);
}
// Groups
foreach (var groupElement in xmlData.Root.Elements("Group"))
{
// Find any variables
ExtractVariables(data, output, groupElement, profileName: groupElement.Attribute("Profile")?.Value, groupName: groupElement.Attribute("Name")?.Value);
}
}
///
/// Extract variables from the XML element
///
/// The file processing data
/// The file output data
/// The Xml element
/// The profile name, if explicitly specified
protected void ExtractVariables(FileProcessingData data, FileOutputData output, XElement element, string profileName = null, string groupName = null)
{
// Loop all elements with the name of variable
foreach (var variableElement in element.Elements("Variable"))
{
try
{
// Create the variable
var variable = new EngineVariable
{
XmlElement = variableElement,
Name = variableElement.Attribute("Name")?.Value,
ProfileName = variableElement.Attribute("Profile")?.Value ?? profileName,
Group = variableElement.Attribute("Group")?.Value ?? groupName,
Value = variableElement.Element("Value")?.Value ?? variableElement.Value,
Comment = variableElement.Element("Comment")?.Value ?? variableElement.Attribute("Comment")?.Value
};
// Convert string empty profile names back to null
if (variable.ProfileName == string.Empty)
variable.ProfileName = null;
// If we have no comment, look at previous element for a comment
if (string.IsNullOrEmpty(variable.Comment) && variableElement.PreviousNode is XComment)
variable.Comment = ((XComment)variableElement.PreviousNode).Value;
// Make sure we got a name at least
if (string.IsNullOrEmpty(variable.Name))
{
data.Error = $"Variable has no name {variableElement}";
break;
}
// Add or update variable
var existing = output.Variables.FirstOrDefault(f =>
f.Name.EqualsIgnoreCase(variable.Name) &&
f.ProfileName.EqualsIgnoreCase(variable.ProfileName));
// If one exists, update it
if (existing != null)
existing.Value = variable.Value;
// Otherwise, add it
else
output.Variables.Add(variable);
}
catch (Exception ex)
{
data.Error = $"Unexpected error parsing variable {variableElement}. {ex.Message}";
// No more processing
break;
}
}
}
#endregion
#region Find References
///
/// Searches the for all files that match the
/// then searches inside them to see if they include the includePath pasted in
///
/// The path to look for being included in any of the files
/// The file processing data
/// A list of already found references to check for circular references
///
protected virtual async Task FindReferencedFilesAsync(string includePath, FileProcessingData data, List existingReferences = null)
{
#region Setup Data
// New empty list
var toProcess = new List();
// Make existing references list if not one
if (existingReferences == null)
existingReferences = new List();
// New list for any reference files found
var filesThatReferenceThisFile = new List();
#endregion
// If we have no path, return
if (string.IsNullOrWhiteSpace(includePath))
return toProcess;
#region Find Files That Reference This File
// Find all files in the monitor path
var allFiles = AllMonitoredFiles;
// For each file, find all resolved references
foreach (var file in allFiles)
{
// If any match this file...
if (file.References.Any(reference => reference.EqualsIgnoreCase(includePath)))
{
// Add this file to be processed
toProcess.Add(file.Path);
// Add as a parent to check
filesThatReferenceThisFile.Add(file.Path);
}
}
#endregion
#region Circular Reference Check
// Add this files own references to the list of references we have found so far as we step up the tree
existingReferences.AddRange(await GetResolvedIncludePathsAsync(includePath));
// Circular reference check
if (existingReferences.Contains(includePath))
{
data.Error = $"Circular reference detected to {includePath}";
return toProcess;
}
#endregion
#region Recursive Step-Up Loop
// Now recursively loop all parents looking for any files that reference them
foreach (var referencedFile in filesThatReferenceThisFile)
{
// Get all files that reference this parent
// NOTE: Don't past the existing references as a reference for referenced files
// Their own references are not related to each other parent reference
var parentReferences = await FindReferencedFilesAsync(referencedFile, data, new List(existingReferences));
// Add them to the list
foreach (var parentReference in parentReferences)
{
// Add this to the list
toProcess.Add(parentReference);
}
}
#endregion
// Return what we found
return toProcess;
}
///
/// Finds all files in the monitor folder that match this engines extension types
///
///
protected async Task FindAllMonitoredFilesAsync()
{
// Clear previous list
AllMonitoredFiles.Clear();
// Get all monitored files
var monitoredFiles = FileHelpers.GetDirectoryFiles(ResolvedMonitorPath, "*.*")
.Where(file => EngineExtensions.Any(ex => ex == "*.*" ? true : Regex.IsMatch(Path.GetFileName(file), ex)))
.Distinct()
.ToList();
// For each find their references
foreach (var file in monitoredFiles)
{
// Get all resolved references
var references = await GetResolvedIncludePathsAsync(file);
// Add to list
AllMonitoredFiles.Add((file, references));
}
}
///
/// Returns a list of resolved paths for all include files in a file
///
/// The full path to the file to check
///
protected virtual async Task GetResolvedIncludePathsAsync(string filePath)
{
// New blank list
var paths = new List();
// Make sure the file exists
if (!FileManager.FileExists(filePath))
return paths;
// Read all the file into memory (it's ok we will never have large files they are text web files)
var fileContents = await FileManager.ReadAllTextAsync(filePath);
// Create a match variable
Match match = null;
// Go though all matches
while (match == null || match.Success)
{
// If we have already run a match once...
if (match != null)
// Remove previous tag and carry on
ReplaceTag(ref fileContents, match, string.Empty);
// Find all special tags that have 2 groups
if (!GetIncludeTag(filePath, fileContents, ref match, out List includePaths))
continue;
// For each include path found in the match
includePaths.ForEach(includePath =>
{
// Strip any profile name (we don't care about that for this)
// If the name have a single : then the right half is the profile name
if (includePath.Count(c => c == ':') == 1)
{
// Make sure this isn't from an absolute path like C:\Some...
var index = includePath.IndexOf(':');
if (includePath.Length
{
// Get extension
var extension = watcher.Filter;
// Dispose of watcher
watcher.Dispose();
// Inform listener
StoppedWatching(extension);
// Log the type
LogTabbed("File Type", watcher.Filter, 1);
});
// Space between each engine log
Log("");
// Let listener know we stopped
if (mWatchers != null)
Stopped();
// Clear watchers
mWatchers = null;
}
#endregion
}
}
