UNCONVENTIONAL COMMIT Bump to 0.0.2

While I would usually try and stick to the conventional commits
standard this commit is a big one.

This commit Bumps us to 0.0.2 And completely changes how you interact
with the program.
Now it is easier as you only need to specify the project directory on
the commandline (OR be in the directory of the site you want to build)

Also introduced is the cssitegen.json file that all projects must use.
This means that static information such as the basename, source, and
destination are kept with the files.

ProjectSettings is used to hopefully make managing a site easier,
although future refactoring may join the RuntimeSettings and
ProjectSettings into one class.

There are some obvious issues with the project in its current state but
pending testing with a live domain, it does appear to actually work as
intended. (if this is true then the code just needs refactoring and
tidying to qualify for a 0.1.0 Release.)

Future features planned include
- Code to generate pages from data
- Template nesting (or a custom template templating language)
- Introduction of image conversion to webp (with fallback to RawCpy)
- consistency enforcement, to ensure that deleted source files mean
  deleted destination files.

Program.cs

@@ -25,6 +25,7 @@ using System.CommandLine;
 using System.Diagnostics;
 using System.Reflection;
 using Spectre.Console;
+using System.Text.Json;
 
 namespace csSiteGen;
 
@@ -34,10 +35,12 @@ class Program
 	static int Main(string[] args)
 	{
 
-		// Get the current versiion number
+		// Get the current version number
 		string? version = Assembly.GetEntryAssembly()?.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion;
 
-		// Configure logger
+		// Only log to file
+		// Logging to the console is BAD practice as it tends to be messy.
+		// TODO: Log to a known location using environment to find the correct location.
 		Log.Logger = new LoggerConfiguration()
 			.MinimumLevel.Debug()
 			.WriteTo.File("log.log")
@@ -55,6 +58,7 @@ class Program
 		}
 
 		// It is very likely that this program will only work on linux. As such it is worth warning the user about this.
+		// With further testing and handling of any edge cases it _may_ be possible to have this work anywhere.
 		if (!OperatingSystem.IsLinux())
 		{
 			Log.Warning("This program has only been tested on linux and cannot be assumed to work on other Operating Systems");
@@ -63,49 +67,64 @@ class Program
 
 		Stopwatch TotalExecutionTime = Stopwatch.StartNew();
 
-		var inputDirectoryOption = new Option<DirectoryInfo>(
-			name: "--input",
-			description: "The directory that contains the site source.");
-		inputDirectoryOption.IsRequired = true;
-		inputDirectoryOption.AddValidator(result =>
+		/* !! IMPORTANT !!
+			WARN:
+		This code uses system.commandline which is still in pre-release
+			the following section of code will contain comments to explain the intent of the programmer
+			which may be useful if system.commandline has breaking changes
+		*/
+
+
+		// First the option for the project directory is created
+		var ProjectDirectoryOption = new Option<DirectoryInfo>(
+				name: "--project",
+				description: "The Directory for the project");
+		ProjectDirectoryOption.IsRequired = false; // it is not required as not providing it infers that the current directory is the project directory
+		// If the option is used then the input is validated before control passes to any of the actual code.
+		ProjectDirectoryOption.AddValidator(result =>
 			{
-					if (!result.GetValueForOption(inputDirectoryOption)!.Exists)
+				if (!result.GetValueForOption(ProjectDirectoryOption)!.Exists)
 				{
-						result.ErrorMessage = $"Input directory {result.GetValueForOption(inputDirectoryOption)!.FullName} does not exist";
+					result.ErrorMessage = $"Project directory {result.GetValueForOption(ProjectDirectoryOption)} does not exist.";
 				}
-				});
-
-		var outputDirectoryOption = new Option<DirectoryInfo>(
-			name: "--output",
-			description: "The directory that the site should be output to.");
-		outputDirectoryOption.IsRequired = true;
+			}
+		);
 
+		// The root command is the entry point for commandline but otherwise does nothing.
 		var rootCommand = new RootCommand("csSiteGen");
 
-		var cleanCommand = new Command("clean", "Clean the output directory");
-		cleanCommand.AddOption(outputDirectoryOption);
-		cleanCommand.SetHandler(async (directory) =>
+
+		// TODO: Verify if the use of async in these functions is necessary
+
+		// This creates the command for cleaning a projects output directory.
+		var cleanCommand = new Command("clean", "Clean the projects output directory");
+		cleanCommand.AddOption(ProjectDirectoryOption); // This command can use the project directory option we created earlier
+		cleanCommand.SetHandler(async (ProjectDirectory) =>
 				{
 					await Task.Run(() =>
 						{
-							Clean(directory);
+							Clean(ProjectDirectory);
 						});
-				} ,outputDirectoryOption);
+				},ProjectDirectoryOption);
 
-		var convertCommand = new Command("convert", "Convert the input directory and place the files in the output directory.");
-		convertCommand.AddOption(inputDirectoryOption);
-		convertCommand.AddOption(outputDirectoryOption);
-		convertCommand.SetHandler(async (inputDir, outputDir) =>
+		// This creates the command for actually converting the project.
+		var convertCommand = new Command("convert", "Convert the projects input directory and place the files in the output directory.");
+		convertCommand.AddOption(ProjectDirectoryOption); // This command can use the project directory option
+		convertCommand.SetHandler(async (ProjectDirectory) =>
 				{
 					await Task.Run(() =>
 					{
-						Convert(inputDir, outputDir);
+						Convert(ProjectDirectory);
 					});
-				}, inputDirectoryOption, outputDirectoryOption);
+				},ProjectDirectoryOption);
 
+		// Adding the commands to the root command makes them actually callable on the commandline
 		rootCommand.AddCommand(cleanCommand);
 		rootCommand.AddCommand(convertCommand);
 
+		// The parser is what actually handles the arguments and dispatches them to the appropriate commands.
+		// This is used instead of the simpler method of just Invoking the root command as it automatically creates usage statements.
+		// It also makes a user aware that a subcommand needs to be used.
 		var parser = new CommandLineBuilder(rootCommand)
 			.UseDefaults()
 			.Build();
@@ -116,21 +135,35 @@ class Program
 		TotalExecutionTime.Stop();
 		Log.Information("TotalExecutionTime {time:000}ms", TotalExecutionTime.ElapsedMilliseconds);
 
+		// Providing there were no early exits it is best to properly close the log before we exit.
 		Log.CloseAndFlush();
 		return 0;
 	}
 
-	static int Convert(DirectoryInfo inputDir, DirectoryInfo outputDir)
+	static int Convert(DirectoryInfo? ProjectDirectory)
 	{
+		Log.Information("Convert command was called, beginning conversion.");
+
+		// WARN: This is only temporary as Spectre.Console does not recognise some Linux terminals
+		// A better solution that checks the terminal value and sets this option should be added in the future.
 		AnsiConsole.Console.Profile.Capabilities.Ansi = true;
 
+
+		// NOTE: Future refactors may merge ProjectSettings and RuntimeSettings
+		ProjectSettings projectSettings = GetProjectSettings(ProjectDirectory);
+
+		DirectoryInfo inputDir = new(projectSettings.Source);
+		DirectoryInfo outputDir = new(projectSettings.Destination);
+
+		RuntimeSettings settings = new(inputDir,outputDir);
+		settings.setBaseUrl(projectSettings.BaseUrl);
+
 		List<SiteFile> siteFiles = new();
 
 		Utils.GetFiles(inputDir).ForEach(x => siteFiles.Add(new SiteFile(x)));
 		Log.Information("SiteFiles: {@sf} {count}", siteFiles, siteFiles.Count);
 
 		Console.WriteLine($"Converting {siteFiles.Count} files from {inputDir.FullName} to {outputDir.FullName}");
-		RuntimeSettings settings = new(inputDir,outputDir);
 
 
 		Dictionary<string,bool> fileStatus = new();
@@ -187,8 +220,16 @@ class Program
 		return 0;
 	}
 
-	static int Clean(DirectoryInfo outputDir)
+	static int Clean(DirectoryInfo? ProjectDirectory)
 	{
+		Log.Information("Clean command was called, Beginning cleaning");
+
+		// NOTE: Future refactors may merge ProjectSettings and RuntimeSettings
+		ProjectSettings projectSettings = GetProjectSettings(ProjectDirectory);
+
+		DirectoryInfo inputDir = new(projectSettings.Source);
+		DirectoryInfo outputDir = new(projectSettings.Destination);
+
 		if (!outputDir.Exists)
 		{
 			Log.Warning("Not deleting {dir} as it doesn't exist",outputDir.FullName);
@@ -220,4 +261,57 @@ class Program
 			return 1;
 		}
 	}
+
+	static void EnforceConsistency(ProjectSettings projectSettings)
+	{
+		// Grab the metadata.
+
+		// Read the metadata.
+
+		// Find deleted files.
+
+		// Figure out what the new name for those files would be.
+
+		// Remove those files.
+
+	}
+
+	static ProjectSettings GetProjectSettings(DirectoryInfo? ProjectDirectory)
+	{
+		// TODO: implement proper error handling where file access is performed.
+
+		if (ProjectDirectory is null)
+		{
+			// use the current directory if no project directory is passed.
+		    ProjectDirectory = new DirectoryInfo(".");
+		}
+		Log.Information("{projectdir} => fullname {pdfn}",ProjectDirectory, ProjectDirectory.FullName);
+		FileInfo projectFile = new (Path.Combine(ProjectDirectory.FullName,"cssitegen.json"));
+
+
+		if (!projectFile.Exists)
+		{
+		    Log.Fatal("Cannot locate project file {pf} in {dir}",projectFile,ProjectDirectory);
+			Environment.Exit(1);
+		}
+
+		Log.Information("Located Project File {pf}",projectFile.FullName);
+		ProjectSettings? projectSettings = JsonSerializer
+			.Deserialize<ProjectSettings>(
+					projectFile
+					.OpenText()
+					.ReadToEnd()
+		);
+
+		if (projectSettings is null)
+		{
+		    Log.Fatal("Cannot deserialize projectFile");
+			Environment.Exit(1);
+		}
+		projectSettings.setProjectRoot(ProjectDirectory);
+
+		Log.Information("{@ps}",projectSettings);
+
+		return projectSettings;
+	}
 }

ProjectSettings/ProjectSettings.cs

@@ -0,0 +1,44 @@
+using System.Text.Json.Serialization;
+// project settings is a user accessible config to set the Source and destination of a site
+// This may include more scope in the future such as holding the site base address etc..
+
+class ProjectSettings
+{
+    private string _Source;
+	private string _Destination;
+	private DirectoryInfo? _ProjectRoot;
+	public string? BaseUrl {get; private set;}
+
+	public string Source {get {
+		if (_ProjectRoot is null)
+		{
+		    return _Source;
+		}
+		return Path.Combine(_ProjectRoot.FullName,_Source);
+	}}
+	public string Destination {get {
+		if (_ProjectRoot is null)
+		{
+		    return _Destination;
+		}
+		return Path.Combine(_ProjectRoot.FullName,_Destination);
+	}}
+
+	[JsonConstructor]
+	public ProjectSettings(String source, String destination, string baseUrl) {
+		_Source = source;
+		_Destination = destination;
+		BaseUrl = baseUrl;
+	}
+
+	public void setProjectRoot(string projectRoot) {
+		_ProjectRoot = new(projectRoot);
+	}
+	public void setProjectRoot(DirectoryInfo projectRoot) {
+		_ProjectRoot = projectRoot;
+	}
+
+	public void setBaseUrl(string baseUrl) {
+		BaseUrl = baseUrl;
+	}
+}

RuntimeSettings/RuntimeSettings.cs

@@ -2,11 +2,12 @@ namespace csSiteGen;
 
 
 /// <summary>
-/// Class <c>RuntimeSettings</p> Contains all the settings that could be loaded from the commandline.
+/// Class <c>RuntimeSettings</c> Contains all the settings that could be loaded from the commandline.
 /// </summary>
 public class RuntimeSettings {
 	public DirectoryInfo InputDirectory {get; private set;}
 	public DirectoryInfo OutputDirectory {get; private set;}
+	public string? BaseUrl {get; private set;}
 
 
 	public RuntimeSettings(string inputDirectory, string outputDirectory){
@@ -26,4 +27,7 @@ public class RuntimeSettings {
 		*/
 	}
 
+	public void setBaseUrl(string? baseurl) {
+		BaseUrl = baseurl;
+	}
 }

SiteFile/SiteFile.ConverterFunctions.cs

@@ -15,6 +15,11 @@ public static class Conversions{
 		{".md", Pandoc},
 	};
 
+	private static readonly string[] BaseUrlFiletypes = {
+		".md",
+		".html"
+	};
+
 
 	/// <summary>
 	///	TEST FUNCTION.
@@ -28,7 +33,7 @@ public static class Conversions{
 	}
 
 	/// <summary>
-	/// Copy the file verbatim
+	/// Copy the file verbatim (doing any baseurl replacements if needed)
 	/// </summary>
 	public static bool RawCpy(FileInfo file, RuntimeSettings settings){
 		FileInfo newPath = new FileInfo(GetNewName(file,settings,null));
@@ -41,8 +46,15 @@ public static class Conversions{
 		}
 
 		try {
+			if (BaseUrlFiletypes.Contains(file.Extension))
+			{
+				File.WriteAllText(newPath.FullName, BaseUrlReplace(file, settings));
+			}
+			else
+			{
 				file.CopyTo(newPath.FullName, overwrite: true);
 			}
+		}
 		catch (Exception e){
 			Log.Fatal(e,"Copy Failed");
 			return false;
@@ -54,6 +66,11 @@ public static class Conversions{
 	/// Execute pandoc on the file, automatically detecting the template to use.
 	/// </summary>
 	public static bool Pandoc(FileInfo file, RuntimeSettings settings){
+		// NOTE: Some of the code later where the tmpfile is created for baseurl replacement may be too safe.
+		// the extension checks may be unnecessary, but this depends on if this function will be retooled to run pandoc for different conversions.
+		// for now I have take the safer approach, but the leaner approach may be used in the future when the project is more mature
+
+
 		Log.Information("Attempting to convert {file} using pandoc",file.Name);
 
 		// Look for pandoc
@@ -91,7 +108,37 @@ public static class Conversions{
 			searchDir = searchDir.Parent;
 		} while (searchDir != settings.InputDirectory); // Check last as we want to search the InputDirectory
 
-		string pandocArgs = $"{file.FullName} -o {GetNewName(file,settings,".html")}";
+		// the empty string is used as it has a defined identity
+		string tmpFile = string.Empty;
+		if (BaseUrlFiletypes.Contains(file.Extension))
+		{
+			Log.Information("Replacing baseurl for file {f}",file.FullName);
+		    tmpFile = Path.Join(Path.GetTempPath(),"pandoc",file.Name);
+			Directory.CreateDirectory(Path.GetDirectoryName(tmpFile)!); // NOTE: It is practially impossible that this would actually return null
+			File.Create(tmpFile).Close(); // TODO: Use the filestream provided by File.Create within a using block to write the text
+			File.WriteAllText(tmpFile,BaseUrlReplace(file,settings));
+
+			if (template is not null)
+			{
+				Log.Information("Replacing baseurl in template file");
+				string tmpTemplateFile = Path.Join(Path.GetTempPath(),"pandoc",template.Name);
+				Directory.CreateDirectory(Path.GetDirectoryName(tmpTemplateFile)!); // NOTE: It is practially impossible that this would actually return null
+				File.Create(tmpTemplateFile).Close(); // TODO: Use the filestream provided by File.Create within a using block to write the text
+				File.WriteAllText(tmpTemplateFile,BaseUrlReplace(template,settings));
+				template = new(tmpTemplateFile);
+			}
+		}
+
+		string pandocArgs;
+		// If we have created a temporary file we need to ensure that we use it.
+		if (!string.IsNullOrEmpty(tmpFile))
+		{
+			pandocArgs = $"{tmpFile} -o {GetNewName(file,settings,".html")}";
+		}
+		else
+		{
+			pandocArgs = $"{file.FullName} -o {GetNewName(file,settings,".html")}";
+		}
 
 		if (template is not null)
 		{
@@ -103,7 +150,18 @@ public static class Conversions{
 			Log.Warning("Pandoc template for {file} not found",file.Name);
 		}
 
-		return RunExternalProgram(pandoc,pandocArgs);
+		if (!Directory.Exists(Path.GetDirectoryName(GetNewName(file,settings,".html"))))
+		{
+		   Directory.CreateDirectory(Path.GetDirectoryName(GetNewName(file,settings,".html"))!);
+		}
+
+		bool pandocReturn = RunExternalProgram(pandoc,pandocArgs);
+		// If we made a tmpfile delete it after running pandoc against it.
+		if (!string.IsNullOrEmpty(tmpFile))
+		{
+		    File.Delete(tmpFile);
+		}
+		return pandocReturn;
 
 	}
 
@@ -138,8 +196,8 @@ public static class Conversions{
 
 			RunProgram.WaitForExit();
 
-			Log.Debug("{program} stdout {stdout}", program, stdout);
-			Log.Debug("{program} stderr {stderr}", program, stderr);
+			Log.Debug("{program} STDOUT:\n{stdout}", program, stdout);
+			Log.Debug("{program} STDERR:\n{stderr}", program, stderr);
 
 			if (RunProgram.ExitCode != 0)
 			{
@@ -155,4 +213,23 @@ public static class Conversions{
 			.Replace(settings.InputDirectory.FullName, settings.OutputDirectory.FullName)
 			.Replace(file.Extension,newExtension ?? file.Extension);
 	}
+
+	private static string? BaseUrlReplace(FileInfo file, RuntimeSettings settings){
+		Log.Information("Doing BaseUrlReplace for {f}", file.FullName);
+		// Read the file
+		using (StreamReader FileReader = file.OpenText())
+		{
+			string filestring = FileReader.ReadToEnd();
+
+			if (settings.BaseUrl is null)
+			{
+				Log.Warning("BaseUrl is null, replacing templateString with nothing.");
+				return filestring.Replace("%BASEURL%","");
+			}
+
+			Log.Information("Replacing templateString with {BaseUrl}",settings.BaseUrl);
+			return filestring.Replace("%BASEURL%",settings.BaseUrl);
+		}
+	}
+
 }

SiteFile/SiteFile.cs

@@ -74,7 +74,11 @@ public partial class SiteFile
 
 		// NOTE: By this point Metadata CANNOT be null as LoadMetadata would either have loaded the JSON or instantiated blank MetaData
 		// The compiler however is unaware of this as it uses side effects, so we tell it there is no possible null value here.
-		return (Metadata!.GetValueOrDefault(info.FullName, DateTime.MinValue) == info.LastWriteTimeUtc);
+		Log.Debug("Attempting to check metadata for file {f}",info.FullName);
+		Log.Debug("METADATA={MD}",Metadata!.GetValueOrDefault(info.FullName,DateTime.MinValue));
+		Log.Debug("FILETIME={FT}",info.LastWriteTimeUtc);
+		Log.Debug("RES={res}",(Metadata!.GetValueOrDefault(info.FullName, DateTime.MinValue) != info.LastWriteTimeUtc));
+		return (Metadata!.GetValueOrDefault(info.FullName, DateTime.MinValue) != info.LastWriteTimeUtc);
 	}
 
 

TODO

@@ -1,5 +1,9 @@
+- UPDATE README
+
+- Make the code such that the metadata file knows what source file an output file came from, this will allow us to delete files from source
+  and force consistency by removing them from the dst directory too.
+
 - Add a pre-commit or other type of hook to ensure that the Testing directory is properly cleaned and reset before commit.
-- Double check that links are processed correctly by testing against a copy of my blog source.
-	- Also add the appropriate metadata to the markdown files to make sure that they are properly detected.
-	- Potentially host a local Test version to fully ensure that everything can be made to work properly.
+	- Ensure that the .gitkeep file is placed into Testing/dst
+
 - Look into the possiblity of using a TemplateTemplate, And generating the pandoc template on the fly making it possible to switch out domian/prefix stuff

Testing/cssitegen.json

@@ -0,0 +1,5 @@
+{
+	"Source" : "./src",
+	"Destination" : "./dst",
+	"BaseUrl" : null
+}

Utils/Utils.GetFiles.cs

@@ -4,6 +4,6 @@ public static partial class Utils {
 
 	// Abstract Directory.GetFiles to get a List as it will be easier to handle later.
 	public static List<FileInfo> GetFiles(DirectoryInfo dir){
-		return dir.GetFiles("*",SearchOption.AllDirectories).ToList();
+		return dir.GetFiles("*",SearchOption.AllDirectories).Where(x => x.Name != ".template").ToList();
 	}
 }

Utils/Utils.PathSearch.cs

@@ -3,6 +3,8 @@ namespace csSiteGen;
 
 public static partial class Utils {
 
+	// As the PathSearch utility will be called for every convertible file
+	// It has been memoized which mean subsequent calls for the same argument just return the result
 	static Dictionary<string,string> PathSearchMemo = new();
 
 	///<summary>

csSiteGen.csproj

@@ -4,7 +4,7 @@
 		<OutputType>Exe</OutputType>
 
 		<Version Condition="'$(RELEASE_VERSION)' != ''">$(RELEASE_VERSION)</Version>
-		<VersionPrefix Condition="'$(RELEASE_VERSION)' == ''">0.0.1</VersionPrefix>
+		<VersionPrefix Condition="'$(RELEASE_VERSION)' == ''">0.0.2</VersionPrefix>
 		<VersionSuffix Condition="'$(RELEASE_VERSION)' == ''">$([System.DateTime]::UtcNow.ToString(`yyyyMMdd-HHmm`))</VersionSuffix>