RSCacheTool/Program.cs

using System;
using System.IO;
using System.Text;
using System.Linq;
using System.Collections.Generic;
using System.IO.Compression;
using System.Threading;
using System.Diagnostics;
using ICSharpCode.SharpZipLib.BZip2;
using NDesk.Options;
using System.Text.RegularExpressions;

namespace RSCacheTool
{
	static class Program
	{
		static string cacheDir = Environment.ExpandEnvironmentVariables(@"%USERPROFILE%\jagexcache\runescape\LIVE\");
		static string outDir = "cache\\";

		static int Main(string[] args)
		{
			bool error = false;

			bool help = false, extract = false, combine = false, overwrite = false, incomplete = false, nameMusic = false;
			int extractArchive = -1, combineArchive = 40;

			OptionSet argsParser = new OptionSet() {
				{ "h", "show this message", val => { help = true; } },

				{ "o", "overwrite existing files, for all actions", val => { overwrite = true; } },

				{ "e:", "extract files from cache, supply a number to extract only a specific archive", val => { 
					extract = true; 
					if (!String.IsNullOrEmpty(val))
						int.TryParse(val, out extractArchive); 
				}},

				{ "c", "combine sound", val => { combine = true; } },
				{ "s=", "archive to combine sounds of, defaults to 40", val => { combineArchive = Convert.ToInt32(val); } },
				{ "i", "merge incomplete files (into special directory)", val => { incomplete = true; } },

				{ "n", "try to name music (archive 40, needs archive 17 file 5 too), renames incompletes too if i is set", val => { nameMusic = true; } },
			};

			List<string> otherArgs = argsParser.Parse(args);

			for (int i = 0; i < otherArgs.Count; i++)
			{
				string parsedPath = otherArgs[i];
				if (!parsedPath.EndsWith("\\"))
					parsedPath += "\\";

				parsedPath = Environment.ExpandEnvironmentVariables(parsedPath);

				if (Directory.Exists(parsedPath))
				{
					if (i == 0)
						outDir = parsedPath;
					else if (i == 1)
						cacheDir = parsedPath;
				}
				else
				{
					Console.WriteLine("The directory: " + parsedPath + " is not valid.");
					error = true;
				}
			}

			if (args.Length == 0 || help)
			{
				Console.WriteLine(
					"Usage: rscachetools [options] outDir cacheDir\n" + 
					"Provides various tools for extracting and manipulating RuneScape's cache files.\n" +
					"\n" +
					"Arguments:\n" +
					"outDir - The directory in which all files generated by this tool will be placed. Default: cache\\\n" +
					"cacheDir - The directory that contains all cache files. Default: %USERPROFILE%\\jagexcache\\runescape\\LIVE\\.\n" +
					"\n" +
					"Options:"
				);

				argsParser.WriteOptionDescriptions(Console.Out);
			}
			else if (!error)
			{
				//create outdir
				if (!Directory.Exists(outDir))
					Directory.CreateDirectory(outDir);

				if (extract)
					ExtractFiles(extractArchive, overwrite);

				if (combine)
					CombineSounds(combineArchive, overwrite, incomplete);

				if (nameMusic)
					NameMusic(incomplete, overwrite);					
			}

			return 0;
		}

		/// <summary>
		/// Rips all files from the cachefile and puts them (structured and given a fitting extension where possible) in the fileDir.
		/// </summary>
		static void ExtractFiles(int archive, bool overwriteExisting)
		{
			int startArchive = 0, endArchive = 255;

			if (archive != -1)
			{
				startArchive = archive;
				endArchive = archive;
			}

			using (FileStream cacheFile = File.Open(cacheDir + "main_file_cache.dat2", FileMode.Open, FileAccess.Read))
			{
				for (int archiveIndex = startArchive; archiveIndex <= endArchive; archiveIndex++)
				{
					string indexFileString = cacheDir + "main_file_cache.idx" + archiveIndex.ToString();

					if (File.Exists(indexFileString))
					{
						FileStream indexFile = File.Open(indexFileString, FileMode.Open, FileAccess.Read);

						int fileCount = (int)indexFile.Length / 6;

						for (int fileIndex = 0; fileIndex < fileCount; fileIndex++)
						{
							bool fileError = false;

							indexFile.Position = fileIndex * 6L;

							uint fileSize = indexFile.ReadBytes(3);
							long startChunkOffset = indexFile.ReadBytes(3) * 520L;

							//Console.WriteLine("New file: archive: {0} file: {1} offset: {3} size: {2}", archiveIndex, fileIndex, fileSize, startChunkOffset);
 
							if (fileSize > 0 && startChunkOffset > 0 && startChunkOffset + fileSize <= cacheFile.Length)
							{
								byte[] buffer = new byte[fileSize];
								int writeOffset = 0;
								long currentChunkOffset = startChunkOffset;

								for (int chunkIndex = 0; writeOffset < fileSize && currentChunkOffset > 0; chunkIndex++)
								{
									cacheFile.Position = currentChunkOffset;

									int chunkSize;
									int checksumFileIndex = 0;

									if (fileIndex < 65536)
									{
										chunkSize = (int)Math.Min(512, fileSize - writeOffset);
									}
									else
									{
										//if file index exceeds 2 bytes, add 65536 and read 2(?) extra bytes
										chunkSize = (int)Math.Min(510, fileSize - writeOffset);

										cacheFile.ReadByte();
										checksumFileIndex = (cacheFile.ReadByte() << 16);
									}

									checksumFileIndex += (int)cacheFile.ReadBytes(2);
									int checksumChunkIndex = (int)cacheFile.ReadBytes(2);
									long nextChunkOffset = cacheFile.ReadBytes(3) * 520L;
									int checksumArchiveIndex = cacheFile.ReadByte();

									//Console.WriteLine("Chunk {2}: archive: {3} file: {1} size: {0} nextoffset: {4}", chunkSize, checksumFileIndex, checksumChunkIndex, checksumArchiveIndex, nextChunkOffset);

									if (checksumFileIndex == fileIndex && checksumChunkIndex == chunkIndex && checksumArchiveIndex == archiveIndex &&
										nextChunkOffset >= 0 && nextChunkOffset < cacheFile.Length)
									{
										cacheFile.Read(buffer, writeOffset, chunkSize);
										writeOffset += chunkSize;
										currentChunkOffset = nextChunkOffset;
									}
									else
									{
										Console.WriteLine("Ignoring file because a chunk's checksum doesn't match, ideally should not happen.");

										fileError = true;
										break;
									}
								}

								if (!fileError)
								{
									//process file
									string outFileDir = outDir + archiveIndex + "\\";
									string outFileName = fileIndex.ToString();
									byte[] tempBuffer;

									//decompress gzip
									if (buffer.Length > 10 && (buffer[9] << 8) + buffer[10] == 0x1f8b) //gzip
									{
										//remove the first 9 bytes cause they seem to be descriptors of sorts (no idea what they do but they are not part of the file)
										tempBuffer = new byte[fileSize - 9];
										Array.Copy(buffer, 9, tempBuffer, 0, fileSize - 9);
										buffer = tempBuffer;

										GZipStream decompressionStream = new GZipStream(new MemoryStream(buffer), CompressionMode.Decompress);

										int readBytes;
										tempBuffer = new byte[0];

										do
										{
											byte[] readBuffer = new byte[100000];
											readBytes = decompressionStream.Read(readBuffer, 0, 100000);

											int storedBytes = tempBuffer.Length;
											Array.Resize(ref tempBuffer, tempBuffer.Length + readBytes);
											Array.Copy(readBuffer, 0, tempBuffer, storedBytes, readBytes);
										}
										while (readBytes == 100000);

										buffer = tempBuffer;

										Console.WriteLine("File decompressed as gzip.");
									}

									//decompress bzip2
									if (buffer.Length > 14 && buffer[9] == 0x31 && buffer[10] == 0x41 && buffer[11] == 0x59 && buffer[12] == 0x26 && buffer[13] == 0x53 && buffer[14] == 0x59) //bzip2
									{
										//remove the first 9 bytes cause they seem to be descriptors of sorts (no idea what they do but they are not part of the file)
										tempBuffer = new byte[fileSize - 9];
										Array.Copy(buffer, 9, tempBuffer, 0, fileSize - 9);
										buffer = tempBuffer;

										//prepend file header
										byte[] magic = new byte[] {
											0x42, 0x5a, //BZ (signature)
											0x68,		//h (version)
											0x31		//*100kB block-size
										};

										tempBuffer = new byte[magic.Length + buffer.Length];
										magic.CopyTo(tempBuffer, 0);
										buffer.CopyTo(tempBuffer, magic.Length);
										buffer = tempBuffer;

										BZip2InputStream decompressionStream = new BZip2InputStream(new MemoryStream(buffer));

										int readBytes;
										tempBuffer = new byte[0];

										do
										{
											byte[] readBuffer = new byte[100000];
											readBytes = decompressionStream.Read(readBuffer, 0, 100000);

											int storedBytes = tempBuffer.Length;
											Array.Resize(ref tempBuffer, tempBuffer.Length + readBytes);
											Array.Copy(readBuffer, 0, tempBuffer, storedBytes, readBytes);
										}
										while (readBytes == 100000);

										buffer = tempBuffer;

										Console.WriteLine("File decompressed as bzip2.");														
									}

									//detect ogg: OggS
									if (buffer.Length > 3 && (buffer[0] << 24) + (buffer[1] << 16) + (buffer[2] << 8) + buffer[3] == 0x4f676753)
										outFileName += ".ogg";

									//detect ogg: 5 bytes - OggS
									if (buffer.Length > 8 && (buffer[5] << 24) + (buffer[6] << 16) + (buffer[7] << 8) + buffer[8] == 0x4f676753)
									{
										tempBuffer = new byte[fileSize - 5];
										Array.Copy(buffer, 5, tempBuffer, 0, fileSize - 5);
										buffer = tempBuffer;

										outFileName += ".ogg";
									}

									//detect jag: JAGA
									if (buffer.Length > 3 && (buffer[0] << 24) + (buffer[1] << 16) + (buffer[2] << 8) + buffer[3] == 0x4a414741)
										outFileName += ".jaga";

									//detect jag: 5 bytes - JAGA
									if (buffer.Length > 8 && (buffer[5] << 24) + (buffer[6] << 16) + (buffer[7] << 8) + buffer[8] == 0x4a414741)
									{
										tempBuffer = new byte[fileSize - 5];
										Array.Copy(buffer, 5, tempBuffer, 0, fileSize - 5);
										buffer = tempBuffer;

										outFileName += ".jaga";
									}

									//detect png: .PNG
									if (buffer.Length > 3 && (uint)(buffer[0] << 24) + (buffer[1] << 16) + (buffer[2] << 8) + buffer[3] == 0x89504e47)
										outFileName += ".png";

									//detect png: 5 bytes - .PNG
									if (buffer.Length > 8 && (uint)(buffer[5] << 24) + (buffer[6] << 16) + (buffer[7] << 8) + buffer[8] == 0x89504e47)
									{
										tempBuffer = new byte[fileSize - 5];
										Array.Copy(buffer, 5, tempBuffer, 0, fileSize - 5);
										buffer = tempBuffer;

										outFileName += ".png";
									}

									//create and write file
									if (!Directory.Exists(outFileDir))
										Directory.CreateDirectory(outFileDir);

									//(over)write file
									if (!File.Exists(outFileDir + outFileName) || overwriteExisting)
									{
										using (FileStream outFile = File.Open(outFileDir + outFileName, FileMode.Create, FileAccess.Write))
										{
											outFile.Write(buffer, 0, buffer.Length);
											Console.WriteLine(outFileDir + outFileName);
										}
									}
									else
										Console.WriteLine("Skipping file because it already exists.");
								}
							}
							else
							{
								Console.WriteLine("Ignoring file because of size or offset.");
							}
						}
					}
				}
			}

			Console.WriteLine("Done extracting files.");
		}

		/// <summary>
		/// Combines the sound files (.jag &amp; .ogg) in the specified archive (40 for the build it was made on), and puts them into the soundtracks directory.
		/// </summary>
		static void CombineSounds(int archive, bool overwriteExisting, bool mergeIncomplete)
		{
			string archiveDir = outDir + archive + "\\";
			string soundDir = outDir + "sound\\";

			//gather all index files
			string[] indexFiles = Directory.GetFiles(archiveDir, "*.jaga", SearchOption.TopDirectoryOnly);

			//create directories
			if (!Directory.Exists(soundDir + "incomplete\\"))
				Directory.CreateDirectory(soundDir + "incomplete\\");

			foreach (string indexFileString in indexFiles)
			{
				string indexFileIdString = Path.GetFileNameWithoutExtension(indexFileString);

				bool incomplete = false;
				List<string> chunkFiles = new List<string>();

				using (FileStream indexFileStream = File.Open(indexFileString, FileMode.Open, FileAccess.Read, FileShare.Read))
				{
					indexFileStream.Position = 32L;

					while (indexFileStream.ReadBytes(4) != 0x4f676753)
					{
						uint fileId = indexFileStream.ReadBytes(4);

						//check if the file exists and add it to the buffer if it does
						if (File.Exists(archiveDir + fileId + ".ogg"))
							chunkFiles.Add(archiveDir + fileId + ".ogg");
						else
							incomplete = true;
					}

					//copy the first chunk to a temp file so SoX can handle the combining
					indexFileStream.Position -= 4L;

					//wait till file is available
					while (true)
					{
						try
						{
							using (FileStream tempIndexFile = File.Open("~index.ogg", FileMode.OpenOrCreate, FileAccess.Write, FileShare.None))
							{
								indexFileStream.CopyTo(tempIndexFile);
								break;
							}
						}
						catch (IOException)
						{
							Thread.Sleep(200);
						}
					}
				}

				if (!incomplete || incomplete && mergeIncomplete)
				{
					string outFile = soundDir + (incomplete ? "incomplete\\" : "") + indexFileIdString + ".ogg";

					if (!overwriteExisting && File.Exists(outFile))
						Console.WriteLine("Skipping track because it already exists.");
					else
					{
						//combine the files with sox
						Console.WriteLine("Running SoX to concatenate ogg audio chunks.");

						Process soxProcess = new Process();
						soxProcess.StartInfo.FileName = "sox.exe";

						soxProcess.StartInfo.Arguments = "--combine concatenate ~index.ogg";
						chunkFiles.ForEach((str) =>
						{
							soxProcess.StartInfo.Arguments += " " + str;
						});
						soxProcess.StartInfo.Arguments += " -C 6 --comment \"Created by RSCacheTool, combined by SoX.\"";
						soxProcess.StartInfo.Arguments += " " + soundDir + "incomplete\\" + indexFileIdString + ".ogg ";
						soxProcess.StartInfo.UseShellExecute = false;

						soxProcess.Start();
						soxProcess.WaitForExit();

						if (soxProcess.ExitCode == 0)
						{
							if (!incomplete)
							{
								//clear space
								if (File.Exists(outFile))
									File.Delete(outFile);

								File.Move(soundDir + "incomplete\\" + indexFileIdString + ".ogg", outFile);
							}

							Console.WriteLine(outFile);
						}
						else
							Console.WriteLine("SoX encountered error code " + soxProcess.ExitCode + " and probably didn't finish processing the files.");
					}
				}
				else
					Console.WriteLine("Skipping track because it's incomplete.");
			}

			//cleanup on isle 4
			File.Delete("~index.ogg");

			Console.WriteLine("Done combining sound.");
		}

		/// <summary>
		/// Tries to parse Archive 17 file 5 to obtain a list of music and their corresponding index file id in archive 40.
		/// Returns a dictionary that can resolve index file id to the name of the track as it appears in-game.
		/// </summary>
		public static void NameMusic(bool incomplete, bool overwrite)
		{
			//the following is based on even more assumptions than normal made while comparing 2 extracted caches, it's therefore probably the first thing to break
			//4B magic number (0x00016902) - 2B a file id? - 2B amount of files (higher than actual entries sometimes) - 2B amount of files

			string resolveFileName = outDir + "17\\5";

			if (File.Exists(resolveFileName))
			{
				using (FileStream resolveFile = File.Open(resolveFileName, FileMode.Open, FileAccess.Read, FileShare.Read))
				{
					Dictionary<int, string> trackIdNames = new Dictionary<int, string>();
					Dictionary<uint, int> fileIdTracks = new Dictionary<uint, int>();

					byte[] magicNumber = new byte[] {
						0x00,
						0x01,
						0x69,
						0x02
					};

					//locate start of the music names (8th magic number) and file ids (12th)
					long namesStartPos = resolveFile.IndexOf(magicNumber, 8);
					long filesStartPos = resolveFile.IndexOf(magicNumber, 12);

					if (namesStartPos != -1 && filesStartPos != -1)
					{
						resolveFile.Position = namesStartPos + 8;
						uint musicCount = resolveFile.ReadBytes(2);
						
						//construct trackIdNames
						Regex regex = new Regex("[" + Regex.Escape(new string(Path.GetInvalidFileNameChars())) + "]");
						for(int i = 0; i < musicCount; i++)
						{
							int trackId = (int)resolveFile.ReadBytes(2);
							string trackName = resolveFile.ReadNullTerminatedString();

							//remove characters that can't be used in files from trackName
							trackName = regex.Replace(trackName, "");

							//add only if the string is of any use
							if (!String.IsNullOrWhiteSpace(trackName))
								trackIdNames.Add(trackId, trackName);
						}

						//construct fileIdTracks
						resolveFile.Position = filesStartPos + 8;
						uint fileCount = resolveFile.ReadBytes(2);
						for (int i = 0; i < fileCount; i++)
						{
							int trackId = (int)resolveFile.ReadBytes(2);
							uint fileId = resolveFile.ReadBytes(4);

							//only add if it doesn't exist already
							if (!fileIdTracks.ContainsKey(fileId))
								fileIdTracks.Add(fileId, trackId);
						}

						//let's do this!
						if (!Directory.Exists(outDir + "sound\\named\\"))
							Directory.CreateDirectory(outDir + "sound\\named\\");

						foreach (string file in Directory.GetFiles(outDir + "sound\\"))
						{
							string fileIdString = Path.GetFileNameWithoutExtension(file);
							uint fileId;
							if (uint.TryParse(fileIdString, out fileId))
							{
								if (fileIdTracks.ContainsKey(fileId))
								{
									int trackId = fileIdTracks[fileId];
									if (trackIdNames.ContainsKey(trackId))
									{
										string trackName = trackIdNames[trackId];
										string destFile = outDir + "sound\\named\\" + trackName + ".ogg";

										if (!File.Exists(destFile) || overwrite)
											File.Copy(file, destFile, true);

										Console.WriteLine(destFile);
									}
								}
							}
						}

						//redundancy, whatever
						if (incomplete)
						{
							if (!Directory.Exists(outDir + "sound\\named\\incomplete\\"))
								Directory.CreateDirectory(outDir + "sound\\named\\incomplete");

							foreach (string file in Directory.GetFiles(outDir + "sound\\incomplete\\"))
							{
								string fileIdString = Path.GetFileNameWithoutExtension(file);
								uint fileId;
								if (uint.TryParse(fileIdString, out fileId))
								{
									if (fileIdTracks.ContainsKey(fileId))
									{
										int trackId = fileIdTracks[fileId];
										if (trackIdNames.ContainsKey(trackId))
										{
											string trackName = trackIdNames[trackId];
											string destFile = outDir + "sound\\named\\incomplete\\" + trackName + ".ogg";

											if (!File.Exists(destFile) || overwrite)
												File.Copy(file, destFile, true);

											Console.WriteLine(destFile);
										}
									}
								}
							}
						}
					}
					else
						Console.WriteLine("Entry points within resolving file could not be found.");
				}
			}
			else
				Console.WriteLine("File for resolving music names (" + resolveFileName + ") does not exist.");

			Console.WriteLine("Done naming music.");
		}

		/// <summary>
		/// Reads a given amount of unsigned bytes from the stream and combines them into one unsigned integer.
		/// </summary>
		public static uint ReadBytes(this Stream stream, int bytes)
		{
			if (bytes < 1 || bytes > 4)
				throw new ArgumentOutOfRangeException();

			uint result = 0;

			for (int i = 0; i < bytes; i++)
				result += (uint)stream.ReadByte() << (bytes - i - 1) * 8;

			return result;
		}

		/// <summary>
		/// Reads ANSI characters into a string until \0 or EOF occurs.
		/// </summary>
		public static string ReadNullTerminatedString(this Stream stream)
		{
			string result = "";
			int readByte = stream.ReadByte();

			while (readByte > 0)
			{
				result += Encoding.Default.GetString(new byte[] { (byte)readByte });
				readByte = stream.ReadByte();
			}

			return result;
		}

		/// <summary>
		/// Returns the stream location of the matchNumber-th occurence of needle, or -1 when there are no(t enough) matches.
		/// </summary>
		public static long IndexOf(this Stream stream, byte[] needle, int matchNumber = 1, int bufferSize = 10000)
		{
			//for resetting after method
			long startPosition = stream.Position;

			byte[] buffer = new byte[bufferSize];
			int offset = 0, readBytes, matches = 0;

			do
			{
				stream.Position = offset;
				readBytes = stream.Read(buffer, 0, bufferSize);

				for (int pos = 0; pos < readBytes - needle.Length + 1; pos++)
				{
					//try to find the rest of the match if the first byte matches
					int matchIndex = 0;
					while (buffer[pos + matchIndex] == needle[matchIndex])
					{
						//full match found
						if (matchIndex == needle.Length - 1)
						{
							//this is the chosen one, return the position
							if (++matches == matchNumber)
							{
								stream.Position = 0;
								return offset + pos;
							}

							break;
						}

						matchIndex++;
					}
				}

				//don't fully add readBytes, so the next string can find the full match if it started on the end of this buffer but couldn't complete
				offset += readBytes - needle.Length + 1;
			}
			while (readBytes == bufferSize);

			//no result
			stream.Position = startPosition;
			return -1;
		}

	}
}