using System; using System.IO; using System.Text; using System.Linq; using System.Collections.Generic; using System.IO.Compression; using System.Threading; using System.Diagnostics; using ICSharpCode.SharpZipLib.BZip2; using NDesk.Options; using System.Text.RegularExpressions; namespace RSCacheTool { static class Program { static string cacheDir = Environment.ExpandEnvironmentVariables(@"%USERPROFILE%\jagexcache\runescape\LIVE\"); static string outDir = "cache\\"; static int Main(string[] args) { bool error = false; bool help = false, extract = false, combine = false, overwrite = false, incomplete = false, nameMusic = false, pauseAfterDone = false; int extractArchive = -1, combineArchive = 40; string combineFile = "", nameFile = ""; OptionSet argsParser = new OptionSet() { { "h", "show this message", val => { help = true; } }, { "o", "overwrite existing files, for all actions", val => { overwrite = true; } }, { "e:", "extract files from cache, supply a number to extract only a specific archive", val => { extract = true; //set val only if it consists solely of numbers if (!String.IsNullOrWhiteSpace(val) && val.All(c => c >= '0' && c <= '9')) int.TryParse(val, out extractArchive); }}, { "c:", "combine sound, supply a number to extract from a different archive (defaults to 40)", val => { combine = true; if (!String.IsNullOrWhiteSpace(val) && val.All(c => c >= '0' && c <= '9')) int.TryParse(val, out combineArchive); }}, { "f=", "single index file (.jaga) to combine sounds of, if you want to fix just one sound", val => { combineFile = val; } }, { "i", "merge incomplete files (into special directory)", val => { incomplete = true; } }, { "n:", "try to name music (archive 40, needs archive 17 file 5 too), renames incompletes too if i is set. If a number is suplied it will only name a single file.", val => { nameMusic = true; if (!String.IsNullOrWhiteSpace(val) && val.All(c => c >= '0' && c <= '9')) nameFile = val; }}, { "p", "pause after running (mainly for easier debugging in VS)", val => { pauseAfterDone = true; } } }; List otherArgs = argsParser.Parse(args); for (int i = 0; i < otherArgs.Count; i++) { string parsedPath = otherArgs[i]; if (!parsedPath.EndsWith("\\")) parsedPath += "\\"; parsedPath = Environment.ExpandEnvironmentVariables(parsedPath); if (Directory.Exists(parsedPath)) { if (i == 0) outDir = parsedPath; else if (i == 1) cacheDir = parsedPath; } else { Console.WriteLine("The directory: " + parsedPath + " is not valid."); error = true; } } if (args.Length == 0 || help) { Console.WriteLine( "Usage: rscachetools [options] outDir cacheDir\n" + "Provides various tools for extracting and manipulating RuneScape's cache files.\n" + "\n" + "Arguments:\n" + "outDir - The directory in which all files generated by this tool will be placed. Default: cache\\\n" + "cacheDir - The directory that contains all cache files. Default: %USERPROFILE%\\jagexcache\\runescape\\LIVE\\.\n" + "\n" + "Options:" ); argsParser.WriteOptionDescriptions(Console.Out); Console.ReadLine(); } else if (!error) { //create outdir if (!Directory.Exists(outDir)) Directory.CreateDirectory(outDir); if (extract) ExtractFiles(extractArchive, overwrite); if (combine) CombineSounds(combineArchive, combineFile, overwrite, incomplete); if (nameMusic) NameMusic(nameFile, incomplete, overwrite); if (pauseAfterDone) Console.ReadLine(); } return 0; } /// /// Rips all files from the cachefile and puts them (structured and given a fitting extension where possible) in the fileDir. /// static void ExtractFiles(int archive, bool overwriteExisting) { int startArchive = 0, endArchive = 255; if (archive != -1) { startArchive = archive; endArchive = archive; } using (FileStream cacheFile = File.Open(cacheDir + "main_file_cache.dat2", FileMode.Open, FileAccess.Read)) { for (int archiveIndex = startArchive; archiveIndex <= endArchive; archiveIndex++) { string indexFileString = cacheDir + "main_file_cache.idx" + archiveIndex.ToString(); if (File.Exists(indexFileString)) { FileStream indexFile = File.Open(indexFileString, FileMode.Open, FileAccess.Read); int fileCount = (int)indexFile.Length / 6; for (int fileIndex = 0; fileIndex < fileCount; fileIndex++) { bool fileError = false; indexFile.Position = fileIndex * 6L; uint fileSize = indexFile.ReadBytes(3); long startChunkOffset = indexFile.ReadBytes(3) * 520L; //Console.WriteLine("New file: archive: {0} file: {1} offset: {3} size: {2}", archiveIndex, fileIndex, fileSize, startChunkOffset); if (fileSize > 0 && startChunkOffset > 0 && startChunkOffset + fileSize <= cacheFile.Length) { byte[] buffer = new byte[fileSize]; int writeOffset = 0; long currentChunkOffset = startChunkOffset; for (int chunkIndex = 0; writeOffset < fileSize && currentChunkOffset > 0; chunkIndex++) { cacheFile.Position = currentChunkOffset; int chunkSize; int checksumFileIndex = 0; if (fileIndex < 65536) { chunkSize = (int)Math.Min(512, fileSize - writeOffset); } else { //if file index exceeds 2 bytes, add 65536 and read 2(?) extra bytes chunkSize = (int)Math.Min(510, fileSize - writeOffset); cacheFile.ReadByte(); checksumFileIndex = (cacheFile.ReadByte() << 16); } checksumFileIndex += (int)cacheFile.ReadBytes(2); int checksumChunkIndex = (int)cacheFile.ReadBytes(2); long nextChunkOffset = cacheFile.ReadBytes(3) * 520L; int checksumArchiveIndex = cacheFile.ReadByte(); //Console.WriteLine("Chunk {2}: archive: {3} file: {1} size: {0} nextoffset: {4}", chunkSize, checksumFileIndex, checksumChunkIndex, checksumArchiveIndex, nextChunkOffset); if (checksumFileIndex == fileIndex && checksumChunkIndex == chunkIndex && checksumArchiveIndex == archiveIndex && nextChunkOffset >= 0 && nextChunkOffset < cacheFile.Length) { cacheFile.Read(buffer, writeOffset, chunkSize); writeOffset += chunkSize; currentChunkOffset = nextChunkOffset; } else { Console.WriteLine("Ignoring file because a chunk's checksum doesn't match, ideally should not happen."); fileError = true; break; } } if (!fileError) { //process file string outFileDir = outDir + archiveIndex + "\\"; string outFileName = fileIndex.ToString(); //remove the first 5 bytes because they are not part of the file byte[] tempBuffer = new byte[fileSize - 5]; Array.Copy(buffer, 5, tempBuffer, 0, fileSize - 5); buffer = tempBuffer; fileSize -= 5; //decompress gzip if (buffer.Length > 5 && (buffer[4] << 8) + buffer[5] == 0x1f8b) //gzip { //remove another 4 non-file bytes tempBuffer = new byte[fileSize - 4]; Array.Copy(buffer, 4, tempBuffer, 0, fileSize - 4); buffer = tempBuffer; fileSize -= 4; GZipStream decompressionStream = new GZipStream(new MemoryStream(buffer), CompressionMode.Decompress); int readBytes; tempBuffer = new byte[0]; do { byte[] readBuffer = new byte[100000]; readBytes = decompressionStream.Read(readBuffer, 0, 100000); int storedBytes = tempBuffer.Length; Array.Resize(ref tempBuffer, tempBuffer.Length + readBytes); Array.Copy(readBuffer, 0, tempBuffer, storedBytes, readBytes); } while (readBytes == 100000); buffer = tempBuffer; Console.WriteLine("File decompressed as gzip."); } //decompress bzip2 if (buffer.Length > 9 && buffer[4] == 0x31 && buffer[5] == 0x41 && buffer[6] == 0x59 && buffer[7] == 0x26 && buffer[8] == 0x53 && buffer[9] == 0x59) //bzip2 { //remove another 4 non-file bytes tempBuffer = new byte[fileSize - 4]; Array.Copy(buffer, 4, tempBuffer, 0, fileSize - 4); buffer = tempBuffer; fileSize -= 4; //prepend file header byte[] magic = { 0x42, 0x5a, //BZ (signature) 0x68, //h (version) 0x31 //*100kB block-size }; tempBuffer = new byte[magic.Length + buffer.Length]; magic.CopyTo(tempBuffer, 0); buffer.CopyTo(tempBuffer, magic.Length); buffer = tempBuffer; BZip2InputStream decompressionStream = new BZip2InputStream(new MemoryStream(buffer)); int readBytes; tempBuffer = new byte[0]; do { byte[] readBuffer = new byte[100000]; readBytes = decompressionStream.Read(readBuffer, 0, 100000); int storedBytes = tempBuffer.Length; Array.Resize(ref tempBuffer, tempBuffer.Length + readBytes); Array.Copy(readBuffer, 0, tempBuffer, storedBytes, readBytes); } while (readBytes == 100000); buffer = tempBuffer; Console.WriteLine("File decompressed as bzip2."); } //detect ogg: OggS if (buffer.Length > 3 && (buffer[0] << 24) + (buffer[1] << 16) + (buffer[2] << 8) + buffer[3] == 0x4f676753) outFileName += ".ogg"; //detect jaga: JAGA if (buffer.Length > 3 && (buffer[0] << 24) + (buffer[1] << 16) + (buffer[2] << 8) + buffer[3] == 0x4a414741) outFileName += ".jaga"; //detect png: .PNG if (buffer.Length > 3 && (uint)(buffer[0] << 24) + (buffer[1] << 16) + (buffer[2] << 8) + buffer[3] == 0x89504e47) outFileName += ".png"; //create and write file if (!Directory.Exists(outFileDir)) Directory.CreateDirectory(outFileDir); //(over)write file if (!File.Exists(outFileDir + outFileName) || overwriteExisting) { using (FileStream outFile = File.Open(outFileDir + outFileName, FileMode.Create, FileAccess.Write)) { outFile.Write(buffer, 0, buffer.Length); Console.WriteLine(outFileDir + outFileName); } } else Console.WriteLine("Skipping file because it already exists."); } } else { Console.WriteLine("Ignoring file because of size or offset."); } } } } } Console.WriteLine("Done extracting files."); } /// /// Combines the sound files (.jaga & .ogg) in the specified archive (40 for the build it was made on), and puts them into the soundtracks directory. /// static void CombineSounds(int archive, string file, bool overwriteExisting, bool mergeIncomplete) { string archiveDir = outDir + archive + "\\"; string soundDir = outDir + "sound\\"; //gather all index files string[] indexFiles = Directory.GetFiles(archiveDir, "*.jaga", SearchOption.TopDirectoryOnly); //create directories if (!Directory.Exists(soundDir + "incomplete\\")) Directory.CreateDirectory(soundDir + "incomplete\\"); foreach (string indexFileString in indexFiles) { string indexFileIdString = Path.GetFileNameWithoutExtension(indexFileString); //skip all others if file is set if (!String.IsNullOrWhiteSpace(file) && indexFileIdString != file) continue; bool incomplete = false; List chunkFiles = new List(); using (FileStream indexFileStream = File.Open(indexFileString, FileMode.Open, FileAccess.Read, FileShare.Read)) { indexFileStream.Position = 32L; while (indexFileStream.ReadBytes(4) != 0x4f676753) { uint fileId = indexFileStream.ReadBytes(4); //check if the file exists and add it to the buffer if it does if (File.Exists(archiveDir + fileId + ".ogg")) chunkFiles.Add(archiveDir + fileId + ".ogg"); else incomplete = true; } //make sure ~index.ogg is not still being used by SoX while (true) { try { //copy the index's audio chunk to a temp file so SoX can handle the combining using (FileStream tempIndexFile = File.Open("~index.ogg", FileMode.Create, FileAccess.Write, FileShare.None)) { indexFileStream.Position -= 4L; //include OggS indexFileStream.CopyTo(tempIndexFile); break; } } catch (IOException) { Thread.Sleep(100); } } } if (!incomplete || incomplete && mergeIncomplete) { string outFile = soundDir + (incomplete ? "incomplete\\" : "") + indexFileIdString + ".ogg"; if (!overwriteExisting && File.Exists(outFile)) Console.WriteLine("Skipping track because it already exists."); else { //combine the files with sox Console.WriteLine("Running SoX to concatenate ogg audio chunks."); Process soxProcess = new Process(); soxProcess.StartInfo.FileName = "sox.exe"; soxProcess.StartInfo.Arguments = "--combine concatenate ~index.ogg"; chunkFiles.ForEach((str) => { soxProcess.StartInfo.Arguments += " " + str; }); soxProcess.StartInfo.Arguments += " -C 6 --comment \"Created by RSCacheTool, combined by SoX.\""; soxProcess.StartInfo.Arguments += " " + soundDir + "incomplete\\" + indexFileIdString + ".ogg "; soxProcess.StartInfo.UseShellExecute = false; soxProcess.Start(); soxProcess.WaitForExit(); if (soxProcess.ExitCode == 0) { if (!incomplete) { //clear space if (File.Exists(outFile)) File.Delete(outFile); File.Move(soundDir + "incomplete\\" + indexFileIdString + ".ogg", outFile); } Console.WriteLine(outFile); } else Console.WriteLine("SoX encountered error code " + soxProcess.ExitCode + " and probably didn't finish processing the files."); } } else Console.WriteLine("Skipping track because it's incomplete."); } Console.WriteLine("Done combining sound."); } /// /// Tries to parse Archive 17 file 5 to obtain a list of music and their corresponding index file id in archive 40. /// Returns a dictionary that can resolve index file id to the name of the track as it appears in-game. /// public static void NameMusic(string file, bool incomplete, bool overwrite) { //the following is based on even more assumptions than normal made while comparing 2 extracted caches, it's therefore probably the first thing to break //4B magic number (0x00016902) - 2B a file id? - 2B amount of files (higher than actual entries sometimes) - 2B amount of files string resolveFileName = outDir + "17\\5"; if (File.Exists(resolveFileName)) { using (FileStream resolveFile = File.Open(resolveFileName, FileMode.Open, FileAccess.Read, FileShare.Read)) { Dictionary trackIdNames = new Dictionary(); Dictionary fileIdTracks = new Dictionary(); byte[] magicNumber = { 0x00, 0x01, 0x69, 0x02 }; //locate start of the music names (8th magic number) and file ids (12th) long namesStartPos = resolveFile.IndexOf(magicNumber, 8); long filesStartPos = resolveFile.IndexOf(magicNumber, 12); if (namesStartPos != -1 && filesStartPos != -1) { resolveFile.Position = namesStartPos + 8; uint musicCount = resolveFile.ReadBytes(2); //construct trackIdNames Regex regex = new Regex("[" + Regex.Escape(new string(Path.GetInvalidFileNameChars())) + "]"); for(int i = 0; i < musicCount; i++) { int trackId = (int)resolveFile.ReadBytes(2); string trackName = resolveFile.ReadNullTerminatedString(); //remove characters that can't be used in files from trackName trackName = regex.Replace(trackName, ""); //add only if the string is of any use if (!String.IsNullOrWhiteSpace(trackName)) trackIdNames.Add(trackId, trackName); } //construct fileIdTracks resolveFile.Position = filesStartPos + 8; uint fileCount = resolveFile.ReadBytes(2); for (int i = 0; i < fileCount; i++) { int trackId = (int)resolveFile.ReadBytes(2); uint fileId = resolveFile.ReadBytes(4); //only add if it doesn't exist already if (!fileIdTracks.ContainsKey(fileId)) fileIdTracks.Add(fileId, trackId); } //let's do this! if (!Directory.Exists(outDir + "sound\\named\\")) Directory.CreateDirectory(outDir + "sound\\named\\"); foreach (string soundFile in Directory.GetFiles(outDir + "sound\\")) { string fileIdString = Path.GetFileNameWithoutExtension(soundFile); if (!String.IsNullOrWhiteSpace(file) && fileIdString != file) continue; uint fileId; if (uint.TryParse(fileIdString, out fileId)) { if (fileIdTracks.ContainsKey(fileId)) { int trackId = fileIdTracks[fileId]; if (trackIdNames.ContainsKey(trackId)) { string trackName = trackIdNames[trackId]; string destFile = outDir + "sound\\named\\" + trackName + ".ogg"; if (!File.Exists(destFile) || overwrite) File.Copy(soundFile, destFile, true); Console.WriteLine(destFile); } } } } //redundancy, whatever if (incomplete) { if (!Directory.Exists(outDir + "sound\\named\\incomplete\\")) Directory.CreateDirectory(outDir + "sound\\named\\incomplete"); foreach (string soundFile in Directory.GetFiles(outDir + "sound\\incomplete\\")) { string fileIdString = Path.GetFileNameWithoutExtension(soundFile); if (!String.IsNullOrWhiteSpace(file) && fileIdString != file) continue; uint fileId; if (uint.TryParse(fileIdString, out fileId)) { if (fileIdTracks.ContainsKey(fileId)) { int trackId = fileIdTracks[fileId]; if (trackIdNames.ContainsKey(trackId)) { string trackName = trackIdNames[trackId]; string destFile = outDir + "sound\\named\\incomplete\\" + trackName + ".ogg"; if (!File.Exists(destFile) || overwrite) File.Copy(soundFile, destFile, true); Console.WriteLine(destFile); } } } } } } else Console.WriteLine("Entry points within resolving file could not be found."); } } else Console.WriteLine("File for resolving music names (" + resolveFileName + ") does not exist."); Console.WriteLine("Done naming music."); } /// /// Reads a given amount of unsigned bytes from the stream and combines them into one unsigned integer. /// public static uint ReadBytes(this Stream stream, int bytes) { if (bytes < 1 || bytes > 4) throw new ArgumentOutOfRangeException(); uint result = 0; for (int i = 0; i < bytes; i++) result += (uint)stream.ReadByte() << (bytes - i - 1) * 8; return result; } /// /// Reads ANSI characters into a string until \0 or EOF occurs. /// public static string ReadNullTerminatedString(this Stream stream) { string result = ""; int readByte = stream.ReadByte(); while (readByte > 0) { result += Encoding.Default.GetString(new byte[] { (byte)readByte }); readByte = stream.ReadByte(); } return result; } /// /// Returns the stream location of the matchNumber-th occurence of needle, or -1 when there are no(t enough) matches. /// public static long IndexOf(this Stream stream, byte[] needle, int matchNumber = 1, int bufferSize = 10000) { //for resetting after method long startPosition = stream.Position; byte[] buffer = new byte[bufferSize]; int offset = 0, readBytes, matches = 0; do { stream.Position = offset; readBytes = stream.Read(buffer, 0, bufferSize); for (int pos = 0; pos < readBytes - needle.Length + 1; pos++) { //try to find the rest of the match if the first byte matches int matchIndex = 0; while (buffer[pos + matchIndex] == needle[matchIndex]) { //full match found if (matchIndex == needle.Length - 1) { //this is the chosen one, return the position if (++matches == matchNumber) { stream.Position = 0; return offset + pos; } break; } matchIndex++; } } //don't fully add readBytes, so the next string can find the full match if it started on the end of this buffer but couldn't complete offset += readBytes - needle.Length + 1; } while (readBytes == bufferSize); //no result stream.Position = startPosition; return -1; } } }