From 4d6eac1c24eb311398b2b4415fe17245a67e1141 Mon Sep 17 00:00:00 2001 From: Skyth <19259897+blueskythlikesclouds@users.noreply.github.com> Date: Sat, 21 Sep 2024 21:47:34 +0300 Subject: [PATCH] Add recompiler for Xenia PPC tests. --- CMakeLists.txt | 1 + PowerRecomp/CMakeLists.txt | 2 +- PowerRecomp/main.cpp | 26 +-- PowerRecomp/pch.h | 1 + PowerRecomp/recompiler.cpp | 37 +++-- PowerRecomp/recompiler.h | 2 +- PowerRecomp/test_recompiler.cpp | 275 ++++++++++++++++++++++++++++++++ PowerRecomp/test_recompiler.h | 10 ++ PowerTests/CMakeLists.txt | 6 + PowerUtils/image.cpp | 2 +- PowerUtils/ppc_context.h | 6 + 11 files changed, 344 insertions(+), 24 deletions(-) create mode 100644 PowerRecomp/test_recompiler.cpp create mode 100644 PowerRecomp/test_recompiler.h create mode 100644 PowerTests/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 07f7d0f..a91d37e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,3 +33,4 @@ add_subdirectory(PowerAnalyse) add_subdirectory(PowerRecomp) add_subdirectory(PowerUtils) add_subdirectory(PowerSample) +add_subdirectory(PowerTests) diff --git a/PowerRecomp/CMakeLists.txt b/PowerRecomp/CMakeLists.txt index 1754753..c170c24 100644 --- a/PowerRecomp/CMakeLists.txt +++ b/PowerRecomp/CMakeLists.txt @@ -2,6 +2,6 @@ cmake_minimum_required (VERSION 3.8) project("PowerRecomp") -add_executable(PowerRecomp "main.cpp" "pch.h" "recompiler.cpp" "recompiler.h" "swa_recompiler.cpp" "swa_recompiler.h") +add_executable(PowerRecomp "main.cpp" "pch.h" "recompiler.cpp" "recompiler.h" "swa_recompiler.cpp" "swa_recompiler.h" "test_recompiler.cpp" "test_recompiler.h") target_precompile_headers(PowerRecomp PUBLIC "pch.h") target_link_libraries(PowerRecomp PRIVATE LibPowerAnalyse tomlplusplus::tomlplusplus xxHash::xxhash) diff --git a/PowerRecomp/main.cpp b/PowerRecomp/main.cpp index 981bd29..9fab12a 100644 --- a/PowerRecomp/main.cpp +++ b/PowerRecomp/main.cpp @@ -1,5 +1,6 @@ #include "pch.h" #include "swa_recompiler.h" +#include "test_recompiler.h" // argv 1: xex file path // argv 2: switches toml file path @@ -7,18 +8,25 @@ int main(int argc, char* argv[]) { - SWARecompiler recompiler; + if (strstr(argv[1], ".xex") != nullptr) + { + SWARecompiler recompiler; - std::println("Loading executable..."); - recompiler.LoadExecutable(argv[1]); + std::println("Loading executable..."); + recompiler.LoadExecutable(argv[1]); - std::println("Loading switch tables..."); - recompiler.LoadSwitchTables(argv[2]); + std::println("Loading switch tables..."); + recompiler.LoadSwitchTables(argv[2]); - std::println("Analysing functions..."); - recompiler.Analyse(); - - recompiler.Recompile(argv[3]); + std::println("Analysing functions..."); + recompiler.Analyse(); + + recompiler.Recompile(argv[3]); + } + else + { + TestRecompiler::RecompileTests(argv[1], argv[2]); + } return 0; } diff --git a/PowerRecomp/pch.h b/PowerRecomp/pch.h index 7181e63..b1b9b75 100644 --- a/PowerRecomp/pch.h +++ b/PowerRecomp/pch.h @@ -10,5 +10,6 @@ #include #include #include +#include #include #include diff --git a/PowerRecomp/recompiler.cpp b/PowerRecomp/recompiler.cpp index 365fd0a..cdac7d7 100644 --- a/PowerRecomp/recompiler.cpp +++ b/PowerRecomp/recompiler.cpp @@ -368,16 +368,18 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in case PPC_INST_DIVDU: println("\tctx.r{}.u64 = ctx.r{}.u64 / ctx.r{}.u64;", insn.operands[0], insn.operands[1], insn.operands[2]); + if (strchr(insn.opcode->name, '.')) + println("\tctx.cr0.compare(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]); break; case PPC_INST_DIVW: - println("\tctx.r{}.s64 = ctx.r{}.s32 / ctx.r{}.s32;", insn.operands[0], insn.operands[1], insn.operands[2]); + println("\tctx.r{}.s32 = ctx.r{}.s32 / ctx.r{}.s32;", insn.operands[0], insn.operands[1], insn.operands[2]); if (strchr(insn.opcode->name, '.')) println("\tctx.cr0.compare(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]); break; case PPC_INST_DIVWU: - println("\tctx.r{}.u64 = ctx.r{}.u32 / ctx.r{}.u32;", insn.operands[0], insn.operands[1], insn.operands[2]); + println("\tctx.r{}.u32 = ctx.r{}.u32 / ctx.r{}.u32;", insn.operands[0], insn.operands[1], insn.operands[2]); if (strchr(insn.opcode->name, '.')) println("\tctx.cr0.compare(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]); break; @@ -400,6 +402,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in case PPC_INST_EXTSW: println("\tctx.r{}.s64 = ctx.r{}.s32;", insn.operands[0], insn.operands[1]); + if (strchr(insn.opcode->name, '.')) + println("\tctx.cr0.compare(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]); break; case PPC_INST_FABS: @@ -785,8 +789,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in break; case PPC_INST_MFOCRF: - println("\tctx.r{}.u64 = (ctx.cr{}.lt << 7) | (ctx.cr{}.gt << 6) | (ctx.cr{}.eq << 5) | (ctx.cr{}.so << 4);", - insn.operands[0], insn.operands[1], insn.operands[1], insn.operands[1], insn.operands[1]); + // TODO: don't hardcode to cr6 + println("\tctx.r{}.u64 = (ctx.cr6.lt << 7) | (ctx.cr6.gt << 6) | (ctx.cr6.eq << 5) | (ctx.cr6.so << 4);", insn.operands[0]); break; case PPC_INST_MFTB: @@ -835,6 +839,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in case PPC_INST_MULHWU: println("\tctx.r{}.u64 = (uint64_t(ctx.r{}.u32) * uint64_t(ctx.r{}.u32)) >> 32;", insn.operands[0], insn.operands[1], insn.operands[2]); + if (strchr(insn.opcode->name, '.')) + println("\tctx.cr0.compare(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]); break; case PPC_INST_MULLD: @@ -1212,6 +1218,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in case PPC_INST_SUBFE: // TODO: do we need to set the carry flag here? println("\tctx.r{}.u64 = ~ctx.r{}.u64 + ctx.r{}.u64 + ctx.xer.ca;", insn.operands[0], insn.operands[1], insn.operands[2]); + if (strchr(insn.opcode->name, '.')) + println("\tctx.cr0.compare(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]); break; case PPC_INST_SUBFIC: @@ -1328,6 +1336,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in case PPC_INST_VCMPEQFP128: println("\tctx.csr.setFlushMode(true);"); println("\t_mm_store_ps(ctx.v{}.f32, _mm_cmpeq_ps(_mm_load_ps(ctx.v{}.f32), _mm_load_ps(ctx.v{}.f32)));", insn.operands[0], insn.operands[1], insn.operands[2]); + if (strchr(insn.opcode->name, '.')) + println("\tctx.cr6.setFromMask(_mm_load_ps(ctx.v{}.f32), 0xF);", insn.operands[0]); break; case PPC_INST_VCMPEQUB: @@ -1595,10 +1605,7 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in } case PPC_INST_VSR: - // TODO: vectorize - println("\ttemp.u64 = ctx.v{}.u8[15] & 0x7;", insn.operands[2]); - println("\tctx.v{}.u64[1] = (ctx.v{}.u64[0] << (64 - temp.u64)) | (ctx.v{}.u64[1] >> temp.u64);", insn.operands[0], insn.operands[1], insn.operands[1]); - println("\tctx.v{}.u64[0] = ctx.v{}.u64[0] >> temp.u64;", insn.operands[0], insn.operands[1]); + println("\t_mm_store_si128((__m128i*)ctx.v{}.u8, _mm_vsr(_mm_load_si128((__m128i*)ctx.v{}.u8), _mm_load_si128((__m128i*)ctx.v{}.u8)));", insn.operands[0], insn.operands[1], insn.operands[2]); break; case PPC_INST_VSRAW128: @@ -1655,7 +1662,7 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in for (size_t i = 0; i < 2; i++) { println("\ttemp.f32 = 3.0f;"); - println("\ttemp.s32 += ctx.v{}.s16[{}];", insn.operands[1], i); // TODO: not sure about the indexing here + println("\ttemp.s32 += ctx.v{}.s16[{}];", insn.operands[1], 1 - i); println("\tvtemp.f32[{}] = temp.f32;", 3 - i); } println("\tvtemp.f32[1] = 0.0f;"); @@ -1710,19 +1717,19 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in return false; } -#if 0 +#if 1 if (strchr(insn.opcode->name, '.')) { int lastLine = out.find_last_of('\n', out.size() - 2); if (out.find("ctx.cr", lastLine + 1) == std::string::npos) - std::println("Instruction at {:X} has RC bit enabled but no comparison was generated", base - 4); + std::println("{} at {:X} has RC bit enabled but no comparison was generated", insn.opcode->name, base - 4); } #endif return true; } -void Recompiler::Recompile(const Function& fn) +bool Recompiler::Recompile(const Function& fn) { auto base = fn.base; auto end = base + fn.size; @@ -1744,6 +1751,7 @@ void Recompiler::Recompile(const Function& fn) println("\tuint32_t ea;\n"); auto switchTable = switchTables.end(); + bool allRecompiled = true; ppc_insn insn; while (base < end) @@ -1768,11 +1776,16 @@ void Recompiler::Recompile(const Function& fn) else { if (!Recompile(fn, base, insn, switchTable)) + { std::println("Unrecognized instruction at 0x{:X}: {}", base - 4, insn.opcode->name); + allRecompiled = false; + } } } println("}}\n"); + + return allRecompiled; } void Recompiler::Recompile(const char* directoryPath) diff --git a/PowerRecomp/recompiler.h b/PowerRecomp/recompiler.h index 13f63e4..366fcbe 100644 --- a/PowerRecomp/recompiler.h +++ b/PowerRecomp/recompiler.h @@ -34,7 +34,7 @@ struct Recompiler bool Recompile(const Function& fn, uint32_t base, const ppc_insn& insn, std::unordered_map::iterator& switchTable); - void Recompile(const Function& fn); + bool Recompile(const Function& fn); void Recompile(const char* directoryPath); diff --git a/PowerRecomp/test_recompiler.cpp b/PowerRecomp/test_recompiler.cpp new file mode 100644 index 0000000..665eae4 --- /dev/null +++ b/PowerRecomp/test_recompiler.cpp @@ -0,0 +1,275 @@ +#include "test_recompiler.h" + +void TestRecompiler::Analyse(const std::string_view& testName) +{ + for (const auto& section : image.sections) + { + if (!(section.flags & SectionFlags_Code)) + { + continue; + } + size_t base = section.base; + uint8_t* data = section.data; + uint8_t* dataEnd = section.data + section.size; + + while (data < dataEnd) + { + if (*(uint32_t*)data == 0) + { + data += 4; + base += 4; + continue; + } + + auto& fn = functions.emplace_back(Function::Analyze(data, dataEnd - data, base)); + image.symbols.emplace(std::format("{}_{:X}", testName, fn.base), fn.base, fn.size, Symbol_Function); + + base += fn.size; + data += fn.size; + } + } + + std::sort(functions.begin(), functions.end(), [](auto& lhs, auto& rhs) { return lhs.base < rhs.base; }); +} + +void TestRecompiler::RecompileTests(const char* srcDirectoryPath, const char* dstDirectoryPath) +{ + std::map> functions; + + for (auto& file : std::filesystem::directory_iterator(srcDirectoryPath)) + { + if (file.path().extension() == ".o") + { + TestRecompiler recompiler; + recompiler.LoadExecutable(file.path().string().c_str()); + auto stem = file.path().stem().string(); + recompiler.Analyse(stem); + + recompiler.println("#include \n"); + recompiler.println("#define __debugbreak()\n"); + + for (auto& fn : recompiler.functions) + { + if (recompiler.Recompile(fn)) + { + functions[stem].emplace(fn.base); + } + else + { + std::println("Function {:X} in {} has unimplemented instructions", fn.base, stem); + } + } + stem += ".cpp"; + recompiler.SaveCurrentOutData(dstDirectoryPath, stem); + } + } + + std::unordered_map symbols; + + for (auto& [fn, addr] : functions) + { + std::ifstream in(std::format("{}/{}.dis", srcDirectoryPath, fn)); + if (in.is_open()) + { + std::string line; + while (std::getline(in, line)) + { + int spaceIndex = line.find(' '); + int bracketIndex = line.find('>'); + if (spaceIndex != std::string::npos && bracketIndex != std::string::npos) + { + size_t address = ~0; + std::from_chars(&line[0], &line[spaceIndex], address, 16); + address &= 0xFFFFF; + if (addr.contains(address)) + symbols.emplace(line.substr(spaceIndex + 2, bracketIndex - spaceIndex - 2), std::format("{}_{:X}", fn, address)); + } + } + } + else + { + std::println("Unable to locate disassembly file for {}", fn); + } + } + + FILE* file = fopen(std::format("{}/main.cpp", dstDirectoryPath).c_str(), "w"); + std::string main; + + std::println(file, "#include "); + std::println(file, "#include "); + std::println(file, "#include \n"); + std::println(file, "#define PPC_CHECK_VALUE_U(lhs, rhs) if (lhs != rhs) std::println(__FUNCTION__ \" \" #lhs \" EXPECTED \" #rhs \" ACTUAL {{:X}}\", lhs)\n"); + std::println(file, "#define PPC_CHECK_VALUE_F(lhs, rhs) if (lhs != rhs) std::println(__FUNCTION__ \" \" #lhs \" EXPECTED \" #rhs \" ACTUAL {{}}\", lhs)\n"); + + for (auto& [fn, addr] : functions) + { + std::ifstream in(std::format("{}/../{}.s", srcDirectoryPath, fn)); + if (in.is_open()) + { + std::string str; + auto getline = [&]() + { + if (std::getline(in, str)) + { + str.erase(str.find_last_not_of(' ') + 1); + str.erase(0, str.find_first_not_of(' ')); + return true; + } + return false; + }; + + while (getline()) + { + if (!str.empty() && str[0] != '#') + { + int colonIndex = str.find(':'); + if (colonIndex != std::string::npos) + { + auto name = str.substr(0, colonIndex); + auto symbol = symbols.find(name); + if (symbol != symbols.end()) + { + std::println(file, "PPC_FUNC({});\n", symbol->second); + std::println(file, "void {}(uint8_t* base) {{", name); + std::println(file, "\tPPCContext ctx{{}};"); + std::println(file, "\tctx.csr.storeValue();"); + + while (getline() && !str.empty() && str[0] == '#') + { + if (str.size() > 1 && str[1] == '_') + { + int registerInIndex = str.find("REGISTER_IN"); + if (registerInIndex != std::string::npos) + { + int spaceIndex = str.find(' ', registerInIndex); + int secondSpaceIndex = str.find(' ', spaceIndex + 1); + auto reg = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1); + if (reg[0] == 'v') + { + int openingBracketIndex = str.find('[', secondSpaceIndex + 1); + int commaIndex0 = str.find(',', openingBracketIndex + 1); + int commaIndex1 = str.find(',', commaIndex0 + 1); + int commaIndex2 = str.find(',', commaIndex1 + 1); + int closingBracketIndex = str.find(']', commaIndex2 + 1); + + std::println(file, "\tctx.{}.u32[3] = 0x{};", reg, str.substr(openingBracketIndex + 1, commaIndex0 - openingBracketIndex - 1)); + std::println(file, "\tctx.{}.u32[2] = 0x{};", reg, str.substr(commaIndex0 + 2, commaIndex1 - commaIndex0 - 2)); + std::println(file, "\tctx.{}.u32[1] = 0x{};", reg, str.substr(commaIndex1 + 2, commaIndex2 - commaIndex1 - 2)); + std::println(file, "\tctx.{}.u32[0] = 0x{};", reg, str.substr(commaIndex2 + 2, closingBracketIndex - commaIndex2 - 2)); + } + else + { + std::println(file, "\tctx.{}.{}64 = {};", + reg, + str.find('.', secondSpaceIndex) != std::string::npos ? 'f' : 'u', + str.substr(secondSpaceIndex + 1)); + } + } + else + { + int memoryInIndex = str.find("MEMORY_IN"); + if (memoryInIndex != std::string::npos) + { + int spaceIndex = str.find(' ', memoryInIndex); + int secondSpaceIndex = str.find(' ', spaceIndex + 1); + auto address = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1); + for (size_t i = secondSpaceIndex + 1, j = 0; i < str.size(); i++) + { + if (str[i] != ' ') + { + std::println(file, "\tbase[0x{} + 0x{:X}] = 0x{}{};", address, j, str[i], str[i + 1]); + ++i; // the loop adds another + ++j; + } + } + } + } + } + } + + while (getline() && (str.empty() || str[0] != '#')) + ; + + std::println(file, "\t{}(ctx, base);", symbol->second); + + do + { + if (str.size() > 1 && str[1] == '_') + { + int registerOutIndex = str.find("REGISTER_OUT"); + if (registerOutIndex != std::string::npos) + { + int spaceIndex = str.find(' ', registerOutIndex); + int secondSpaceIndex = str.find(' ', spaceIndex + 1); + auto reg = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1); + if (reg[0] == 'c') + continue; // TODO + if (reg[0] == 'v') + { + int openingBracketIndex = str.find('[', secondSpaceIndex + 1); + int commaIndex0 = str.find(',', openingBracketIndex + 1); + int commaIndex1 = str.find(',', commaIndex0 + 1); + int commaIndex2 = str.find(',', commaIndex1 + 1); + int closingBracketIndex = str.find(']', commaIndex2 + 1); + + std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[3], 0x{});", reg, str.substr(openingBracketIndex + 1, commaIndex0 - openingBracketIndex - 1)); + std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[2], 0x{});", reg, str.substr(commaIndex0 + 2, commaIndex1 - commaIndex0 - 2)); + std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[1], 0x{});", reg, str.substr(commaIndex1 + 2, commaIndex2 - commaIndex1 - 2)); + std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[0], 0x{});", reg, str.substr(commaIndex2 + 2, closingBracketIndex - commaIndex2 - 2)); + } + else + { + std::println(file, "\tPPC_CHECK_VALUE_{}(ctx.{}.{}64, {});", + str.find('.', secondSpaceIndex) != std::string::npos ? 'F' : 'U', + reg, + str.find('.', secondSpaceIndex) != std::string::npos ? 'f' : 'u', + str.substr(secondSpaceIndex + 1)); + } + } + else + { + int memoryOutIndex = str.find("MEMORY_OUT"); + if (memoryOutIndex != std::string::npos) + { + int spaceIndex = str.find(' ', memoryOutIndex); + int secondSpaceIndex = str.find(' ', spaceIndex + 1); + auto address = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1); + for (size_t i = secondSpaceIndex + 1, j = 0; i < str.size(); i++) + { + if (str[i] != ' ') + { + std::println(file, "\tPPC_CHECK_VALUE_U(base[0x{} + 0x{:X}], 0x{}{});", address, j, str[i], str[i + 1]); + ++i; // the loop adds another + ++j; + } + } + } + } + } + } while (getline() && !str.empty() && str[0] == '#'); + + std::println(file, "}}\n"); + + std::format_to(std::back_inserter(main), "\t{}(base);\n", name); + } + else + { + std::println("Found no symbol for {}", name); + } + } + } + } + } + else + { + std::println("Unable to locate source file for {}", fn); + } + } + + std::println(file, "void main() {{"); + std::println(file, "\tuint8_t* base = reinterpret_cast(VirtualAlloc(nullptr, 0x100000000, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));"); + fwrite(main.data(), 1, main.size(), file); + std::println(file, "}}"); + + fclose(file); +} diff --git a/PowerRecomp/test_recompiler.h b/PowerRecomp/test_recompiler.h new file mode 100644 index 0000000..2ef368c --- /dev/null +++ b/PowerRecomp/test_recompiler.h @@ -0,0 +1,10 @@ +#pragma once +#include "recompiler.h" + +struct TestRecompiler : Recompiler +{ + void Analyse(const std::string_view& testName); + void Reset(); + + static void RecompileTests(const char* srcDirectoryPath, const char* dstDirectoryPath); +}; diff --git a/PowerTests/CMakeLists.txt b/PowerTests/CMakeLists.txt new file mode 100644 index 0000000..36e9af3 --- /dev/null +++ b/PowerTests/CMakeLists.txt @@ -0,0 +1,6 @@ +project("PowerTests") + +file(GLOB TestFiles *.cpp) +add_executable(PowerTests ${TestFiles}) + +target_link_libraries(PowerTests PUBLIC PowerUtils) diff --git a/PowerUtils/image.cpp b/PowerUtils/image.cpp index 7a88bd0..5be2475 100644 --- a/PowerUtils/image.cpp +++ b/PowerUtils/image.cpp @@ -80,7 +80,7 @@ Image ElfLoadImage(const uint8_t* data, size_t size) for (size_t i = 0; i < numSections; i++) { const auto& section = sections[i]; - if (section.sh_type == 0 || section.sh_addr == 0) + if (section.sh_type == 0) { continue; } diff --git a/PowerUtils/ppc_context.h b/PowerUtils/ppc_context.h index 752ef69..66e288f 100644 --- a/PowerUtils/ppc_context.h +++ b/PowerUtils/ppc_context.h @@ -533,3 +533,9 @@ inline __m128i _mm_vctsxs(__m128 a) return result; } + +inline __m128i _mm_vsr(__m128i a, __m128i b) +{ + b = _mm_srli_epi64(_mm_slli_epi64(b, 61), 61); + return _mm_castps_si128(_mm_insert_ps(_mm_castsi128_ps(_mm_srl_epi64(a, b)), _mm_castsi128_ps(_mm_srl_epi64(_mm_srli_si128(a, 4), b)), 0x10)); +}