Add recompiler for Xenia PPC tests.

CMakeLists.txt

@@ -33,3 +33,4 @@ add_subdirectory(PowerAnalyse)
 add_subdirectory(PowerRecomp)
 add_subdirectory(PowerUtils)
 add_subdirectory(PowerSample)
+add_subdirectory(PowerTests)

PowerRecomp/CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required (VERSION 3.8)
 
 project("PowerRecomp")
 
-add_executable(PowerRecomp "main.cpp" "pch.h" "recompiler.cpp" "recompiler.h" "swa_recompiler.cpp" "swa_recompiler.h")
+add_executable(PowerRecomp "main.cpp" "pch.h" "recompiler.cpp" "recompiler.h" "swa_recompiler.cpp" "swa_recompiler.h" "test_recompiler.cpp" "test_recompiler.h")
 target_precompile_headers(PowerRecomp PUBLIC "pch.h")
 target_link_libraries(PowerRecomp PRIVATE LibPowerAnalyse tomlplusplus::tomlplusplus xxHash::xxhash)

PowerRecomp/main.cpp

@@ -1,5 +1,6 @@
 #include "pch.h"
 #include "swa_recompiler.h"
+#include "test_recompiler.h"
 
 // argv 1: xex file path
 // argv 2: switches toml file path
@@ -7,18 +8,25 @@
 
 int main(int argc, char* argv[])
 {
-    SWARecompiler recompiler;
+    if (strstr(argv[1], ".xex") != nullptr)
+    {
+        SWARecompiler recompiler;
 
-    std::println("Loading executable...");
-    recompiler.LoadExecutable(argv[1]);
+        std::println("Loading executable...");
+        recompiler.LoadExecutable(argv[1]);
 
-    std::println("Loading switch tables...");
-    recompiler.LoadSwitchTables(argv[2]);
+        std::println("Loading switch tables...");
+        recompiler.LoadSwitchTables(argv[2]);
 
-    std::println("Analysing functions...");
-    recompiler.Analyse();
-    
-    recompiler.Recompile(argv[3]);
+        std::println("Analysing functions...");
+        recompiler.Analyse();
+
+        recompiler.Recompile(argv[3]);
+    }
+    else
+    {
+        TestRecompiler::RecompileTests(argv[1], argv[2]);
+    }
 
     return 0;
 }

PowerRecomp/pch.h

@@ -10,5 +10,6 @@
 #include <print>
 #include <toml++/toml.hpp>
 #include <unordered_map>
+#include <unordered_set>
 #include <xbox.h>
 #include <xxhash.h>

PowerRecomp/recompiler.cpp

@@ -368,16 +368,18 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
 
     case PPC_INST_DIVDU:
         println("\tctx.r{}.u64 = ctx.r{}.u64 / ctx.r{}.u64;", insn.operands[0], insn.operands[1], insn.operands[2]);
+        if (strchr(insn.opcode->name, '.'))
+            println("\tctx.cr0.compare<int32_t>(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]);
         break;
 
     case PPC_INST_DIVW:
-        println("\tctx.r{}.s64 = ctx.r{}.s32 / ctx.r{}.s32;", insn.operands[0], insn.operands[1], insn.operands[2]);
+        println("\tctx.r{}.s32 = ctx.r{}.s32 / ctx.r{}.s32;", insn.operands[0], insn.operands[1], insn.operands[2]);
         if (strchr(insn.opcode->name, '.'))
             println("\tctx.cr0.compare<int32_t>(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]);
         break;
 
     case PPC_INST_DIVWU:
-        println("\tctx.r{}.u64 = ctx.r{}.u32 / ctx.r{}.u32;", insn.operands[0], insn.operands[1], insn.operands[2]);
+        println("\tctx.r{}.u32 = ctx.r{}.u32 / ctx.r{}.u32;", insn.operands[0], insn.operands[1], insn.operands[2]);
         if (strchr(insn.opcode->name, '.'))
             println("\tctx.cr0.compare<int32_t>(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]);
         break;
@@ -400,6 +402,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
 
     case PPC_INST_EXTSW:
         println("\tctx.r{}.s64 = ctx.r{}.s32;", insn.operands[0], insn.operands[1]);
+        if (strchr(insn.opcode->name, '.'))
+            println("\tctx.cr0.compare<int32_t>(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]);
         break;
 
     case PPC_INST_FABS:
@@ -785,8 +789,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
         break;
 
     case PPC_INST_MFOCRF:
-        println("\tctx.r{}.u64 = (ctx.cr{}.lt << 7) | (ctx.cr{}.gt << 6) | (ctx.cr{}.eq << 5) | (ctx.cr{}.so << 4);",
-            insn.operands[0], insn.operands[1], insn.operands[1], insn.operands[1], insn.operands[1]);
+        // TODO: don't hardcode to cr6
+        println("\tctx.r{}.u64 = (ctx.cr6.lt << 7) | (ctx.cr6.gt << 6) | (ctx.cr6.eq << 5) | (ctx.cr6.so << 4);", insn.operands[0]);
         break;
 
     case PPC_INST_MFTB:
@@ -835,6 +839,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
 
     case PPC_INST_MULHWU:
         println("\tctx.r{}.u64 = (uint64_t(ctx.r{}.u32) * uint64_t(ctx.r{}.u32)) >> 32;", insn.operands[0], insn.operands[1], insn.operands[2]);
+        if (strchr(insn.opcode->name, '.'))
+            println("\tctx.cr0.compare<int32_t>(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]);
         break;
 
     case PPC_INST_MULLD:
@@ -1212,6 +1218,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
     case PPC_INST_SUBFE:
         // TODO: do we need to set the carry flag here?
         println("\tctx.r{}.u64 = ~ctx.r{}.u64 + ctx.r{}.u64 + ctx.xer.ca;", insn.operands[0], insn.operands[1], insn.operands[2]);
+        if (strchr(insn.opcode->name, '.'))
+            println("\tctx.cr0.compare<int32_t>(ctx.r{}.s32, 0, ctx.xer);", insn.operands[0]);
         break;
 
     case PPC_INST_SUBFIC:
@@ -1328,6 +1336,8 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
     case PPC_INST_VCMPEQFP128:
         println("\tctx.csr.setFlushMode(true);");
         println("\t_mm_store_ps(ctx.v{}.f32, _mm_cmpeq_ps(_mm_load_ps(ctx.v{}.f32), _mm_load_ps(ctx.v{}.f32)));", insn.operands[0], insn.operands[1], insn.operands[2]);
+        if (strchr(insn.opcode->name, '.'))
+            println("\tctx.cr6.setFromMask(_mm_load_ps(ctx.v{}.f32), 0xF);", insn.operands[0]);
         break;
 
     case PPC_INST_VCMPEQUB:
@@ -1595,10 +1605,7 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
     }
 
     case PPC_INST_VSR:
-        // TODO: vectorize
-        println("\ttemp.u64 = ctx.v{}.u8[15] & 0x7;", insn.operands[2]);
-        println("\tctx.v{}.u64[1] = (ctx.v{}.u64[0] << (64 - temp.u64)) | (ctx.v{}.u64[1] >> temp.u64);", insn.operands[0], insn.operands[1], insn.operands[1]);
-        println("\tctx.v{}.u64[0] = ctx.v{}.u64[0] >> temp.u64;", insn.operands[0], insn.operands[1]);
+        println("\t_mm_store_si128((__m128i*)ctx.v{}.u8, _mm_vsr(_mm_load_si128((__m128i*)ctx.v{}.u8), _mm_load_si128((__m128i*)ctx.v{}.u8)));", insn.operands[0], insn.operands[1], insn.operands[2]);
         break;
 
     case PPC_INST_VSRAW128:
@@ -1655,7 +1662,7 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
             for (size_t i = 0; i < 2; i++)
             {
                 println("\ttemp.f32 = 3.0f;");
-                println("\ttemp.s32 += ctx.v{}.s16[{}];", insn.operands[1], i); // TODO: not sure about the indexing here
+                println("\ttemp.s32 += ctx.v{}.s16[{}];", insn.operands[1], 1 - i);
                 println("\tvtemp.f32[{}] = temp.f32;", 3 - i);
             }
             println("\tvtemp.f32[1] = 0.0f;");
@@ -1710,19 +1717,19 @@ bool Recompiler::Recompile(const Function& fn, uint32_t base, const ppc_insn& in
         return false;
     }
 
-#if 0
+#if 1
     if (strchr(insn.opcode->name, '.'))
     {
         int lastLine = out.find_last_of('\n', out.size() - 2);
         if (out.find("ctx.cr", lastLine + 1) == std::string::npos)
-            std::println("Instruction at {:X} has RC bit enabled but no comparison was generated", base - 4);
+            std::println("{} at {:X} has RC bit enabled but no comparison was generated", insn.opcode->name, base - 4);
     }
 #endif
     
     return true;
 }
 
-void Recompiler::Recompile(const Function& fn)
+bool Recompiler::Recompile(const Function& fn)
 {
     auto base = fn.base;
     auto end = base + fn.size;
@@ -1744,6 +1751,7 @@ void Recompiler::Recompile(const Function& fn)
     println("\tuint32_t ea;\n");
 
     auto switchTable = switchTables.end();
+    bool allRecompiled = true;
 
     ppc_insn insn;
     while (base < end)
@@ -1768,11 +1776,16 @@ void Recompiler::Recompile(const Function& fn)
         else
         {
             if (!Recompile(fn, base, insn, switchTable))
+            {
                 std::println("Unrecognized instruction at 0x{:X}: {}", base - 4, insn.opcode->name);
+                allRecompiled = false;
+            }
         }
     }
 
     println("}}\n");
+
+    return allRecompiled;
 }
 
 void Recompiler::Recompile(const char* directoryPath)

PowerRecomp/recompiler.h

@@ -34,7 +34,7 @@ struct Recompiler
 
     bool Recompile(const Function& fn, uint32_t base, const ppc_insn& insn, std::unordered_map<size_t, SwitchTable>::iterator& switchTable);
 
-    void Recompile(const Function& fn);
+    bool Recompile(const Function& fn);
 
     void Recompile(const char* directoryPath);
 

PowerRecomp/test_recompiler.cpp

@@ -0,0 +1,275 @@
+#include "test_recompiler.h"
+
+void TestRecompiler::Analyse(const std::string_view& testName)
+{
+    for (const auto& section : image.sections)
+    {
+        if (!(section.flags & SectionFlags_Code))
+        {
+            continue;
+        }
+        size_t base = section.base;
+        uint8_t* data = section.data;
+        uint8_t* dataEnd = section.data + section.size;
+
+        while (data < dataEnd)
+        {
+            if (*(uint32_t*)data == 0)
+            {
+                data += 4;
+                base += 4;
+                continue;
+            }
+
+            auto& fn = functions.emplace_back(Function::Analyze(data, dataEnd - data, base));
+            image.symbols.emplace(std::format("{}_{:X}", testName, fn.base), fn.base, fn.size, Symbol_Function);
+            
+            base += fn.size;
+            data += fn.size;
+        }
+    }
+
+    std::sort(functions.begin(), functions.end(), [](auto& lhs, auto& rhs) { return lhs.base < rhs.base; });
+}
+
+void TestRecompiler::RecompileTests(const char* srcDirectoryPath, const char* dstDirectoryPath)
+{
+    std::map<std::string, std::unordered_set<size_t>> functions;
+
+    for (auto& file : std::filesystem::directory_iterator(srcDirectoryPath))
+    {
+        if (file.path().extension() == ".o")
+        {
+            TestRecompiler recompiler;
+            recompiler.LoadExecutable(file.path().string().c_str());
+            auto stem = file.path().stem().string();
+            recompiler.Analyse(stem);
+
+            recompiler.println("#include <ppc_context.h>\n");
+            recompiler.println("#define __debugbreak()\n");
+
+            for (auto& fn : recompiler.functions)
+            {
+                if (recompiler.Recompile(fn))
+                {
+                    functions[stem].emplace(fn.base);
+                }
+                else
+                {
+                    std::println("Function {:X} in {} has unimplemented instructions", fn.base, stem);
+                }
+            }
+            stem += ".cpp";
+            recompiler.SaveCurrentOutData(dstDirectoryPath, stem);
+        }
+    }
+
+    std::unordered_map<std::string, std::string> symbols;
+
+    for (auto& [fn, addr] : functions)
+    {
+        std::ifstream in(std::format("{}/{}.dis", srcDirectoryPath, fn));
+        if (in.is_open())
+        {
+            std::string line;
+            while (std::getline(in, line))
+            {
+                int spaceIndex = line.find(' ');
+                int bracketIndex = line.find('>');
+                if (spaceIndex != std::string::npos && bracketIndex != std::string::npos)
+                {
+                    size_t address = ~0;
+                    std::from_chars(&line[0], &line[spaceIndex], address, 16);
+                    address &= 0xFFFFF;
+                    if (addr.contains(address))
+                        symbols.emplace(line.substr(spaceIndex + 2, bracketIndex - spaceIndex - 2), std::format("{}_{:X}", fn, address));
+                }
+            }
+        }
+        else
+        {
+            std::println("Unable to locate disassembly file for {}", fn);
+        }
+    }
+
+    FILE* file = fopen(std::format("{}/main.cpp", dstDirectoryPath).c_str(), "w");
+    std::string main;
+
+    std::println(file, "#include <ppc_context.h>");
+    std::println(file, "#include <Windows.h>");
+    std::println(file, "#include <print>\n");
+    std::println(file, "#define PPC_CHECK_VALUE_U(lhs, rhs) if (lhs != rhs) std::println(__FUNCTION__ \" \" #lhs \" EXPECTED \" #rhs \" ACTUAL {{:X}}\", lhs)\n");
+    std::println(file, "#define PPC_CHECK_VALUE_F(lhs, rhs) if (lhs != rhs) std::println(__FUNCTION__ \" \" #lhs \" EXPECTED \" #rhs \" ACTUAL {{}}\", lhs)\n");
+
+    for (auto& [fn, addr] : functions)
+    {
+        std::ifstream in(std::format("{}/../{}.s", srcDirectoryPath, fn));
+        if (in.is_open())
+        {
+            std::string str;
+            auto getline = [&]()
+                {
+                    if (std::getline(in, str))
+                    {
+                        str.erase(str.find_last_not_of(' ') + 1);
+                        str.erase(0, str.find_first_not_of(' '));
+                        return true;
+                    }
+                    return false;
+                };
+
+            while (getline())
+            {
+                if (!str.empty() && str[0] != '#')
+                {
+                    int colonIndex = str.find(':');
+                    if (colonIndex != std::string::npos)
+                    {
+                        auto name = str.substr(0, colonIndex);
+                        auto symbol = symbols.find(name);
+                        if (symbol != symbols.end())
+                        {
+                            std::println(file, "PPC_FUNC({});\n", symbol->second);
+                            std::println(file, "void {}(uint8_t* base) {{", name);
+                            std::println(file, "\tPPCContext ctx{{}};");
+                            std::println(file, "\tctx.csr.storeValue();");
+
+                            while (getline() && !str.empty() && str[0] == '#')
+                            {
+                                if (str.size() > 1 && str[1] == '_')
+                                {
+                                    int registerInIndex = str.find("REGISTER_IN");
+                                    if (registerInIndex != std::string::npos)
+                                    {
+                                        int spaceIndex = str.find(' ', registerInIndex);
+                                        int secondSpaceIndex = str.find(' ', spaceIndex + 1);
+                                        auto reg = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1);
+                                        if (reg[0] == 'v')
+                                        {
+                                            int openingBracketIndex = str.find('[', secondSpaceIndex + 1);
+                                            int commaIndex0 = str.find(',', openingBracketIndex + 1);
+                                            int commaIndex1 = str.find(',', commaIndex0 + 1);
+                                            int commaIndex2 = str.find(',', commaIndex1 + 1);
+                                            int closingBracketIndex = str.find(']', commaIndex2 + 1);
+
+                                            std::println(file, "\tctx.{}.u32[3] = 0x{};", reg, str.substr(openingBracketIndex + 1, commaIndex0 - openingBracketIndex - 1));
+                                            std::println(file, "\tctx.{}.u32[2] = 0x{};", reg, str.substr(commaIndex0 + 2, commaIndex1 - commaIndex0 - 2));
+                                            std::println(file, "\tctx.{}.u32[1] = 0x{};", reg, str.substr(commaIndex1 + 2, commaIndex2 - commaIndex1 - 2));
+                                            std::println(file, "\tctx.{}.u32[0] = 0x{};", reg, str.substr(commaIndex2 + 2, closingBracketIndex - commaIndex2 - 2));
+                                        }
+                                        else
+                                        {
+                                            std::println(file, "\tctx.{}.{}64 = {};",
+                                                reg,
+                                                str.find('.', secondSpaceIndex) != std::string::npos ? 'f' : 'u',
+                                                str.substr(secondSpaceIndex + 1));
+                                        }
+                                    }
+                                    else
+                                    {
+                                        int memoryInIndex = str.find("MEMORY_IN");
+                                        if (memoryInIndex != std::string::npos)
+                                        {
+                                            int spaceIndex = str.find(' ', memoryInIndex);
+                                            int secondSpaceIndex = str.find(' ', spaceIndex + 1);
+                                            auto address = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1);
+                                            for (size_t i = secondSpaceIndex + 1, j = 0; i < str.size(); i++)
+                                            {
+                                                if (str[i] != ' ')
+                                                {
+                                                    std::println(file, "\tbase[0x{} + 0x{:X}] = 0x{}{};", address, j, str[i], str[i + 1]);
+                                                    ++i; // the loop adds another
+                                                    ++j;
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+
+                            while (getline() && (str.empty() || str[0] != '#'))
+                                ;
+
+                            std::println(file, "\t{}(ctx, base);", symbol->second);
+
+                            do
+                            {
+                                if (str.size() > 1 && str[1] == '_')
+                                {
+                                    int registerOutIndex = str.find("REGISTER_OUT");
+                                    if (registerOutIndex != std::string::npos)
+                                    {
+                                        int spaceIndex = str.find(' ', registerOutIndex);
+                                        int secondSpaceIndex = str.find(' ', spaceIndex + 1);
+                                        auto reg = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1);
+                                        if (reg[0] == 'c')
+                                            continue; // TODO
+                                        if (reg[0] == 'v')
+                                        {
+                                            int openingBracketIndex = str.find('[', secondSpaceIndex + 1);
+                                            int commaIndex0 = str.find(',', openingBracketIndex + 1);
+                                            int commaIndex1 = str.find(',', commaIndex0 + 1);
+                                            int commaIndex2 = str.find(',', commaIndex1 + 1);
+                                            int closingBracketIndex = str.find(']', commaIndex2 + 1);
+
+                                            std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[3], 0x{});", reg, str.substr(openingBracketIndex + 1, commaIndex0 - openingBracketIndex - 1));
+                                            std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[2], 0x{});", reg, str.substr(commaIndex0 + 2, commaIndex1 - commaIndex0 - 2));
+                                            std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[1], 0x{});", reg, str.substr(commaIndex1 + 2, commaIndex2 - commaIndex1 - 2));
+                                            std::println(file, "\tPPC_CHECK_VALUE_U(ctx.{}.u32[0], 0x{});", reg, str.substr(commaIndex2 + 2, closingBracketIndex - commaIndex2 - 2));
+                                        }
+                                        else
+                                        {
+                                            std::println(file, "\tPPC_CHECK_VALUE_{}(ctx.{}.{}64, {});",
+                                                str.find('.', secondSpaceIndex) != std::string::npos ? 'F' : 'U',
+                                                reg,
+                                                str.find('.', secondSpaceIndex) != std::string::npos ? 'f' : 'u',
+                                                str.substr(secondSpaceIndex + 1));
+                                        }
+                                    }
+                                    else
+                                    {
+                                        int memoryOutIndex = str.find("MEMORY_OUT");
+                                        if (memoryOutIndex != std::string::npos)
+                                        {
+                                            int spaceIndex = str.find(' ', memoryOutIndex);
+                                            int secondSpaceIndex = str.find(' ', spaceIndex + 1);
+                                            auto address = str.substr(spaceIndex + 1, secondSpaceIndex - spaceIndex - 1);
+                                            for (size_t i = secondSpaceIndex + 1, j = 0; i < str.size(); i++)
+                                            {
+                                                if (str[i] != ' ')
+                                                {
+                                                    std::println(file, "\tPPC_CHECK_VALUE_U(base[0x{} + 0x{:X}], 0x{}{});", address, j, str[i], str[i + 1]);
+                                                    ++i; // the loop adds another
+                                                    ++j;
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            } while (getline() && !str.empty() && str[0] == '#');
+
+                            std::println(file, "}}\n");
+
+                            std::format_to(std::back_inserter(main), "\t{}(base);\n", name);
+                        }
+                        else
+                        {
+                            std::println("Found no symbol for {}", name);
+                        }
+                    }
+                }
+            }
+        }
+        else
+        {
+            std::println("Unable to locate source file for {}", fn);
+        }
+    }
+
+    std::println(file, "void main() {{");
+    std::println(file, "\tuint8_t* base = reinterpret_cast<uint8_t*>(VirtualAlloc(nullptr, 0x100000000, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));");
+    fwrite(main.data(), 1, main.size(), file);
+    std::println(file, "}}");
+
+    fclose(file);
+}

PowerRecomp/test_recompiler.h

@@ -0,0 +1,10 @@
+#pragma once
+#include "recompiler.h"
+
+struct TestRecompiler : Recompiler
+{
+    void Analyse(const std::string_view& testName);
+    void Reset();
+    
+    static void RecompileTests(const char* srcDirectoryPath, const char* dstDirectoryPath);
+};

PowerTests/CMakeLists.txt

@@ -0,0 +1,6 @@
+project("PowerTests")
+
+file(GLOB TestFiles *.cpp)
+add_executable(PowerTests ${TestFiles})
+
+target_link_libraries(PowerTests PUBLIC PowerUtils)

PowerUtils/image.cpp

@@ -80,7 +80,7 @@ Image ElfLoadImage(const uint8_t* data, size_t size)
     for (size_t i = 0; i < numSections; i++)
     {
         const auto& section = sections[i];
-        if (section.sh_type == 0 || section.sh_addr == 0)
+        if (section.sh_type == 0)
         {
             continue;
         }

PowerUtils/ppc_context.h

@@ -533,3 +533,9 @@ inline __m128i _mm_vctsxs(__m128 a)
 
     return result;
 }
+
+inline __m128i _mm_vsr(__m128i a, __m128i b)
+{
+    b = _mm_srli_epi64(_mm_slli_epi64(b, 61), 61);
+    return _mm_castps_si128(_mm_insert_ps(_mm_castsi128_ps(_mm_srl_epi64(a, b)), _mm_castsi128_ps(_mm_srl_epi64(_mm_srli_si128(a, 4), b)), 0x10));
+}