Store the data writes of mods' ExeFS patch files into main before the game runs

An IPS, IPS32 or pchtxt file for main that writes main's .rodata, .data or .bss was refused as a whole. Such writes
need no lifting: tools/exefs_patches.py now puts them in build-cache/exefs.data, and runtime/mods.c stores them into
the loaded main before any game code runs, as emulators and Atmosphere apply a patch file to the module before its
dynamic linker runs. A DT_RELR word gets main's base added; bytes over words an explicit-addend relocation fills change
nothing and are dropped; a write into the dynamic linker's tables, a lazily bound function pointer or outside main's
image keeps the file unsupported. A mod whose data lines do not all fit the build keeps its code patches off too.

The mods are applied from the bottom one up (mods_exefs_apply), so where two mods write the same bytes the higher one
wins, as it does for files. --probe-mods checks it with two mods writing one byte.

The lifted code reads some of main's bytes at lift time (the jump tables). recomp/gen.py lists those ranges in
<gen>/main/folded.txt, and the setup checks the data writes against them after the lift: a file that writes them
becomes unsupported, since the change would not reach the generated code. Generated C without mods is unchanged.

This is the data part of #790 (ericdallo's ExeFS work), split out on its own; the plugin part follows separately.

Co-Authored-By: itaskaev2 <165683571+itaskaev2@users.noreply.github.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
authored and itaskaev2 committed 2026-11-06 15:51:42 +01:00
1 parent df9ba65a91
commit fa94e36549
13 files changed
+1001 -102

No files matched your search

+209 -33
View File
@@ -15,7 +15,9 @@
// when its build id is main's (an IPS file's name, a pchtxt file's @nsobid line); others are another version's and
// counted. The setup builds every mod's files into patch sets and writes build-cache/exefs.status next to totk.exe
// (tools/exefs_patches.py has the format): a file whose line is missing or has another content hash waits for the
// rebuild at start (cheats.c), and its mod stays off until then.
// rebuild at start (cheats.c), and its mod stays off until then. A file's writes into main's data (.rodata, .data,
// .bss) are lines of build-cache/exefs.data, stored into the loaded main before the game runs (mods_exefs_data), as
// emulators apply a patch file to the module before its dynamic linker runs.
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
@@ -170,22 +172,26 @@ static int same_build(const char *hex, size_t n) {
return 1;
}
// build-cache/exefs.status next to totk.exe, read once; "" when there is none.
static const char *exefs_status(void) {
static char *text;
if (text) return text;
// build-cache/<name> next to totk.exe, read once into *text; "" when there is none (mods_probe_cache sets the text).
static char *status_text, *data_text;
static const char *build_cache(const char *name, char **text) {
if (*text) return *text;
char path[2100];
uint64_t n = 0;
if (plat_exe_path(path, sizeof path)) {
char *cut = strrchr(path, '/'), *back = strrchr(path, '\\');
if (back > cut) cut = back;
if (cut) snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/exefs.status");
text = (char *)plat_file_read(path, &n);
if (cut && snprintf(cut + 1, sizeof path - (size_t)(cut + 1 - path), "build-cache/%s", name) <
(int)(sizeof path - (size_t)(cut + 1 - path)))
*text = (char *)plat_file_read(path, &n);
}
if (!text) text = calloc(1, 1);
return text;
if (!*text) *text = calloc(1, 1);
return *text;
}
static const char *exefs_status(void) { return build_cache("exefs.status", &status_text); }
static const char *exefs_data(void) { return build_cache("exefs.data", &data_text); }
// The status line of key: its kind, built-in sets and note, NUL-terminated copies; 0 when none.
static int status_line(const char *key, const char *content, char *kind, char *over, char *note) {
char want[20];
@@ -194,8 +200,9 @@ static int status_line(const char *key, const char *content, char *kind, char *o
const char *end = strchr(line, '\n');
size_t len = end ? (size_t)(end - line) : strlen(line);
if (len > 18 && !memcmp(line, want, 18)) {
char copy[600];
snprintf(copy, sizeof copy, "%.*s", (int)(len < sizeof copy ? len : sizeof copy - 1), line + 18);
char copy[600]; // the line after the two hashes, without the next line
size_t rest = len - 18;
snprintf(copy, sizeof copy, "%.*s", (int)(rest < sizeof copy ? rest : sizeof copy - 1), line + 18);
char *t1 = strchr(copy, '\t'), *t2 = t1 ? strchr(t1 + 1, '\t') : NULL;
if (!t1 || !t2) return 0;
*t1 = *t2 = 0;
@@ -269,9 +276,14 @@ static int exefs_file_cb(const char *leaf, int kind, uint64_t size, void *ctx) {
m->sets[m->nsets++] = (uint32_t)strtoul(key, NULL, 16);
size_t k = strlen(m->overrides);
if (*over) snprintf(m->overrides + k, sizeof m->overrides - k, "%s%s", k ? "," : "", over);
} else if (!strcmp(kind_s, "data") && m->ndata < (int)(sizeof m->data / sizeof *m->data)) {
m->data[m->ndata++] = (uint32_t)strtoul(key, NULL, 16);
} else if (!strcmp(kind_s, "other")) {
m->other_version++;
}
unsigned long bytes; // a code or data file's data writes: exefs.data must hold them all (mods_exefs_data)
if ((!strcmp(kind_s, "code") || !strcmp(kind_s, "data")) && sscanf(note, "data %lu", &bytes) == 1)
m->data_bytes += (uint32_t)bytes;
return 1;
}
@@ -326,13 +338,13 @@ static int scan_cb(const char *name, int kind, uint64_t size, void *ctx) {
if (m->cheats[0]) plat_fs_list(m->cheats, cheats_cb, &m->kind);
int unsupported = exefs_check(m, path);
// a mod whose code part the port cannot use is off as a whole: its RomFS files may expect that code
m->problem = strchr(m->name, '|') ? MOD_BAD_NAME
: m->kind & MOD_SUBSDK ? MOD_PLUGIN
: unsupported ? MOD_EXEFS_UNSUPPORTED
: m->pending ? MOD_EXEFS_LATER
: m->nsets || m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK
: m->kind & MOD_EXEFS ? MOD_EXEFS_OTHER
: MOD_EMPTY;
m->problem = strchr(m->name, '|') ? MOD_BAD_NAME
: m->kind & MOD_SUBSDK ? MOD_PLUGIN
: unsupported ? MOD_EXEFS_UNSUPPORTED
: m->pending ? MOD_EXEFS_LATER
: m->nsets || m->ndata || m->kind & (MOD_ROMFS | MOD_CHEATS) ? MOD_OK
: m->kind & MOD_EXEFS ? MOD_EXEFS_OTHER
: MOD_EMPTY;
m->enabled = !listed(disabled, m->name);
nmods++;
return 1;
@@ -376,10 +388,11 @@ void mods_peek_romfs(void (*cb)(const char *name, const char *romfs, void *ctx),
plat_fs_list(folder, peek_cb, &pk);
}
static int scanned;
void mods_scan(void) {
static int done;
if (done) return;
done = 1;
if (scanned) return;
scanned = 1;
if (!mods_folder(dir, sizeof dir)) return;
if (!plat_dir_exists(dir)) {
rt_log("mods: no folder %s", dir);
@@ -403,18 +416,171 @@ void mods_scan(void) {
rt_log("mods: %d folders in %s", nmods, dir);
}
void mods_exefs_switch(void) {
mods_scan();
for (int i = 0; i < nmods; i++) {
const ModInfo *m = &mods[i];
if (!start_enabled[i] || m->problem != MOD_OK || !m->nsets) continue;
for (int k = 0; k < m->nsets; k++) {
char name[32];
snprintf(name, sizeof name, "mod_%08x", m->sets[k]);
int set = rt_patch_set(name);
if (set >= 0) rt_patch_switch(set, 1);
else rt_log("mods: %s: patch set %s is not in this build", m->name, name);
// ---- ExeFS data writes (build-cache/exefs.data; tools/exefs_patches.py data_line has the format)
// 1 when key names one of m's usable ExeFS patch files: one with a patch set, or one with data writes only.
static int owns(const ModInfo *m, uint32_t key) {
for (int k = 0; k < m->nsets; k++)
if (m->sets[k] == key) return 1;
for (int k = 0; k < m->ndata; k++)
if (m->data[k] == key) return 1;
return 0;
}
static int hex_digit(char c) {
return c >= '0' && c <= '9' ? c - '0'
: c >= 'a' && c <= 'f' ? c - 'a' + 10
: c >= 'A' && c <= 'F' ? c - 'A' + 10
: -1;
}
// One line: "<file hash>\t<content hash>\t<offset in main>\t<bytes>[\tbase]", all hex. 0 when malformed.
typedef struct { char key[9], content[9]; uint64_t off; uint8_t bytes[256]; int n, base; } DataRun;
static int parse_run(const char *line, size_t len, DataRun *r) {
char copy[1100], *f[6];
if (len >= sizeof copy) return 0;
memcpy(copy, line, len);
copy[len] = 0;
int nf = 0;
for (char *p = copy; p && nf < 6; nf++) {
f[nf] = p;
p = strchr(p, '\t');
if (p) *p++ = 0;
}
if (nf < 4 || nf > 5 || strlen(f[0]) != 8 || strlen(f[1]) != 8) return 0;
memcpy(r->key, f[0], 9);
memcpy(r->content, f[1], 9);
char *end;
r->off = strtoull(f[2], &end, 16);
size_t hex = strlen(f[3]);
if (!*f[2] || *end || !hex || hex % 2 || hex / 2 > sizeof r->bytes) return 0;
r->n = (int)(hex / 2);
for (int i = 0; i < r->n; i++) {
int hi = hex_digit(f[3][2 * i]), lo = hex_digit(f[3][2 * i + 1]);
if (hi < 0 || lo < 0) return 0;
r->bytes[i] = (uint8_t)(hi << 4 | lo);
}
r->base = nf == 5;
return nf == 4 || (!strcmp(f[4], "base") && r->n == 8);
}
// Stores n bytes into the loaded main at a: a read-only page (.rodata, loader.c seal) is writable for its part of the
// copy only, as cheats.c wr_main does. 0 when a page is neither readable nor writable.
static int store_main(uint64_t a, const uint8_t *from, int n) {
while (n > 0) {
uint64_t page = a & ~0xFFFull, end;
int k = (int)(page + 0x1000 - a);
if (k > n) k = n;
int prot = plat_mem_state(page, page + 0x1000, &end);
if (prot != PLAT_PROT_READ && prot != PLAT_PROT_RW) return 0;
if (prot == PLAT_PROT_READ && !plat_mem_protect(page, 0x1000, PLAT_PROT_RW)) return 0;
memcpy((void *)(uintptr_t)a, from, (size_t)k);
if (prot == PLAT_PROT_READ) plat_mem_protect(page, 0x1000, PLAT_PROT_READ);
a += (uint64_t)k;
from += k;
n -= k;
}
return 1;
}
long mods_exefs_data(const ModInfo *m, uint64_t base, uint64_t size, uint64_t code_end,
int (*store)(uint64_t addr, const uint8_t *bytes, int n)) {
if (!store) store = store_main;
long total = 0, found = 0;
for (int pass = 0; pass < 2; pass++) { // pass 0 checks every line, pass 1 stores
// exefs.data must hold every byte the status lines count: one missing or cut short would leave the code
// patches on without their data
if (pass && found != (long)m->data_bytes) {
rt_log("mods: %s: build-cache/exefs.data holds %ld of the %u bytes its ExeFS patch files write", m->name,
found, m->data_bytes);
return -1;
}
for (const char *line = exefs_data(), *next; *line; line = next) {
const char *nl = strchr(line, '\n');
size_t len = nl ? (size_t)(nl - line) : strlen(line);
next = nl ? nl + 1 : line + len;
if (len && line[len - 1] == '\r') len--;
char key[9];
snprintf(key, sizeof key, "%.*s", (int)(len < 8 ? len : 8), line);
if (!len || *line == '#' || !owns(m, (uint32_t)strtoul(key, NULL, 16))) continue;
DataRun r;
char kind[16], over[256], note[256];
// a line whose file the status does not list as built (another content) or that leaves main's data: the
// two build-cache files and this build disagree
if (!parse_run(line, len, &r) || !status_line(r.key, r.content, kind, over, note) ||
(strcmp(kind, "code") && strcmp(kind, "data")) || r.off < code_end || r.off > size ||
(uint64_t)r.n > size - r.off) {
rt_log("mods: %s: a line of build-cache/exefs.data does not fit this build: %.*s", m->name, (int)len,
line);
return -1;
}
if (!pass) {
found += r.n;
continue;
}
if (r.base) { // a DT_RELR word holds its addend; the dynamic linker adds the module's base
uint64_t w;
memcpy(&w, r.bytes, 8);
w += base;
memcpy(r.bytes, &w, 8);
}
if (!store(base + r.off, r.bytes, r.n)) {
rt_log("mods: %s: cannot write main+%#llx", m->name, (unsigned long long)r.off);
return -1;
}
total += r.n;
}
}
return total;
}
static char *dup_text(const char *s) {
if (!s) return NULL;
size_t n = strlen(s) + 1;
char *d = malloc(n);
if (d) memcpy(d, s, n);
return d;
}
void mods_probe_cache(const char *status, const char *data) {
free(status_text);
free(data_text);
status_text = dup_text(status);
data_text = dup_text(data);
}
void mods_rescan(void) {
free(mods);
mods = NULL;
nmods = 0;
scanned = 0;
}
// Switches the patch set mod_<key> on; 0 when this build lacks it (logged).
static int set_on(const ModInfo *m, uint32_t key) {
char name[32];
snprintf(name, sizeof name, "mod_%08x", key);
int set = rt_patch_set(name);
if (set >= 0) rt_patch_switch(set, 1);
else rt_log("mods: %s: patch set %s is not in this build", m->name, name);
return set >= 0;
}
void mods_exefs_apply(uint64_t base, uint64_t size, uint64_t code_end,
int (*store)(uint64_t addr, const uint8_t *bytes, int n)) {
// bottom first: the top mod's data writes land last and win, as its RomFS files do. Patch sets are only switches
// (rt_patch_switch), so their order does not matter.
for (int i = nmods - 1; i >= 0; i--) {
const ModInfo *m = &mods[i];
if (!start_enabled[i] || m->problem != MOD_OK || !(m->nsets || m->ndata)) continue;
// the data first: a mod whose data writes cannot all be stored keeps its code patches off too, as half of a
// patch file is worse than none
long stored = size ? mods_exefs_data(m, base, size, code_end, store) : 0;
if (stored < 0) {
rt_log("mods: %s: its ExeFS patch files stay off: their data writes could not all be stored", m->name);
continue;
}
for (int k = 0; k < m->nsets; k++) set_on(m, m->sets[k]);
char over[256];
snprintf(over, sizeof over, "%s", m->overrides);
for (char *s = strtok(over, ","); s; s = strtok(NULL, ",")) {
@@ -422,10 +588,20 @@ void mods_exefs_switch(void) {
if (set >= 0) rt_patch_switch(set, 0);
rt_log("mods: %s replaces the port's patch set %s; it stays off", m->name, s);
}
rt_log("mods: %s: %d ExeFS patch files on", m->name, m->nsets);
rt_log("mods: %s: %d ExeFS patch files on, %ld bytes of main's data stored", m->name, m->nsets + m->ndata,
stored);
}
}
void mods_exefs_switch(void) {
mods_scan();
const RtLoaded *main_mod = NULL;
for (int i = 0; ldr_module(i); i++)
if (!strcmp(ldr_module(i)->name, "main")) main_mod = ldr_module(i);
if (main_mod) mods_exefs_apply(main_mod->base, main_mod->size, main_mod->text_end, NULL);
else mods_exefs_apply(0, 0, 0, NULL);
}
int mods_exefs_pending(uint32_t *key) {
int n = 0;
for (int i = 0; i < nmods; i++) {