Skip to content

require() resolution cache #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 68 additions & 3 deletions load.c
Original file line number Diff line number Diff line change
Expand Up @@ -944,13 +944,17 @@ rb_require_safe(VALUE fname, int safe)
{
volatile VALUE result = Qnil;
rb_thread_t *th = GET_THREAD();
rb_vm_t *vm = GET_VM();
volatile VALUE errinfo = th->errinfo;
int state;
struct {
int safe;
} volatile saved;
char *volatile ftptr = 0;

if (strncmp(StringValuePtr(fname), "enumerator", 11) == 0)
return Qfalse;

if (RUBY_DTRACE_REQUIRE_ENTRY_ENABLED()) {
RUBY_DTRACE_REQUIRE_ENTRY(StringValuePtr(fname),
rb_sourcefile(),
Expand All @@ -960,9 +964,11 @@ rb_require_safe(VALUE fname, int safe)
PUSH_TAG();
saved.safe = rb_safe_level();
if ((state = EXEC_TAG()) == 0) {
VALUE path;
VALUE ipath, path = 0;
long handle;
int found;
st_data_t key, val;
char *vals;

rb_set_safe_level_force(safe);
FilePathValue(fname);
Expand All @@ -974,8 +980,36 @@ rb_require_safe(VALUE fname, int safe)
rb_sourceline());
}

path = rb_str_encode_ospath(fname);
found = search_required(path, &path, safe);
ipath = rb_str_encode_ospath(fname);
if (vm->require_cache.read && RSTRING_PTR(ipath)[0] != '/') {
key = (st_data_t)RSTRING_PTR(ipath);
if (st_lookup(vm->require_cache.tbl, key, &val)) {
vals = (char *)val;
found = vals[0];
if (found) {
/*fprintf(stderr, "FOUND '%s' => '%s'\n", (char *)key, vals);*/
path = rb_str_new_cstr(vals+1);
vals[1] = 0;
}
} else {
found = search_required(ipath, &path, safe);
/*if (found && path)
fprintf(stderr, "MISSING '%s' => '%s'\n", RSTRING_PTR(ipath), RSTRING_PTR(path));*/
}
} else {
found = search_required(ipath, &path, safe);
/*if (found && path)
fprintf(stderr, "SEARCHING '%s' found '%s'\n", RSTRING_PTR(ipath), RSTRING_PTR(path));*/
}
if (vm->require_cache.write && RSTRING_PTR(ipath)[0] != '/' && (!found || (found && path))) {
/*fprintf(stderr, "WRITING '%s' => '%s'\n", RSTRING_PTR(ipath), found ? RSTRING_PTR(path) : "");*/
fprintf(vm->require_cache.out, "%s\n", RSTRING_PTR(ipath));
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These fprintfs happen back to back, so I can't figure out why one writes sane data (to stderr) and the other ends up corrupting the cache file.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tmm1: RSTRING_PTR is not assured to be NULL-terminated. Try StringValueCStr() instead?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought that might be why, so I tried fprintf("%.*s", RSTRING_LEN(ipath), RSTRING_PTR(ipath)) but that didn't help either.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just tried StringValueCStr and no difference. It's super weird that the corruption always happens after loading thread.bundle, so I wonder if that's affecting the open file handle somehow.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be related to this, which calls back into load.c:

ext/thread/thread.c:    rb_provide("thread.rb");

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hah. Sounds like a fun one. I'm definitely digging this tomorrow if you haven't figured it out by then. :))

if (found)
fprintf(vm->require_cache.out, "%c%s\n", found, RSTRING_PTR(path));
else
fprintf(vm->require_cache.out, "%s\n", "");
fsync(fileno(vm->require_cache.out));
}

if (RUBY_DTRACE_FIND_REQUIRE_RETURN_ENABLED()) {
RUBY_DTRACE_FIND_REQUIRE_RETURN(StringValuePtr(fname),
Expand Down Expand Up @@ -1154,6 +1188,35 @@ rb_f_autoload_p(VALUE obj, VALUE sym)
return rb_mod_autoload_p(klass, sym);
}

void
require_cache_setup()
{
rb_vm_t *vm = GET_VM();
FILE *file;
char *env, line1[PATH_MAX+3], line2[PATH_MAX+3];

if ((env = getenv("REQUIRE_CACHE_WRITE"))) {
unsetenv("REQUIRE_CACHE_WRITE", "");
vm->require_cache.write = 1;
vm->require_cache.out = fopen(env, "w");
} else if ((env = getenv("REQUIRE_CACHE_READ"))) {
file = fopen(env, "r");
if (!file) return;

vm->require_cache.read = 1;
vm->require_cache.tbl = st_init_strtable();
while (1) {
if (!fgets(line1, PATH_MAX+2, file)) break;
if (!fgets(line2, PATH_MAX+2, file)) break;
line1[strlen(line1)-1] = 0;
line2[strlen(line2)-1] = 0;
/*fprintf(stderr, "INSERTING '%s' => '%s'\n", line1, line2);*/
st_insert(vm->require_cache.tbl, (st_data_t)ruby_strdup(line1), (st_data_t)ruby_strdup(line2));
}
fclose(file);
}
}

void
Init_load()
{
Expand Down Expand Up @@ -1187,4 +1250,6 @@ Init_load()

ruby_dln_librefs = rb_ary_tmp_new(0);
rb_gc_register_mark_object(ruby_dln_librefs);

require_cache_setup();
}
3 changes: 3 additions & 0 deletions vm.c
Original file line number Diff line number Diff line change
Expand Up @@ -1823,6 +1823,9 @@ ruby_vm_destruct(rb_vm_t *vm)
#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
struct rb_objspace *objspace = vm->objspace;
#endif
if (vm->require_cache.write)
fclose(vm->require_cache.out);

rb_gc_force_recycle(vm->self);
vm->main_thread = 0;
if (th) {
Expand Down
6 changes: 6 additions & 0 deletions vm_core.h
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,12 @@ typedef struct rb_vm_struct {
VALUE loaded_features_snapshot;
struct st_table *loaded_features_index;
struct st_table *loading_table;
struct {
unsigned int read:1;
struct st_table *tbl;
unsigned int write:1;
FILE *out;
} require_cache;

/* signal */
struct {
Expand Down