Skip to content

require() resolution cache #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed

require() resolution cache #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2d02be5
add REQUIRE_CACHE_{READ_WRITE} env vars
tmm1 Feb 11, 2015
25e98f7
cache lookup failures too
tmm1 Feb 11, 2015
bbbca3c
clear out REQUIRE_CACHE_WRITE for any subprocesses
tmm1 Feb 12, 2015
c9721f4
change debug msg
tmm1 Feb 12, 2015
1089cf1
use st_lookup instead of st_delete, and emulate search_required behavior
tmm1 Feb 12, 2015
8afbab4
short circuit require("enumerator")
tmm1 Mar 6, 2015
faa994c
ignore READ variable in WRITE mode
tmm1 Mar 6, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 68 additions & 3 deletions load.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -944,13 +944,17 @@ rb_require_safe(VALUE fname, int safe)
{
volatile VALUE result = Qnil;
rb_thread_t *th = GET_THREAD();
rb_vm_t *vm = GET_VM();
volatile VALUE errinfo = th->errinfo;
int state;
struct {
int safe;
} volatile saved;
char *volatile ftptr = 0;

if (strncmp(StringValuePtr(fname), "enumerator", 11) == 0)
return Qfalse;

if (RUBY_DTRACE_REQUIRE_ENTRY_ENABLED()) {
RUBY_DTRACE_REQUIRE_ENTRY(StringValuePtr(fname),
rb_sourcefile(),
Expand All @@ -960,9 +964,11 @@ rb_require_safe(VALUE fname, int safe)
PUSH_TAG();
saved.safe = rb_safe_level();
if ((state = EXEC_TAG()) == 0) {
VALUE path;
VALUE ipath, path = 0;
long handle;
int found;
st_data_t key, val;
char *vals;

rb_set_safe_level_force(safe);
FilePathValue(fname);
Expand All @@ -974,8 +980,36 @@ rb_require_safe(VALUE fname, int safe)
rb_sourceline());
}

path = rb_str_encode_ospath(fname);
found = search_required(path, &path, safe);
ipath = rb_str_encode_ospath(fname);
if (vm->require_cache.read && RSTRING_PTR(ipath)[0] != '/') {
key = (st_data_t)RSTRING_PTR(ipath);
if (st_lookup(vm->require_cache.tbl, key, &val)) {
vals = (char *)val;
found = vals[0];
if (found) {
/*fprintf(stderr, "FOUND '%s' => '%s'\n", (char *)key, vals);*/
path = rb_str_new_cstr(vals+1);
vals[1] = 0;
}
} else {
found = search_required(ipath, &path, safe);
/*if (found && path)
fprintf(stderr, "MISSING '%s' => '%s'\n", RSTRING_PTR(ipath), RSTRING_PTR(path));*/
}
} else {
found = search_required(ipath, &path, safe);
/*if (found && path)
fprintf(stderr, "SEARCHING '%s' found '%s'\n", RSTRING_PTR(ipath), RSTRING_PTR(path));*/
}
if (vm->require_cache.write && RSTRING_PTR(ipath)[0] != '/' && (!found || (found && path))) {
/*fprintf(stderr, "WRITING '%s' => '%s'\n", RSTRING_PTR(ipath), found ? RSTRING_PTR(path) : "");*/
fprintf(vm->require_cache.out, "%s\n", RSTRING_PTR(ipath));
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These fprintfs happen back to back, so I can't figure out why one writes sane data (to stderr) and the other ends up corrupting the cache file.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tmm1: RSTRING_PTR is not assured to be NULL-terminated. Try StringValueCStr() instead?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought that might be why, so I tried fprintf("%.*s", RSTRING_LEN(ipath), RSTRING_PTR(ipath)) but that didn't help either.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just tried StringValueCStr and no difference. It's super weird that the corruption always happens after loading thread.bundle, so I wonder if that's affecting the open file handle somehow.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be related to this, which calls back into load.c:

ext/thread/thread.c:    rb_provide("thread.rb");

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hah. Sounds like a fun one. I'm definitely digging this tomorrow if you haven't figured it out by then. :))

if (found)
fprintf(vm->require_cache.out, "%c%s\n", found, RSTRING_PTR(path));
else
fprintf(vm->require_cache.out, "%s\n", "");
fsync(fileno(vm->require_cache.out));
}

if (RUBY_DTRACE_FIND_REQUIRE_RETURN_ENABLED()) {
RUBY_DTRACE_FIND_REQUIRE_RETURN(StringValuePtr(fname),
Expand Down Expand Up @@ -1154,6 +1188,35 @@ rb_f_autoload_p(VALUE obj, VALUE sym)
return rb_mod_autoload_p(klass, sym);
}

void
require_cache_setup()
{
rb_vm_t *vm = GET_VM();
FILE *file;
char *env, line1[PATH_MAX+3], line2[PATH_MAX+3];

if ((env = getenv("REQUIRE_CACHE_WRITE"))) {
unsetenv("REQUIRE_CACHE_WRITE", "");
vm->require_cache.write = 1;
vm->require_cache.out = fopen(env, "w");
} else if ((env = getenv("REQUIRE_CACHE_READ"))) {
file = fopen(env, "r");
if (!file) return;

vm->require_cache.read = 1;
vm->require_cache.tbl = st_init_strtable();
while (1) {
if (!fgets(line1, PATH_MAX+2, file)) break;
if (!fgets(line2, PATH_MAX+2, file)) break;
line1[strlen(line1)-1] = 0;
line2[strlen(line2)-1] = 0;
/*fprintf(stderr, "INSERTING '%s' => '%s'\n", line1, line2);*/
st_insert(vm->require_cache.tbl, (st_data_t)ruby_strdup(line1), (st_data_t)ruby_strdup(line2));
}
fclose(file);
}
}

void
Init_load()
{
Expand Down Expand Up @@ -1187,4 +1250,6 @@ Init_load()

ruby_dln_librefs = rb_ary_tmp_new(0);
rb_gc_register_mark_object(ruby_dln_librefs);

require_cache_setup();
}
3 changes: 3 additions & 0 deletions vm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1823,6 +1823,9 @@ ruby_vm_destruct(rb_vm_t *vm)
#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
struct rb_objspace *objspace = vm->objspace;
#endif
if (vm->require_cache.write)
fclose(vm->require_cache.out);

rb_gc_force_recycle(vm->self);
vm->main_thread = 0;
if (th) {
Expand Down
6 changes: 6 additions & 0 deletions vm_core.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,12 @@ typedef struct rb_vm_struct {
VALUE loaded_features_snapshot;
struct st_table *loaded_features_index;
struct st_table *loading_table;
struct {
unsigned int read:1;
struct st_table *tbl;
unsigned int write:1;
FILE *out;
} require_cache;

/* signal */
struct {
Expand Down