Skip to content

Always use typed arrays in MEMFS #7918

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kripken merged 2 commits into from
Jan 23, 2019
Merged

Always use typed arrays in MEMFS #7918

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a4305f4
remove MEMFS_APPEND_TO_TYPED_ARRAYS option - always use typed arrays …
kripken Jan 23, 2019
59411e6
omit a test that was always wrong
kripken Jan 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 12 additions & 27 deletions src/library_memfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,33 +113,18 @@ mergeInto(LibraryManager.library, {
// May allocate more, to provide automatic geometric increase and amortized linear performance appending writes.
// Never shrinks the storage.
expandFileStorage: function(node, newCapacity) {
#if !MEMFS_APPEND_TO_TYPED_ARRAYS
// If we are asked to expand the size of a file that already exists, revert to using a standard JS array to store the file
// instead of a typed array. This makes resizing the array more flexible because we can just .push() elements at the back to
// increase the size.
if (node.contents && node.contents.subarray && newCapacity > node.contents.length) {
node.contents = MEMFS.getFileDataAsRegularArray(node);
node.usedBytes = node.contents.length; // We might be writing to a lazy-loaded file which had overridden this property, so force-reset it.
}
#endif

if (!node.contents || node.contents.subarray) { // Keep using a typed array if creating a new storage, or if old one was a typed array as well.
var prevCapacity = node.contents ? node.contents.length : 0;
if (prevCapacity >= newCapacity) return; // No need to expand, the storage was already large enough.
// Don't expand strictly to the given requested limit if it's only a very small increase, but instead geometrically grow capacity.
// For small filesizes (<1MB), perform size*2 geometric increase, but for large sizes, do a much more conservative size*1.125 increase to
// avoid overshooting the allocation cap by a very large margin.
var CAPACITY_DOUBLING_MAX = 1024 * 1024;
newCapacity = Math.max(newCapacity, (prevCapacity * (prevCapacity < CAPACITY_DOUBLING_MAX ? 2.0 : 1.125)) | 0);
if (prevCapacity != 0) newCapacity = Math.max(newCapacity, 256); // At minimum allocate 256b for each file when expanding.
var oldContents = node.contents;
node.contents = new Uint8Array(newCapacity); // Allocate new storage.
if (node.usedBytes > 0) node.contents.set(oldContents.subarray(0, node.usedBytes), 0); // Copy old data over to the new storage.
return;
}
// Not using a typed array to back the file storage. Use a standard JS array instead.
if (!node.contents && newCapacity > 0) node.contents = [];
while (node.contents.length < newCapacity) node.contents.push(0);
var prevCapacity = node.contents ? node.contents.length : 0;
if (prevCapacity >= newCapacity) return; // No need to expand, the storage was already large enough.
// Don't expand strictly to the given requested limit if it's only a very small increase, but instead geometrically grow capacity.
// For small filesizes (<1MB), perform size*2 geometric increase, but for large sizes, do a much more conservative size*1.125 increase to
// avoid overshooting the allocation cap by a very large margin.
var CAPACITY_DOUBLING_MAX = 1024 * 1024;
newCapacity = Math.max(newCapacity, (prevCapacity * (prevCapacity < CAPACITY_DOUBLING_MAX ? 2.0 : 1.125)) | 0);
if (prevCapacity != 0) newCapacity = Math.max(newCapacity, 256); // At minimum allocate 256b for each file when expanding.
var oldContents = node.contents;
node.contents = new Uint8Array(newCapacity); // Allocate new storage.
if (node.usedBytes > 0) node.contents.set(oldContents.subarray(0, node.usedBytes), 0); // Copy old data over to the new storage.
return;
},

// Performs an exact resize of the backing file storage to the given size, if the size is not exactly this, the storage is fully reallocated.
Expand Down
6 changes: 0 additions & 6 deletions src/settings.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -610,12 +610,6 @@ var FS_LOG = 0;
// case-sensitive, like on Linux.
var CASE_INSENSITIVE_FS = 0;

// If set to nonzero, MEMFS will always utilize typed arrays as the backing
// store for appending data to files. The default behavior is to use typed
// arrays for files when the file size doesn't change after initial creation,
// and for files that do change size, use normal JS arrays instead.
var MEMFS_APPEND_TO_TYPED_ARRAYS = 0;

// If set to 0, does not build in any filesystem support. Useful if you are just
// doing pure computation, but not reading files or using any streams (including
// fprintf, and other stdio.h things) or anything related. The one exception is
Expand Down
21 changes: 9 additions & 12 deletions tests/test_browser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1278,11 +1278,10 @@ def test_file_db(self):
shutil.move('test.html', 'third.html')

def test_fs_idbfs_sync(self):
for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
for extra in [[], ['-DEXTRA_WORK']]:
secret = str(time.time())
self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=mode + ['-lidbfs.js', '-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_test', '_success']'''])
self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=mode + ['-lidbfs.js', '-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_test', '_success']'''] + extra)
for extra in [[], ['-DEXTRA_WORK']]:
secret = str(time.time())
self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=['-lidbfs.js', '-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_test', '_success']'''])
self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=['-lidbfs.js', '-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_test', '_success']'''] + extra)

def test_fs_idbfs_fsync(self):
# sync from persisted state into memory before main()
Expand All @@ -1300,16 +1299,14 @@ def test_fs_idbfs_fsync(self):
''')

args = ['--pre-js', 'pre.js', '-s', 'EMTERPRETIFY=1', '-s', 'EMTERPRETIFY_ASYNC=1', '-lidbfs.js', '-s', 'EXIT_RUNTIME=1']
for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
secret = str(time.time())
self.btest(path_from_root('tests', 'fs', 'test_idbfs_fsync.c'), '1', force_c=True, args=args + mode + ['-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])
self.btest(path_from_root('tests', 'fs', 'test_idbfs_fsync.c'), '1', force_c=True, args=args + mode + ['-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])
secret = str(time.time())
self.btest(path_from_root('tests', 'fs', 'test_idbfs_fsync.c'), '1', force_c=True, args=args + ['-DFIRST', '-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])
self.btest(path_from_root('tests', 'fs', 'test_idbfs_fsync.c'), '1', force_c=True, args=args + ['-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])

def test_fs_memfs_fsync(self):
args = ['-s', 'EMTERPRETIFY=1', '-s', 'EMTERPRETIFY_ASYNC=1', '-s', 'EXIT_RUNTIME=1']
for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
secret = str(time.time())
self.btest(path_from_root('tests', 'fs', 'test_memfs_fsync.c'), '1', force_c=True, args=args + mode + ['-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main']'''])
secret = str(time.time())
self.btest(path_from_root('tests', 'fs', 'test_memfs_fsync.c'), '1', force_c=True, args=args + ['-DSECRET=\"' + secret + '\"', '-s', '''EXPORTED_FUNCTIONS=['_main']'''])

def test_fs_workerfs_read(self):
secret = 'a' * 10
Expand Down
15 changes: 3 additions & 12 deletions tests/test_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4462,7 +4462,7 @@ def test_files(self):

mem_file = 'src.cpp.o.js.mem'
orig_args = self.emcc_args
for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1'], ['-s', 'SYSCALL_DEBUG=1']]:
for mode in [[], ['-s', 'SYSCALL_DEBUG=1']]:
print(mode)
self.emcc_args = orig_args + mode
try_delete(mem_file)
Expand Down Expand Up @@ -4534,11 +4534,7 @@ def test_getcwd_with_non_ascii_name(self):
def test_fwrite_0(self):
test_path = path_from_root('tests', 'core', 'test_fwrite_0')
src, output = (test_path + s for s in ('.c', '.out'))

orig_args = self.emcc_args
for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
self.emcc_args = orig_args + mode
self.do_run_from_file(src, output)
self.do_run_from_file(src, output)

def test_fgetc_ungetc(self):
print('TODO: update this test once the musl ungetc-on-EOF-stream bug is fixed upstream and reaches us')
Expand Down Expand Up @@ -4766,10 +4762,7 @@ def test_utf16_textdecoder(self):
def test_wprintf(self):
test_path = path_from_root('tests', 'core', 'test_wprintf')
src, output = (test_path + s for s in ('.cpp', '.out'))
orig_args = self.emcc_args
for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
self.emcc_args = orig_args + mode
self.do_run_from_file(src, output)
self.do_run_from_file(src, output)

def test_write_stdout_fileno(self):
test_path = path_from_root('tests', 'core', 'test_write_stdout_fileno')
Expand Down Expand Up @@ -4841,9 +4834,7 @@ def test_fs_writeFile(self, js_engines=None):
out = path_from_root('tests', 'fs', 'test_writeFile.out')
self.do_run_from_file(src, out, js_engines=js_engines)

@also_with_noderawfs
def test_fs_write(self, js_engines=None):
self.emcc_args = ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']
src = path_from_root('tests', 'fs', 'test_write.cpp')
out = path_from_root('tests', 'fs', 'test_write.out')
self.do_run_from_file(src, out, js_engines=js_engines)
Expand Down