Skip to content

Commit

Permalink
fs: improve readFile performance
Browse files Browse the repository at this point in the history
This increases the maximum buffer size per read to 512kb when using
`fs.readFile`. This is important to improve the read performance for
bigger files.

PR-URL: #27063
Refs: #25741
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Jamie Davis <davisjam@vt.edu>
  • Loading branch information
BridgeAR committed Apr 7, 2019
1 parent 97c0a34 commit eb2d416
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 21 deletions.
12 changes: 5 additions & 7 deletions lib/fs.js
Original file line number Diff line number Diff line change
Expand Up @@ -266,19 +266,17 @@ function readFileAfterStat(err, stats) {

const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0;

if (size === 0) {
context.buffers = [];
context.read();
return;
}

if (size > kMaxLength) {
err = new ERR_FS_FILE_TOO_LARGE(size);
return context.close(err);
}

try {
context.buffer = Buffer.allocUnsafeSlow(size);
if (size === 0) {
context.buffers = [];
} else {
context.buffer = Buffer.allocUnsafeSlow(size);
}
} catch (err) {
return context.close(err);
}
Expand Down
34 changes: 20 additions & 14 deletions lib/internal/fs/read_file_context.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,32 @@ const { Buffer } = require('buffer');

const { FSReqCallback, close, read } = internalBinding('fs');

const kReadFileBufferLength = 8 * 1024;
// Use 64kb in case the file type is not a regular file and thus do not know the
// actual file size. Increasing the value further results in more frequent over
// allocation for small files and consumes CPU time and memory that should be
// used else wise.
// Use up to 512kb per read otherwise to partition reading big files to prevent
// blocking other threads in case the available threads are all in use.
const kReadFileUnknownBufferLength = 64 * 1024;
const kReadFileBufferLength = 512 * 1024;

function readFileAfterRead(err, bytesRead) {
const context = this.context;

if (err)
return context.close(err);

if (bytesRead === 0)
return context.close();

context.pos += bytesRead;

if (context.size !== 0) {
if (context.pos === context.size)
context.close();
else
context.read();
if (context.pos === context.size || bytesRead === 0) {
context.close();
} else {
// Unknown size, just read until we don't get bytes.
context.buffers.push(context.buffer.slice(0, bytesRead));
if (context.size === 0) {
// Unknown size, just read until we don't get bytes.
const buffer = bytesRead === kReadFileUnknownBufferLength ?
context.buffer : context.buffer.slice(0, bytesRead);
context.buffers.push(buffer);
}
context.read();
}
}
Expand Down Expand Up @@ -60,7 +65,7 @@ class ReadFileContext {
constructor(callback, encoding) {
this.fd = undefined;
this.isUserFd = undefined;
this.size = undefined;
this.size = 0;
this.callback = callback;
this.buffers = null;
this.buffer = null;
Expand All @@ -75,9 +80,10 @@ class ReadFileContext {
let length;

if (this.size === 0) {
buffer = this.buffer = Buffer.allocUnsafeSlow(kReadFileBufferLength);
buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength);
offset = 0;
length = kReadFileBufferLength;
length = kReadFileUnknownBufferLength;
this.buffer = buffer;
} else {
buffer = this.buffer;
offset = this.pos;
Expand Down

0 comments on commit eb2d416

Please sign in to comment.