diff --git a/Changelog.md b/Changelog.md index 2fd300885b..6dd7f49787 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,17 @@ # Changelog +## [1.0.2] - singlestore-nodejs (unreleased) + +### Features + +- **`infileStream` query option** — stream in-memory bytes for `LOAD DATA LOCAL INFILE` without reading a client file; SQL filename is ignored (aligned with SingleStore stream-based load documentation). +- **`local_infile` connection option** — explicit enable/disable for `LOAD DATA LOCAL INFILE` (default enabled); clearer client error when disabled. +- **Prepared statement / pool cluster** — connection-level `infileStreamFactory` is passed through `statement.execute()` and `PoolCluster.query()`. + +### Documentation + +- README and site docs describe both local-file (`infileStreamFactory`) and in-memory (`infileStream`) loads wherever `LOAD DATA LOCAL INFILE` is documented. + ## [3.14.3](https://github.com/sidorares/node-mysql2/compare/v3.14.2...v3.14.3) (2025-07-29) diff --git a/README.md b/README.md index 45dfc817bc..fe2f90c865 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ [docs-mysql-server]: https://singlestore-labs.github.io/singlestore-nodejs/docs/documentation/mysql-server [docs-promise-wrapper]: https://singlestore-labs.github.io/singlestore-nodejs/documentation/promise-wrapper [docs-authentication-switch]: https://singlestore-labs.github.io/singlestore-nodejs/docs/documentation/authentication-switch -[docs-streams]: https://sidorares.github.io/node-mysql2/docs/documentation/extras +[docs-load-data]: https://singlestore-labs.github.io/singlestore-nodejs/docs/documentation/extras#load-data-local-infile +[docs-streams]: https://singlestore-labs.github.io/singlestore-nodejs/docs/documentation/extras#connecting-using-custom-stream [docs-typescript-docs]: https://sidorares.github.io/node-mysql2/docs/documentation/typescript-examples [docs-qs-pooling]: https://singlestore-labs.github.io/singlestore-nodejs/docs#using-connection-pools [docs-qs-first-query]: https://singlestore-labs.github.io/singlestore-nodejs/docs#first-query @@ -73,6 +74,7 @@ MySQL2 is mostly API compatible with [Node MySQL][node-mysql] and supports major - [Promise Wrapper][docs-promise-wrapper] - Compression - SSL and [Authentication Switch][docs-authentication-switch] +- [LOAD DATA LOCAL INFILE][docs-load-data] (local files and in-memory streams) - [Custom Streams][docs-streams] - [Pooling][docs-qs-pooling] @@ -86,6 +88,7 @@ SingleStore Node.js Driver is mostly API compatible with [node-mysql][node-mysql - **Consistent Data Type Handling**: Handles SingleStore data types more consistently than generic MySQL drivers - **Configurable Connection Behavior**: Allows you to configure connection behavior by setting session variables - **MySQL Compatibility**: Maintains full compatibility with MySQL wire protocol +- **Bulk load**: `LOAD DATA LOCAL INFILE` from client files (`infileStreamFactory`) or in-memory streams (`infileStream`) For more information about SingleStore, visit the [official documentation][singlestore-docs]. @@ -100,7 +103,7 @@ If you are using TypeScript, you will need to install `@types/node`. ## Quick Start ```javascript -const singlestore = require('singlestore_nodejs'); +const singlestore = require('singlestore-nodejs'); // Create connection const connection = singlestore.createConnection({ @@ -140,12 +143,23 @@ const mysql = require('mysql2'); const connection = mysql.createConnection({...}); // After (SingleStore Node.js Driver) -const singlestore = require('singlestore_nodejs'); +const singlestore = require('singlestore-nodejs'); const connection = singlestore.createConnection({...}); // All existing queries work as-is! ``` +## Load data from local files and in-memory streams + +Use `LOAD DATA LOCAL INFILE` to bulk-load CSV (or other `LOAD DATA` formats) into SingleStore. The driver reads data on the **client** (your Node process), not from the database server disk. + +- **Local file:** set `infileStreamFactory` on the connection or query so the driver opens the path the server requests (for example `fs.createReadStream(path)`). +- **In-memory stream:** pass `infileStream` on the query (a Node.js `Readable`). The filename in SQL is ignored; use a placeholder such as `':stream:'`. This matches [SingleStore stream-based load](https://docs.singlestore.com/db/v9.1/load-data/load-data-from-files/load-data-from-local-files/). + +`local_infile` defaults to `true`. Set `local_infile: false` to refuse file and stream loads on that connection. + +See [LOAD DATA LOCAL INFILE][docs-load-data] for examples. + ## Documentation 📚 **Full documentation:** [SingleStore Nodejs Driver][docs-base] diff --git a/lib/commands/command.js b/lib/commands/command.js index 3d318868bb..7c8d900bda 100644 --- a/lib/commands/command.js +++ b/lib/commands/command.js @@ -2,6 +2,7 @@ const EventEmitter = require('events').EventEmitter; const Timers = require('timers'); +const { rewriteLocalInfileError } = require('./local_infile.js'); class Command extends EventEmitter { constructor() { @@ -28,6 +29,7 @@ class Command extends EventEmitter { if (packet && packet.isError()) { const err = packet.asError(connection.clientEncoding); err.sql = this.sql || this.query; + rewriteLocalInfileError(err, connection, err.sql); if (this.queryTimeout) { Timers.clearTimeout(this.queryTimeout); this.queryTimeout = null; diff --git a/lib/commands/execute.js b/lib/commands/execute.js index a5403375a0..cd3cc80b72 100644 --- a/lib/commands/execute.js +++ b/lib/commands/execute.js @@ -1,6 +1,7 @@ 'use strict'; const Command = require('./command.js'); +const { assignLocalInfileOptions } = require('./local_infile.js'); const Query = require('./query.js'); const Packets = require('../packets/index.js'); @@ -27,7 +28,7 @@ class Execute extends Command { this._resultIndex = 0; this._localStream = null; this._unpipeStream = function () {}; - this._streamFactory = options.infileStreamFactory; + assignLocalInfileOptions(this, options); this._connection = null; } diff --git a/lib/commands/local_infile.js b/lib/commands/local_infile.js new file mode 100644 index 0000000000..b818d3ac29 --- /dev/null +++ b/lib/commands/local_infile.js @@ -0,0 +1,61 @@ +'use strict'; + +const LOCAL_INFILE_DISABLED_MESSAGE = + 'Local infile is disabled by connector. Enable `local_infile` to allow local infile commands'; + +const LOCAL_INFILE_DISABLED_ERRNOS = new Set([1148, 3948, 4166]); + +function localInfileAllowed(connection) { + return ( + !connection || !connection.config || connection.config.localInfile !== false + ); +} + +function localInfileDisabledError() { + const err = new Error(LOCAL_INFILE_DISABLED_MESSAGE); + err.code = 'ER_CLIENT_LOCAL_FILES_DISABLED'; + err.errno = 3948; + return err; +} + +function assignLocalInfileOptions(command, options) { + const source = options || {}; + command._streamFactory = source.infileStreamFactory; + command._infileStream = null; + if (source.infileStream !== undefined && source.infileStream !== null) { + if ( + typeof source.infileStream.read !== 'function' || + typeof source.infileStream.on !== 'function' + ) { + throw new TypeError('infileStream must be a Readable stream'); + } + command._infileStream = source.infileStream; + } +} + +function rewriteLocalInfileError(err, connection, sql) { + if (!err || localInfileAllowed(connection)) { + return err; + } + if (!LOCAL_INFILE_DISABLED_ERRNOS.has(err.errno)) { + return err; + } + // 1148 is a generic "command not allowed" code. Only rewrite it for LOCAL INFILE. + if ( + err.errno === 1148 && + !/load\s+(data\s+)?local\s+infile/i.test(sql || '') + ) { + return err; + } + err.message = LOCAL_INFILE_DISABLED_MESSAGE; + err.sqlMessage = LOCAL_INFILE_DISABLED_MESSAGE; + return err; +} + +module.exports = { + LOCAL_INFILE_DISABLED_MESSAGE, + assignLocalInfileOptions, + localInfileAllowed, + localInfileDisabledError, + rewriteLocalInfileError, +}; diff --git a/lib/commands/prepare.js b/lib/commands/prepare.js index ab217a8230..8ca37d90dd 100644 --- a/lib/commands/prepare.js +++ b/lib/commands/prepare.js @@ -24,8 +24,17 @@ class PreparedStatementInfo { callback = parameters; parameters = []; } - return this._connection.addCommand( - new Execute({ statement: this, values: parameters }, callback) + const connection = this._connection; + return connection.addCommand( + new Execute( + { + statement: this, + sql: this.query, + values: parameters, + infileStreamFactory: connection.config.infileStreamFactory, + }, + callback + ) ); } } diff --git a/lib/commands/query.js b/lib/commands/query.js index 117e4bac70..876182c548 100644 --- a/lib/commands/query.js +++ b/lib/commands/query.js @@ -6,6 +6,11 @@ const Timers = require('timers'); const Readable = require('stream').Readable; const Command = require('./command.js'); +const { + assignLocalInfileOptions, + localInfileAllowed, + localInfileDisabledError, +} = require('./local_infile.js'); const Packets = require('../packets/index.js'); const getTextParser = require('../parsers/text_parser.js'); const staticParser = require('../parsers/static_text_parser.js'); @@ -32,7 +37,7 @@ class Query extends Command { this._resultIndex = 0; this._localStream = null; this._unpipeStream = function () {}; - this._streamFactory = options.infileStreamFactory; + assignLocalInfileOptions(this, options); this._connection = null; } @@ -137,9 +142,29 @@ class Query extends Command { } _streamLocalInfile(connection, path) { - if (this._streamFactory) { - this._localStream = this._streamFactory(path); + if (!localInfileAllowed(connection)) { + const err = localInfileDisabledError(); + err.sql = this.sql; + this._localStreamError = err; + connection.writePacket(EmptyPacket); + return this.infileOk; + } + + // A direct stream ignores the filename the server asked for. + if (this._infileStream) { + this._localStream = this._infileStream; } else { + const config = connection.config || {}; + const factory = + this._streamFactory === undefined + ? config.infileStreamFactory + : this._streamFactory; + if (factory) { + this._localStream = factory(path); + } + } + + if (!this._localStream) { this._localStreamError = new Error( `As a result of LOCAL INFILE command server wants to read ${path} file, but as of v2.0 you must provide streamFactory option returning ReadStream.` ); diff --git a/lib/connection_config.js b/lib/connection_config.js index a191b86da6..6c320dd33a 100644 --- a/lib/connection_config.js +++ b/lib/connection_config.js @@ -32,6 +32,7 @@ const validOptions = { host: 1, insecureAuth: 1, infileStreamFactory: 1, + local_infile: 1, isServer: 1, keepAliveInitialDelay: 1, localAddress: 1, @@ -179,6 +180,18 @@ class ConnectionConfig { ConnectionConfig.getDefaultFlags(options), options.flags || '' ); + if (options.local_infile !== undefined) { + this.localInfile = ConnectionConfig.resolveLocalInfile( + options.local_infile + ); + if (this.localInfile) { + this.clientFlags |= ClientConstants.LOCAL_FILES; + } else { + this.clientFlags &= ~ClientConstants.LOCAL_FILES; + } + } else { + this.localInfile = (this.clientFlags & ClientConstants.LOCAL_FILES) !== 0; + } // Default connection attributes // https://dev.mysql.com/doc/refman/8.0/en/performance-schema-connection-attribute-tables.html const defaultConnectAttributes = { @@ -242,6 +255,22 @@ class ConnectionConfig { return flags; } + static resolveLocalInfile(value) { + if (typeof value === 'string') { + const normalized = value.toLowerCase(); + if ( + normalized === 'false' || + normalized === '0' || + normalized === 'off' || + normalized === 'no' + ) { + return false; + } + return true; + } + return value !== false && value !== 0; + } + static getDefaultFlags(options) { const defaultFlags = [ 'LONG_PASSWORD', diff --git a/lib/pool_cluster.js b/lib/pool_cluster.js index ef68e60fe6..df4fee8658 100644 --- a/lib/pool_cluster.js +++ b/lib/pool_cluster.js @@ -103,6 +103,13 @@ class PoolNamespace { } return; } + if ( + query._streamFactory === undefined && + conn.config && + conn.config.infileStreamFactory + ) { + query._streamFactory = conn.config.infileStreamFactory; + } try { conn.query(query).once('end', () => { conn.release(); diff --git a/test/integration/connection/test-load-infile.test.cjs b/test/integration/connection/test-load-infile.test.cjs index 8f0f1b333c..ff9b353097 100644 --- a/test/integration/connection/test-load-infile.test.cjs +++ b/test/integration/connection/test-load-infile.test.cjs @@ -3,7 +3,12 @@ const common = require('../../common.test.cjs'); const { assert } = require('poku'); const fs = require('node:fs'); +const { Readable } = require('node:stream'); const process = require('node:process'); +const mysql = require('../../../index.js'); + +const LOCAL_INFILE_DISABLED_MESSAGE = + 'Local infile is disabled by connector. Enable `local_infile` to allow local infile commands'; if (`${process.env.MYSQL_CONNECTION_URL}`.includes('pscale_pw_')) { console.log('skipping test for planetscale'); @@ -76,6 +81,8 @@ const createMyStream = function () { }; let streamResult; +let memoryResult; +let disabledErr; connection.query( { sql: sql, @@ -87,7 +94,56 @@ connection.query( throw err; } streamResult = result; - connection.end(); + connection.query( + { + sql: `LOAD DATA LOCAL INFILE ':stream:' INTO TABLE ${table} FIELDS TERMINATED BY ',' (id, title)`, + infileStream: Readable.from([ + '31,From Memory\n', + '32,Still', + ' Memory\n', + ]), + infileStreamFactory: () => { + throw new Error( + 'infileStreamFactory must not run when infileStream is set' + ); + }, + }, + (memErr, memResult) => { + if (memErr) { + throw memErr; + } + memoryResult = memResult; + const disabled = mysql.createConnection({ + ...common.config, + local_infile: false, + }); + disabled.query( + [ + `CREATE TEMPORARY TABLE \`${table}\` (`, + '`id` int(11) unsigned NOT NULL,', + '`title` varchar(255)', + ')', + ].join('\n'), + (createErr) => { + if (createErr) { + throw createErr; + } + disabled.query( + { + sql: `LOAD DATA LOCAL INFILE ':stream:' INTO TABLE ${table} FIELDS TERMINATED BY ',' (id, title)`, + infileStream: Readable.from(['99,should-not-load\n']), + }, + (loadError) => { + disabledErr = loadError; + disabled.end(() => { + connection.end(); + }); + } + ); + } + ); + } + ); } ); @@ -104,4 +160,6 @@ process.on('exit', () => { assert.equal(loadResult.affectedRows, 0); assert.equal(streamResult.affectedRows, 2); + assert.equal(memoryResult.affectedRows, 2); + assert.equal(disabledErr.message, LOCAL_INFILE_DISABLED_MESSAGE); }); diff --git a/test/unit/commands/test-local-infile.test.cjs b/test/unit/commands/test-local-infile.test.cjs new file mode 100644 index 0000000000..d46ba60d67 --- /dev/null +++ b/test/unit/commands/test-local-infile.test.cjs @@ -0,0 +1,206 @@ +'use strict'; + +const { EventEmitter } = require('node:events'); +const { PassThrough } = require('node:stream'); +const { assert } = require('poku'); + +const Command = require('../../../lib/commands/command.js'); +const Query = require('../../../lib/commands/query.js'); +const Packets = require('../../../lib/packets/index.js'); +const ClientConstants = require('../../../lib/constants/client.js'); +const { + LOCAL_INFILE_DISABLED_MESSAGE, + rewriteLocalInfileError, +} = require('../../../lib/commands/local_infile.js'); + +class FakeConnection extends EventEmitter { + constructor(config) { + super(); + this.packets = []; + this.stream = new EventEmitter(); + this.clientEncoding = 'utf8'; + this._resetSequenceId = () => {}; + this.config = { + localInfile: true, + clientFlags: ClientConstants.LOCAL_FILES, + ...config, + }; + } + + writePacket(packet) { + this.packets.push(packet); + } +} + +function payload(packet) { + return packet.buffer.slice(packet.start + 4, packet.end).toString(); +} + +const directStream = new PassThrough(); +let factoryCalled = false; +const query = new Query( + { + sql: "LOAD DATA LOCAL INFILE ':stream:' INTO TABLE t", + infileStream: directStream, + infileStreamFactory() { + factoryCalled = true; + return new PassThrough(); + }, + }, + () => {} +); +const connection = new FakeConnection(); +const next = query._streamLocalInfile(connection, '/etc/passwd'); +assert.equal(next, query.infileOk); +assert.equal(factoryCalled, false); +directStream.write('1,2,3\n'); +directStream.write('4,5'); +directStream.end(',6\n'); +assert.equal(payload(connection.packets[0]), '1,2,3\n'); +assert.equal(payload(connection.packets[1]), '4,5'); +assert.equal(payload(connection.packets[2]), ',6\n'); + +let seenPath; +const fallback = new Query( + { sql: "LOAD DATA LOCAL INFILE 'requested.csv' INTO TABLE t" }, + () => {} +); +let fallbackStream; +const fallbackConnection = new FakeConnection({ + infileStreamFactory(path) { + seenPath = path; + fallbackStream = new PassThrough(); + return fallbackStream; + }, +}); +fallback._streamLocalInfile(fallbackConnection, 'requested.csv'); +fallbackStream.end('7,fallback\n'); +assert.equal(seenPath, 'requested.csv'); +assert.equal(payload(fallbackConnection.packets[0]), '7,fallback\n'); + +process.nextTick(() => { + assert.equal(connection.packets[connection.packets.length - 1].length(), 4); + assert.equal( + fallbackConnection.packets[fallbackConnection.packets.length - 1].length(), + 4 + ); + query._unpipeStream(); + fallback._unpipeStream(); +}); + +const optedOut = new Query( + { + sql: "LOAD DATA LOCAL INFILE 'x.csv' INTO TABLE t", + infileStreamFactory: null, + }, + () => {} +); +let optedOutCalled = false; +const optedOutConnection = new FakeConnection({ + infileStreamFactory() { + optedOutCalled = true; + return new PassThrough(); + }, +}); +optedOut._streamLocalInfile(optedOutConnection, 'x.csv'); +assert.equal(optedOutCalled, false); +assert.equal( + optedOut._localStreamError.message, + 'As a result of LOCAL INFILE command server wants to read x.csv file, but as of v2.0 you must provide streamFactory option returning ReadStream.' +); + +const blockedStream = new PassThrough(); +const blocked = new Query( + { + sql: "LOAD DATA LOCAL INFILE ':stream:' INTO TABLE t", + infileStream: blockedStream, + infileStreamFactory() { + throw new Error('factory must not run when local infile is disabled'); + }, + }, + () => {} +); +const blockedConnection = new FakeConnection({ + localInfile: false, + clientFlags: 0, +}); +blocked._streamLocalInfile(blockedConnection, '/etc/passwd'); +assert.equal(blocked._localStreamError.message, LOCAL_INFILE_DISABLED_MESSAGE); +assert.equal(blockedConnection.packets.length, 1); +assert.equal(blockedConnection.packets[0].length(), 4); +assert.equal(blockedStream.readableLength, 0); + +assert.throws( + () => + new Query( + { + sql: 'LOAD DATA LOCAL INFILE "x"', + infileStream: Buffer.from('nope'), + }, + () => {} + ), + (err) => + err instanceof TypeError && + err.message === 'infileStream must be a Readable stream' +); + +const unrelated = new Error('command not allowed'); +unrelated.errno = 1148; +unrelated.sqlMessage = unrelated.message; +rewriteLocalInfileError(unrelated, blockedConnection, 'SELECT 1'); +assert.equal(unrelated.message, 'command not allowed'); + +const localErr = new Error('Loading local data is disabled'); +localErr.errno = 3948; +localErr.sqlMessage = localErr.message; +rewriteLocalInfileError( + localErr, + blockedConnection, + "LOAD DATA LOCAL INFILE 'x' INTO TABLE t" +); +assert.equal(localErr.message, LOCAL_INFILE_DISABLED_MESSAGE); + +class LocalQuery extends Command { + constructor(sql, cb) { + super(); + this.sql = sql; + this.onResult = cb; + } + + start() { + return this.start; + } +} + +const packet = Packets.Error.toPacket( + { + code: 1148, + message: 'The used command is not allowed with this MySQL version', + }, + 'utf8' +); +packet.offset = packet.start + 4; +let commandError; +const command = new LocalQuery( + "LOAD DATA LOCAL INFILE 'x' INTO TABLE t", + (err) => { + commandError = err; + } +); +command.execute(packet, blockedConnection); +assert.equal(commandError.message, LOCAL_INFILE_DISABLED_MESSAGE); +assert.equal(commandError.errno, 1148); + +const Execute = require('../../../lib/commands/execute.js'); +const executeFactory = () => new PassThrough(); +const execute = new Execute( + { + statement: { id: 1 }, + sql: "LOAD DATA LOCAL INFILE 'a.csv' INTO TABLE t", + values: [], + infileStreamFactory: executeFactory, + }, + () => {} +); +assert.equal(execute._streamFactory, executeFactory); +assert.equal(execute._infileStream, null); diff --git a/test/unit/connection/test-connection_config.test.cjs b/test/unit/connection/test-connection_config.test.cjs index 9be92f061b..11776984e7 100644 --- a/test/unit/connection/test-connection_config.test.cjs +++ b/test/unit/connection/test-connection_config.test.cjs @@ -1,6 +1,7 @@ 'use strict'; const ConnectionConfig = require('../../../lib/connection_config.js'); +const ClientConstants = require('../../../lib/constants/client.js'); const { assert } = require('poku'); const expectedMessage = "SSL profile must be an object, instead it's a boolean"; @@ -76,3 +77,41 @@ assert.strictEqual( ).database, 'database$' ); + +const hasLocalFiles = (config) => + (config.clientFlags & ClientConstants.LOCAL_FILES) !== 0; + +const defaultConfig = new ConnectionConfig({}); +assert.strictEqual(defaultConfig.localInfile, true); +assert.strictEqual(hasLocalFiles(defaultConfig), true); + +const disabledConfig = new ConnectionConfig({ local_infile: false }); +assert.strictEqual(disabledConfig.localInfile, false); +assert.strictEqual(hasLocalFiles(disabledConfig), false); + +assert.strictEqual( + new ConnectionConfig({ local_infile: 0 }).localInfile, + false +); +assert.strictEqual( + new ConnectionConfig({ local_infile: 'false' }).localInfile, + false +); + +const optedOut = new ConnectionConfig({ flags: '-LOCAL_FILES' }); +assert.strictEqual(optedOut.localInfile, false); +assert.strictEqual(hasLocalFiles(optedOut), false); + +const forcedOn = new ConnectionConfig({ + local_infile: true, + flags: '-LOCAL_FILES', +}); +assert.strictEqual(forcedOn.localInfile, true); +assert.strictEqual(hasLocalFiles(forcedOn), true); + +const forcedOff = new ConnectionConfig({ + local_infile: false, + flags: 'LOCAL_FILES', +}); +assert.strictEqual(forcedOff.localInfile, false); +assert.strictEqual(hasLocalFiles(forcedOff), false); diff --git a/test/unit/pool-cluster/test-load-infile-factory.test.cjs b/test/unit/pool-cluster/test-load-infile-factory.test.cjs new file mode 100644 index 0000000000..e0c873ca9f --- /dev/null +++ b/test/unit/pool-cluster/test-load-infile-factory.test.cjs @@ -0,0 +1,43 @@ +'use strict'; + +const { EventEmitter } = require('node:events'); +const { PassThrough } = require('node:stream'); +const { assert } = require('poku'); + +const PoolCluster = require('../../../lib/pool_cluster.js'); + +const cluster = new PoolCluster({ canRetry: false }); +const namespace = cluster.of('MASTER'); +const connectionFactory = function connectionFactory() {}; +let queried; + +namespace.getConnection = (cb) => { + cb(null, { + config: { infileStreamFactory: connectionFactory }, + query(command) { + queried = command; + return new EventEmitter(); + }, + }); +}; + +const inherited = namespace.query({ sql: 'SELECT 1' }); +assert.equal(inherited._streamFactory, connectionFactory); +assert.equal(queried, inherited); + +const customFactory = function customFactory() {}; +const explicit = namespace.query({ + sql: 'SELECT 1', + infileStreamFactory: customFactory, +}); +assert.equal(explicit._streamFactory, customFactory); + +const stream = new PassThrough(); +const streamed = namespace.query({ + sql: "LOAD DATA LOCAL INFILE ':stream:' INTO TABLE t", + infileStream: stream, +}); +assert.equal(streamed._infileStream, stream); +assert.equal(streamed._streamFactory, connectionFactory); + +cluster.end(); diff --git a/typings/mysql/lib/Connection.d.ts b/typings/mysql/lib/Connection.d.ts index c8014cf661..6a7814b23c 100644 --- a/typings/mysql/lib/Connection.d.ts +++ b/typings/mysql/lib/Connection.d.ts @@ -172,6 +172,14 @@ export interface ConnectionOptions { */ infileStreamFactory?: (path: string) => Readable; + /** + * Allow `LOAD DATA LOCAL INFILE`, including in-memory streams. (Default: true) + * + * Set to `false` to refuse both file and stream loads. Equivalent connection + * URLs use `local_infile=false`. + */ + local_infile?: boolean; + /** * Determines if column values should be converted to native JavaScript types. * diff --git a/typings/mysql/lib/protocol/sequences/Query.d.ts b/typings/mysql/lib/protocol/sequences/Query.d.ts index eee40b714d..9081903a70 100644 --- a/typings/mysql/lib/protocol/sequences/Query.d.ts +++ b/typings/mysql/lib/protocol/sequences/Query.d.ts @@ -83,6 +83,13 @@ export interface QueryOptions { * By specifying a function that returns a readable stream, an arbitrary stream can be sent when sending a local fs file. */ infileStreamFactory?: (path: string) => Readable; + + /** + * In-memory bytes for `LOAD DATA LOCAL INFILE`. + * When set, this stream is uploaded and the filename requested by the server is ignored. + * Takes precedence over `infileStreamFactory`. + */ + infileStream?: Readable; } export interface StreamOptions { diff --git a/website/docs/api-and-configurations.mdx b/website/docs/api-and-configurations.mdx index 6be8a31df5..dd548107fd 100644 --- a/website/docs/api-and-configurations.mdx +++ b/website/docs/api-and-configurations.mdx @@ -20,7 +20,7 @@ const mysql = require('mysql2'); const connection = mysql.createConnection({...}); // After (SingleStore Node.js Driver) -const singlestore = require('singlestore_nodejs'); +const singlestore = require('singlestore-nodejs'); const connection = singlestore.createConnection({...}); // All existing queries work as-is! @@ -29,3 +29,13 @@ const connection = singlestore.createConnection({...}); :::info If you find any incompatibility with [node-mysql2][node-mysql2] or issues specific to SingleStoreDB, please report via GitHub issue. We will address reported issues on priority basis. ::: + +## LOAD DATA LOCAL INFILE + +Bulk load from the **client** (not the server disk): + +- **`infileStreamFactory(path)`** — connection and/or query option; return a `Readable` for the path the server requests (typical for files on the app host). +- **`infileStream`** — query option only; upload in-memory bytes; SQL filename is ignored. See [extras: LOAD DATA LOCAL INFILE](/docs/documentation/extras#load-data-local-infile). +- **`local_infile`** — connection option (default `true`); set `false` to disable local file and stream loads. + +Compatible with [SingleStore stream-based load](https://docs.singlestore.com/db/v9.1/load-data/load-data-from-files/load-data-from-local-files/). diff --git a/website/docs/documentation/00-index.mdx b/website/docs/documentation/00-index.mdx index da3a7f5695..a9c997020b 100644 --- a/website/docs/documentation/00-index.mdx +++ b/website/docs/documentation/00-index.mdx @@ -24,7 +24,7 @@ _This driver is based on [node-mysql2][node-mysql2] and maintains API compatibil - [Prepared Statements](/docs/documentation/prepared-statements) - [Promise Wrapper](/docs/documentation/promise-wrapper) - [Authentication Switch](/docs/documentation/authentication-switch) -- [More Features](/docs/documentation/extras) +- [More Features](/docs/documentation/extras) — includes [LOAD DATA LOCAL INFILE](/docs/documentation/extras#load-data-local-infile) (local files via `infileStreamFactory` and in-memory streams via `infileStream`) - [SingleStore Server][singlestore-docs] - Pooling - [SSL](/docs/documentation/ssl) diff --git a/website/docs/documentation/extras.mdx b/website/docs/documentation/extras.mdx index 92f692935c..cb28c6255d 100644 --- a/website/docs/documentation/extras.mdx +++ b/website/docs/documentation/extras.mdx @@ -41,43 +41,31 @@ connection.query(options, (err, results) => { }); ``` -## Sending tabular data with 'load infile' and local stream: +## LOAD DATA LOCAL INFILE {#load-data-local-infile} -In addition to sending local fs files you can send any stream using `infileStreamFactory` query option. If set, it has to be a function that return a readable stream. It gets file path from query as a parameter. +Bulk-load tabular data with `LOAD DATA LOCAL INFILE`. **Local** means the bytes come from your Node.js process (a file on the app host or an in-memory stream), not from the SingleStore server filesystem. See also [SingleStore: Load Data from Local Files](https://docs.singlestore.com/db/v9.1/load-data/load-data-from-files/load-data-from-local-files/) (including stream-based loads). -Note: starting from version 2.0 `infileStreamFactory` is required parameter for `LOAD DATA LOCAL INFILE`. Response from server indicates that it wants access to a local file and no `infileStreamFactory` option is provided the query ends with error. +| Source | Option | Notes | +| --- | --- | --- | +| File on the client machine | `infileStreamFactory(path)` | Called with the path from the server; return a `Readable` (for example `fs.createReadStream(path)`). Connection-level or per-query. | +| Bytes in memory | `infileStream` | Per-query only. Server filename is **ignored**; use `':stream:'` in SQL if you like. Chunks need not align to row boundaries. Takes precedence over `infileStreamFactory`. | -```js -// local file -connection.query( - 'LOAD DATA LOCAL INFILE "/tmp/data.csv" INTO TABLE test FIELDS TERMINATED BY ? (id, title)', - onInserted1 -); -// local stream -const sql = - 'LOAD DATA LOCAL INFILE "mystream" INTO TABLE test FIELDS TERMINATED BY ? (id, title)'; -connection.query( - { - sql: sql, - infileStreamFactory: function (path) { - return getStream(); - }, - }, - onInserted2 -); -``` +The driver does **not** open filesystem paths automatically. For every load you must supply `infileStream` or `infileStreamFactory`, or the query fails. + +`local_infile` defaults to `true` (client advertises `LOCAL_FILES`). Set `local_infile: false` or `local_infile=false` on a connection URI to block file and in-memory loads with a clear client error. -The `infileStreamFactory` option may also be set at a connection-level: +### Load from a local file + +Set `infileStreamFactory` on the connection (or on the query). Then run `LOAD DATA LOCAL INFILE` with the file path in SQL or as a `?` placeholder. ```js const fs = require('fs'); -const mysql = require('mysql2'); +const singlestore = require('singlestore-nodejs'); -const connection = mysql.createConnection({ +const connection = singlestore.createConnection({ user: 'test', database: 'test', infileStreamFactory: (path) => { - // Validate file path const validPaths = ['/tmp/data.csv']; if (!validPaths.includes(path)) { throw new Error( @@ -91,7 +79,25 @@ const connection = mysql.createConnection({ }); connection.query( - 'LOAD DATA LOCAL INFILE "/tmp/data.csv" INTO TABLE test', + 'LOAD DATA LOCAL INFILE "/tmp/data.csv" INTO TABLE test FIELDS TERMINATED BY ? (id, title)', + [','], + onInserted +); +``` + +### Stream from memory (no file on disk) + +Pass `infileStream` on the query. The filename in the SQL is ignored. + +```js +const { Readable } = require('stream'); +const singlestore = require('singlestore-nodejs'); + +connection.query( + { + sql: "LOAD DATA LOCAL INFILE ':stream:' INTO TABLE test FIELDS TERMINATED BY ',' (id, title)", + infileStream: Readable.from(['1,2,3\n', '4,5,6\n7,8,9\n']), + }, onInserted ); ``` diff --git a/website/docs/history-and-why-mysq2.mdx b/website/docs/history-and-why-mysq2.mdx index 3c5620485f..a475880de9 100644 --- a/website/docs/history-and-why-mysq2.mdx +++ b/website/docs/history-and-why-mysq2.mdx @@ -13,7 +13,8 @@ SingleStore Node.js Driver is mostly API compatible with [node-mysql][node-mysql - Compression - [Prepared Statements](/docs/documentation/prepared-statements) - SSL and [Authentication Switch](/docs/documentation/authentication-switch) -- [Custom Streams](/docs/documentation/extras#connecting-using-custom-stream) +- [LOAD DATA LOCAL INFILE](/docs/documentation/extras#load-data-local-infile) (local files and in-memory `infileStream`) +- [Custom connection transport streams](/docs/documentation/extras#connecting-using-custom-stream) - [Pooling](/docs/#using-connection-pools) ## Why Choose SingleStore Node.js Driver? diff --git a/website/docs/index.mdx b/website/docs/index.mdx index 040c6e1f9c..e21cbdec7e 100644 --- a/website/docs/index.mdx +++ b/website/docs/index.mdx @@ -147,6 +147,17 @@ connection.query(
+### Load data from local files or memory + +Use `LOAD DATA LOCAL INFILE` to bulk-load into SingleStore from the **client** (your Node app), not from the server disk. + +- **Local file:** `infileStreamFactory` on the connection or query (for example `fs.createReadStream(path)`). +- **In-memory stream:** `infileStream` on the query; the filename in SQL is ignored. + +Details and examples: [LOAD DATA LOCAL INFILE](/docs/documentation/extras#load-data-local-infile). + +
+ ### Using Prepared Statements SingleStore-Nodejs supports prepared statements primarily for compatibility with MySQL drivers and existing codebases. However, **SingleStore automatically compiles and optimizes both text and binary protocol queries**, so you get performance benefits even with regular queries. diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx index 41d55299a5..a16d7748c1 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx @@ -12,7 +12,8 @@ O MySQL2 é maioritariamente compatível com a API do [mysqljs][node-mysql] e su - [_Promise Wrapper_](/docs/documentation/promise-wrapper) - Compressão - SSL e [_Authentication Switch_](/docs/documentation/authentication-switch) -- [_Streams_ Personalizados](/docs/documentation/extras#connecting-using-custom-stream) +- [LOAD DATA LOCAL INFILE](/docs/documentation/extras#load-data-local-infile) (arquivos locais e `infileStream` em memória) +- [_Streams_ de transporte da conexão](/docs/documentation/extras#connecting-using-custom-stream) - [Conjunto de Conexões (_Pooling_)](/docs/#using-connection-pools) [node-mysql]: https://github.com/mysqljs/mysql diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx index f7600d5007..a688164db3 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/history-and-why-mysq2.mdx @@ -12,7 +12,8 @@ MySQL2 大部分 API 与 [mysqljs][node-mysql] 兼容，并支持大部分功能 - [Promise封装](/docs/documentation/promise-wrapper) - 支持压缩 - SSL 和 [Authentication Switch](/docs/documentation/authentication-switch) -- [自定义流](/docs/documentation/extras#connecting-using-custom-stream) +- [LOAD DATA LOCAL INFILE](/docs/documentation/extras#load-data-local-infile)（本地文件与内存 `infileStream`） +- [自定义连接传输流](/docs/documentation/extras#connecting-using-custom-stream) - [连接池](/docs/#using-connection-pools) [node-mysql]: https://github.com/mysqljs/mysql diff --git a/website/test/fixtures/external-code-embed/QueryOptions.txt b/website/test/fixtures/external-code-embed/QueryOptions.txt index e12549ffce..f26196f2a7 100644 --- a/website/test/fixtures/external-code-embed/QueryOptions.txt +++ b/website/test/fixtures/external-code-embed/QueryOptions.txt @@ -60,4 +60,10 @@ export interface QueryOptions { * By specifying a function that returns a readable stream, an arbitrary stream can be sent when sending a local fs file. */ infileStreamFactory?: (path: string) => Readable; + + /** + * In-memory bytes for LOAD DATA LOCAL INFILE. Filename in SQL is ignored. + * Takes precedence over infileStreamFactory. + */ + infileStream?: Readable; } \ No newline at end of file diff --git a/website/test/resources/external-code-embed/random-methods.txt b/website/test/resources/external-code-embed/random-methods.txt index ddf6fee73f..aaaa6eca38 100644 --- a/website/test/resources/external-code-embed/random-methods.txt +++ b/website/test/resources/external-code-embed/random-methods.txt @@ -69,6 +69,12 @@ export interface QueryOptions { * By specifying a function that returns a readable stream, an arbitrary stream can be sent when sending a local fs file. */ infileStreamFactory?: (path: string) => Readable; + + /** + * In-memory bytes for LOAD DATA LOCAL INFILE. Filename in SQL is ignored. + * Takes precedence over infileStreamFactory. + */ + infileStream?: Readable; } class Pool extends EventEmitter {