Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make lz4 module optional #246

Merged
merged 4 commits into from
Apr 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion .eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,14 @@
"@typescript-eslint/no-throw-literal": "off",
"no-restricted-syntax": "off",
"no-case-declarations": "off",
"max-classes-per-file": "off"
"max-classes-per-file": "off",
"import/no-extraneous-dependencies": [
"error",
{
"devDependencies": true,
"optionalDependencies": true
}
]
}
}
]
Expand Down
4 changes: 2 additions & 2 deletions lib/DBSQLSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import IOperation from './contracts/IOperation';
import DBSQLOperation from './DBSQLOperation';
import Status from './dto/Status';
import InfoValue from './dto/InfoValue';
import { definedOrError } from './utils';
import { definedOrError, LZ4 } from './utils';
kravets-levko marked this conversation as resolved.
Show resolved Hide resolved
import CloseableCollection from './utils/CloseableCollection';
import { LogLevel } from './contracts/IDBSQLLogger';
import HiveDriverError from './errors/HiveDriverError';
Expand Down Expand Up @@ -190,7 +190,7 @@ export default class DBSQLSession implements IDBSQLSession {
...getArrowOptions(clientConfig),
canDownloadResult: options.useCloudFetch ?? clientConfig.useCloudFetch,
parameters: getQueryParameters(this.sessionHandle, options.namedParameters, options.ordinalParameters),
canDecompressLZ4Result: clientConfig.useLZ4Compression,
canDecompressLZ4Result: clientConfig.useLZ4Compression && Boolean(LZ4),
});
const response = await this.handleResponse(operationPromise);
const operation = this.createOperation(response);
Expand Down
9 changes: 7 additions & 2 deletions lib/result/ArrowResultHandler.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import LZ4 from 'lz4';
import { TGetResultSetMetadataResp, TRowSet } from '../../thrift/TCLIService_types';
import HiveDriverError from '../errors/HiveDriverError';
import IClientContext from '../contracts/IClientContext';
import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
import { ArrowBatch, hiveSchemaToArrowSchema } from './utils';
import { LZ4 } from '../utils';

export default class ArrowResultHandler implements IResultsProvider<ArrowBatch> {
protected readonly context: IClientContext;
Expand All @@ -24,6 +25,10 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
// so it's possible to infer Arrow schema from Hive schema ignoring `useArrowNativeTypes` option
this.arrowSchema = arrowSchema ?? hiveSchemaToArrowSchema(schema);
this.isLZ4Compressed = lz4Compressed ?? false;

if (this.isLZ4Compressed && !LZ4) {
throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
}
}

public async hasMore() {
Expand All @@ -47,7 +52,7 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
let totalRowCount = 0;
rowSet?.arrowBatches?.forEach(({ batch, rowCount }) => {
if (batch) {
batches.push(this.isLZ4Compressed ? LZ4.decode(batch) : batch);
batches.push(this.isLZ4Compressed ? LZ4!.decode(batch) : batch);
totalRowCount += rowCount.toNumber(true);
}
});
Expand Down
9 changes: 7 additions & 2 deletions lib/result/CloudFetchResultHandler.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import LZ4 from 'lz4';
import fetch, { RequestInfo, RequestInit, Request } from 'node-fetch';
import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types';
import HiveDriverError from '../errors/HiveDriverError';
import IClientContext from '../contracts/IClientContext';
import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
import { ArrowBatch } from './utils';
import { LZ4 } from '../utils';

export default class CloudFetchResultHandler implements IResultsProvider<ArrowBatch> {
protected readonly context: IClientContext;
Expand All @@ -24,6 +25,10 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
this.context = context;
this.source = source;
this.isLZ4Compressed = lz4Compressed ?? false;

if (this.isLZ4Compressed && !LZ4) {
throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
}
}

public async hasMore() {
Expand Down Expand Up @@ -58,7 +63,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
}

if (this.isLZ4Compressed) {
batch.batches = batch.batches.map((buffer) => LZ4.decode(buffer));
batch.batches = batch.batches.map((buffer) => LZ4!.decode(buffer));
}
return batch;
}
Expand Down
3 changes: 2 additions & 1 deletion lib/utils/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import definedOrError from './definedOrError';
import buildUserAgentString from './buildUserAgentString';
import formatProgress, { ProgressUpdateTransformer } from './formatProgress';
import LZ4 from './lz4';

export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer };
export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer, LZ4 };
16 changes: 16 additions & 0 deletions lib/utils/lz4.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import type LZ4Namespace from 'lz4';

type LZ4Module = typeof LZ4Namespace;

function tryLoadLZ4Module(): LZ4Module | undefined {
try {
return require('lz4'); // eslint-disable-line global-require
} catch (err) {
const isModuleNotFoundError = err instanceof Error && 'code' in err && err.code === 'MODULE_NOT_FOUND';
if (!isModuleNotFoundError) {
throw err;
}
}
}

export default tryLoadLZ4Module();
34 changes: 25 additions & 9 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@
"dependencies": {
"apache-arrow": "^13.0.0",
"commander": "^9.3.0",
"lz4": "^0.6.5",
"node-fetch": "^2.6.12",
"node-int64": "^0.4.0",
"open": "^8.4.2",
Expand All @@ -83,5 +82,8 @@
"thrift": "^0.16.0",
"uuid": "^9.0.0",
"winston": "^3.8.2"
},
"optionalDependencies": {
"lz4": "^0.6.5"
}
}
Loading