diff --git a/package-lock.json b/package-lock.json index 3d98c5a..ba60f5e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,19 @@ { "name": "@foxxmd/logging", - "version": "0.1.9", + "version": "0.1.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@foxxmd/logging", - "version": "0.1.9", + "version": "0.1.10", "license": "MIT", "dependencies": { "pino": "^8.19.0", + "pino-abstract-transport": "^1.1.0", "pino-pretty": "^11.0.0", - "pino-roll": "^1.0.1" + "pino-roll": "^1.0.1", + "pump": "^3.0.0" }, "devDependencies": { "@types/chai": "^4.3.0", @@ -19,6 +21,7 @@ "@types/dateformat": "^5.0.2", "@types/mocha": "^9.1.0", "@types/node": "^18.0.0", + "@types/pump": "^1.1.3", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "dateformat": "^5.0.3", @@ -607,6 +610,15 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/pump": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@types/pump/-/pump-1.1.3.tgz", + "integrity": "sha512-ZyooTTivmOwPfOwLVaszkF8Zq6mvavgjuHYitZhrIjfQAJDH+kIP3N+MzpG1zDAslsHvVz6Q8ECfivix3qLJaQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", diff --git a/package.json b/package.json index e365ff5..d4a2ee9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@foxxmd/logging", "type": "module", - "version": "0.1.9", + "version": "0.1.10", "repository": "https://github.com/foxxmd/logging", "description": "A typed, opinionated, batteries-included, Pino-based logging solution for backend TS/JS projects", "scripts": { @@ -49,6 +49,7 @@ "@types/dateformat": "^5.0.2", "@types/mocha": "^9.1.0", "@types/node": "^18.0.0", + "@types/pump": "^1.1.3", "chai": "^4.3.6", "chai-as-promised": "^7.1.1", "dateformat": "^5.0.3", @@ -68,8 +69,10 @@ }, "dependencies": { "pino": "^8.19.0", + "pino-abstract-transport": "^1.1.0", "pino-pretty": "^11.0.0", - "pino-roll": "^1.0.1" + "pino-roll": "^1.0.1", + "pump": "^3.0.0" }, "overrides": { "with-local-tmp-dir": { diff --git a/src/destinations.ts b/src/destinations.ts index e9461f5..ccf536c 100644 --- a/src/destinations.ts +++ b/src/destinations.ts @@ -3,13 +3,16 @@ import { LogLevelStreamEntry, LogLevel, StreamDestination, - FileDestination, + FileDestination, JsonPrettyDestination, } from "./types.js"; -import {destination} from "pino"; -import {build} from "pino-pretty" +import {destination, DestinationStream} from "pino"; +import {build, prettyFactory} from "pino-pretty" import {PRETTY_OPTS_FILE, prettyOptsConsoleFactory, prettyOptsFileFactory} from "./pretty.js"; import {fileOrDirectoryIsWriteable} from "./util.js"; import path from "path"; +import {Transform, TransformCallback} from "node:stream"; +import pump from 'pump'; +import abstractTransport from 'pino-abstract-transport'; const pRoll = pinoRoll as unknown as typeof pinoRoll.default; @@ -113,6 +116,75 @@ export const buildDestinationStream = (level: LogLevel, options: StreamDestinati } } +/** + * Creates a `LogLevelStreamEntry` stream that writes the raw log data with prettified log line, as either a JSON string or object, to a `NodeJs.WriteableStream` or [Sonic Boom `DestinationStream`](https://github.com/pinojs/sonic-boom) at or above the minimum `level` + * + * Use this to get raw log data + formatted line rendered by pino-pretty. The prettified line is set to the `line` key in the object. + * + * WARNING: This is not a fast operation. The log data must be parsed to json before being prettified, which also parses data to json. If you only need raw log data you should pass a plain Passthrough or Transform stream as an additional destination and then JSON.parse() on 'data' event manually. + * + * If used with `object: true` then + * + * * the `destination` cannot be a file or SonicBoom object + * * the `destination` stream passed MUST be set to `objectMode: true` + * + * @see DestinationStream + * */ +export const buildDestinationJsonPrettyStream = (level: LogLevel, options: JsonPrettyDestination): LogLevelStreamEntry => { + const {object = false} = options; + const factoryOpts = prettyOptsConsoleFactory(options); + const prettyFunc = prettyFactory(factoryOpts); + + const stream = new Transform({ + objectMode: object, + autoDestroy: true, + transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback) { + const data = JSON.parse(chunk.toString()); + const line = prettyFunc(chunk); + data.line = line.substring(0, line.length - 1); + if(object) { + callback(null, data); + } else { + callback(null, `${JSON.stringify(data)}\n`); + } + } + }); + + let destinationStream: DestinationStream | NodeJS.WritableStream; + + if(typeof options.destination === 'object' && typeof options.destination.write === 'function') { + destinationStream = options.destination; + } else { + destinationStream = destination({...factoryOpts, dest: stream}); + } + + if(object) { + // @ts-ignore + pump(stream, destinationStream) + + return { + level: level, + stream: stream + } + } + + // @ts-ignore + const transport = abstractTransport(function (source) { + source.on('unknown', function (line) { + destinationStream.write(line + '\n') + }) + + // @ts-ignore + pump(source, stream, destinationStream); + return stream; + }, { parse: 'lines' }); + + return { + level: level, + stream: transport + } +} + /** * Creates a `LogLevelStreamEntry` stream that writes to STDOUT at or above the minimum `level` * diff --git a/src/factory.ts b/src/factory.ts index 44192a7..79a0f77 100644 --- a/src/factory.ts +++ b/src/factory.ts @@ -9,13 +9,14 @@ import { } from "./pretty.js"; import { buildDestinationStream, + buildDestinationJsonPrettyStream, buildDestinationRollingFile, buildDestinationStdout, buildDestinationStderr, buildDestinationFile } from "./destinations.js"; import {buildLogger} from './loggers.js'; -import {FileDestination, PRETTY_COLORS, PRETTY_COLORS_STR, PRETTY_ISO8601, StreamDestination} from './types.js' +import {FileDestination, PRETTY_COLORS, PRETTY_COLORS_STR, PRETTY_ISO8601, StreamDestination, JsonPrettyDestination} from './types.js' export { PRETTY_OPTS_CONSOLE, @@ -29,6 +30,7 @@ export { prettyOptsFileFactory, prettyOptsConsoleFactory, buildDestinationStream, + buildDestinationJsonPrettyStream, buildDestinationStdout, buildDestinationStderr, buildDestinationFile, @@ -38,5 +40,6 @@ export { export type { FileDestination, - StreamDestination + StreamDestination, + JsonPrettyDestination } diff --git a/src/types.ts b/src/types.ts index 990d8d5..a78c18c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -155,6 +155,12 @@ export type FileLogOptionsParsed = (Omit & {level: false export type FileDestination = Omit & FileOptionsParsed; export type StreamDestination = Omit & {destination: number | DestinationStream | NodeJS.WritableStream}; +export type JsonPrettyDestination = StreamDestination & { + /** + * Specify if the stream should output log as object or stringified JSON + * */ + object?: boolean +}; export type LogOptionsParsed = Omit, 'file'> & { file: FileLogOptionsParsed } diff --git a/tests/index.test.ts b/tests/index.test.ts index fdef798..9353b4c 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -14,7 +14,12 @@ import {sleep} from "../src/util.js"; import {LogData, LOG_LEVEL_NAMES, PRETTY_ISO8601} from "../src/types.js"; import withLocalTmpDir from 'with-local-tmp-dir'; import {readdirSync,} from 'node:fs'; -import {buildDestinationStream, buildDestinationRollingFile, buildDestinationFile} from "../src/destinations.js"; +import { + buildDestinationStream, + buildDestinationRollingFile, + buildDestinationFile, + buildDestinationJsonPrettyStream +} from "../src/destinations.js"; import {buildLogger} from "../src/loggers.js"; import {readFileSync} from "fs"; import path from "path"; @@ -44,6 +49,28 @@ const testConsoleLogger = (config?: object): [Logger, Transform, Transform] => { return [logger, testStream, rawStream]; } +const testObjectLogger = (config?: object, object?: boolean): [Logger, Transform, Transform] => { + const opts = parseLogOptions(config, process.cwd()); + const testStream = new PassThrough({objectMode: true}); + const rawStream = new PassThrough(); + const logger = buildLogger('debug', [ + buildDestinationJsonPrettyStream( + opts.console, + { + destination: testStream, + object, + colorize: false, + ...opts + } + ), + { + level: opts.console, + stream: rawStream + } + ]); + return [logger, testStream, rawStream]; +} + const testFileRollingLogger = async (config?: object, logBaseDir = process.cwd()) => { const opts = parseLogOptions(config, logBaseDir); const { @@ -220,6 +247,33 @@ describe('Transports', function () { }); }); + describe('Pretty Object Stream', function () { + it('Writes pretty line to stream as jsonified object', async function () { + const [defaultLogger, testStream] = testObjectLogger(undefined, false); + const race = Promise.race([ + pEvent(testStream, 'data'), + sleep(10) + ]) as Promise; + defaultLogger.debug('Test'); + const res = (await race).toString(); + expect(res).to.not.be.undefined; + expect(res).match(/"line":".*\sDEBUG\s*:\s*Test"/).is.not.null; + }); + + it('Writes pretty line to stream as object', async function () { + const [defaultLogger, testStream] = testObjectLogger(undefined, true); + const race = Promise.race([ + pEvent(testStream, 'data'), + sleep(10) + ]) as Promise; + defaultLogger.debug('Test'); + const res = await race; + expect(res).to.not.be.undefined; + expect(res.line).to.not.be.undefined; + expect(res.line).match(/DEBUG\s*:\s*Test/).is.not.null; + }); + }); + describe('File', async function () { it('Does NOT write to file when file is false', async function () {