diff --git a/lib/cli/cli.js b/lib/cli/cli.js index 3520073209..7c6460f349 100644 --- a/lib/cli/cli.js +++ b/lib/cli/cli.js @@ -249,6 +249,34 @@ export async function parseCommandLine() { ' to increase the level of detail.', type: 'count' }) + /* + OpenSearch cli Options + */ + + .option('opensearch.host', { + describe: 'The OpenSearch host', + group: 'OpenSearch' + }) + .option('opensearch.port', { + describe: 'The OpenSearch port', + type: 'number', + default: 9200, + group: 'OpenSearch' + }) + .option('opensearch.index', { + describe: 'The index name for storing metrics', + default: 'sitespeed', + group: 'OpenSearch' + }) + .option('opensearch.auth.username', { + describe: 'OpenSearch authentication username', + group: 'OpenSearch' + }) + .option('opensearch.auth.password', { + describe: 'OpenSearch authentication password', + group: 'OpenSearch' + }) + /* Browsertime cli options */ diff --git a/lib/plugins/opensearch/index.js b/lib/plugins/opensearch/index.js new file mode 100644 index 0000000000..59d03bad68 --- /dev/null +++ b/lib/plugins/opensearch/index.js @@ -0,0 +1,118 @@ +import { SitespeedioPlugin } from '@sitespeed.io/plugin'; +import { getLogger } from '@sitespeed.io/log'; +import { Client } from '@opensearch-project/opensearch'; + +const log = getLogger('sitespeedio.plugin.opensearch'); + +export default class OpenSearchPlugin extends SitespeedioPlugin { + constructor(options, context, queue) { + super({ name: 'opensearch', options, context, queue }); + } + + open(context, options) { + this.opensearchOptions = options.opensearch; + this.index = this.opensearchOptions?.index; + this.options = options; + this.make = context.messageMaker('opensearch').make; + this.storageManager = context.storageManager; + + // Initialize OpenSearch client connection + // Register a logger for this plugin, a unique name so we can filter the log + // And save the log for later + this.log = context.getLogger('sitespeedio.plugin.opensearch'); + this.log.info('Plugin opensearch started'); + this.initializeClient(); + } + + initializeClient() { + const { + host, + port, + protocol, + auth + } = this.opensearchOptions || {}; + + if (!host || !port) { + throw new Error('OpenSearch host and port must be defined'); + } + + this.client = new Client({ + node: `${protocol}://${host}:${port}`, + auth: auth?.username + ? { + username: auth.username, + password: auth.password + } + : undefined, + ssl: { + rejectUnauthorized: false + } + }); + + log.info(`Connected to ${protocol}://${host}:${port}`); + } + + async processMessage(message, queue) { + + if (message.type === 'browsertime.pageSummary') { + //log.info('Received browsertime.pageSummary message data:'); + //log.info(JSON.stringify(message.data, null, 2)); + + try { + await this.sendMetricsToOpenSearch(message.data); + } catch (error) { + log.error('Failed to send metrics to OpenSearch', error); + } + } + } + + async sendMetricsToOpenSearch(data) { + + const document = this.transformMetrics(data); + + try { + const response = await this.client.index({ + index: this.index, + body: document, + refresh: false + }); + + log.info(`Indexed into ${this.index} id=${response.body?._id}`); + + } catch (err) { + log.error('OpenSearch indexing failed'); + log.error(JSON.stringify(err.meta?.body, null, 2)); + throw err; + } + } + + transformMetrics(data) { + const webVitals = data.googleWebVitals?.[0] || {}; + const browserTimings = + data.browserScripts?.[0]?.timings?.pageTimings || {}; + + return { + timestamp: new Date().toISOString(), + url: data.info?.url || 'unknown', + + lcp: webVitals.largestContentfulPaint ?? 0, + cls: webVitals.cumulativeLayoutShift ?? 0, + + browserpbbackendtime: browserTimings.backEndTime ?? 0, + browserpbdomcontentloadedtime: + browserTimings.domContentLoadedTime ?? 0, + browserpbdominteractivetime: + browserTimings.domInteractiveTime ?? 0, + browserpbdomainlookuptime: + browserTimings.domainLookupTime ?? 0, + browserpbfrontEndTime: + browserTimings.frontEndTime ?? 0, + browserpbpageLoadTime: + browserTimings.pageLoadTime ?? 0, + browserpbserverConnectionTime: + browserTimings.serverConnectionTime ?? 0, + browserpbserverResponseTime: + browserTimings.serverResponseTime ?? 0 + }; + } +} diff --git a/lib/plugins/opensearch/package-lock.json b/lib/plugins/opensearch/package-lock.json new file mode 100644 index 0000000000..e939e6ab51 --- /dev/null +++ b/lib/plugins/opensearch/package-lock.json @@ -0,0 +1,87 @@ +{ + "name": "sitespeed-opensearch-plugin", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "sitespeed-opensearch-plugin", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@opensearch-project/opensearch": "^2.8.0" + } + }, + "node_modules/@opensearch-project/opensearch": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-2.13.0.tgz", + "integrity": "sha512-Bu3jJ7pKzumbMMeefu7/npAWAvFu5W9SlbBow1ulhluqUpqc7QoXe0KidDrMy7Dy3BQrkI6llR3cWL4lQTZOFw==", + "license": "Apache-2.0", + "dependencies": { + "aws4": "^1.11.0", + "debug": "^4.3.1", + "hpagent": "^1.2.0", + "json11": "^2.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0" + }, + "engines": { + "node": ">=10", + "yarn": "^1.22.10" + } + }, + "node_modules/aws4": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/json11": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/json11/-/json11-2.0.2.tgz", + "integrity": "sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ==", + "license": "MIT", + "bin": { + "json11": "dist/cli.mjs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + } + } +} diff --git a/lib/plugins/opensearch/package.json b/lib/plugins/opensearch/package.json new file mode 100644 index 0000000000..1440e31e80 --- /dev/null +++ b/lib/plugins/opensearch/package.json @@ -0,0 +1,21 @@ +{ + "name": "sitespeed-opensearch-plugin", + "version": "1.0.0", + "description": "Custom Sitespeed.io plugin to send metrics to OpenSearch", + "type": "module", + "main": "index.js", + "keywords": [ + "sitespeed", + "opensearch", + "performance", + "plugin" + ], + "author": "Shivaram", + "license": "MIT", + "dependencies": { + "@opensearch-project/opensearch": "^2.8.0" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + } +} diff --git a/package.json b/package.json index 2161e1eb38..b6c7c47c4c 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "dependencies": { "@aws-sdk/client-s3": "3.911.0", "@google-cloud/storage": "7.17.2", + "@opensearch-project/opensearch": "^2.0.0", "@sitespeed.io/log": "1.0.0", "@sitespeed.io/plugin": "1.0.0", "@tgwf/co2": "0.16.9", diff --git a/test/opensearchTest.js b/test/opensearchTest.js new file mode 100644 index 0000000000..59fb1e849d --- /dev/null +++ b/test/opensearchTest.js @@ -0,0 +1,170 @@ +import test from 'ava'; +// eslint-disable-next-line unicorn/no-named-default +import { default as OpenSearchPlugin } from '../lib/plugins/opensearch/index.js'; + +// ---- Minimal Sitespeed context ---- +function createContext() { + return { + messageMaker: () => ({ + make: () => {} + }), + storageManager: {}, + getLogger: () => ({ + info: () => {}, + error: () => {} + }) + }; +} + +function createValidOptions() { + return { + opensearch: { + host: 'localhost', + port: 9200, + protocol: 'http', + index: 'test-index' + } + }; +} + +test('plugin initializes correctly with valid config', t => { + const context = createContext(); + const options = createValidOptions(); + + const plugin = new OpenSearchPlugin(options, context); + plugin.open(context, options); + + t.is(plugin.index, 'test-index'); + t.truthy(plugin.client); +}); + +test('initializeClient throws when host missing', t => { + const context = createContext(); + const options = { + opensearch: { + port: 9200, + protocol: 'http', + index: 'test-index' + } + }; + + const plugin = new OpenSearchPlugin(options, context); + + const error = t.throws(() => { + plugin.open(context, options); + }); + + t.is(error.message, 'OpenSearch host and port must be defined'); +}); + +test('transformMetrics extracts metrics correctly', t => { + const plugin = new OpenSearchPlugin({}, createContext()); + + const input = { + info: { url: 'https://example.com' }, + googleWebVitals: [ + { + largestContentfulPaint: 2500, + cumulativeLayoutShift: 0.2 + } + ], + browserScripts: [ + { + timings: { + pageTimings: { + backEndTime: 50, + domContentLoadedTime: 100, + domInteractiveTime: 120, + domainLookupTime: 20, + frontEndTime: 300, + pageLoadTime: 800, + serverConnectionTime: 30, + serverResponseTime: 60 + } + } + } + ] + }; + + const result = plugin.transformMetrics(input); + + t.is(result.url, 'https://example.com'); + t.is(result.lcp, 2500); + t.is(result.cls, 0.2); + t.is(result.browserpbbackendtime, 50); + t.is(result.browserpbpageLoadTime, 800); +}); + +test('transformMetrics handles missing data safely', t => { + const plugin = new OpenSearchPlugin({}, createContext()); + + const result = plugin.transformMetrics({}); + + t.is(result.url, 'unknown'); + t.is(result.lcp, 0); + t.is(result.cls, 0); + t.is(result.browserpbpageLoadTime, 0); +}); + +test('sendMetricsToOpenSearch calls client.index', async t => { + const context = createContext(); + const options = createValidOptions(); + + const plugin = new OpenSearchPlugin(options, context); + plugin.open(context, options); + + let indexCalled = false; + + plugin.client = { + index: async params => { + indexCalled = true; + t.is(params.index, 'test-index'); + t.truthy(params.body); + t.is(params.refresh, false); + return { body: { _id: '123' } }; + } + }; + + await plugin.sendMetricsToOpenSearch({ + info: { url: 'https://example.com' } + }); + + t.true(indexCalled); +}); + +test('processMessage triggers indexing for browsertime.pageSummary', async t => { + const context = createContext(); + const options = createValidOptions(); + + const plugin = new OpenSearchPlugin(options, context); + plugin.open(context, options); + + let called = false; + + plugin.sendMetricsToOpenSearch = async () => { + called = true; + }; + + await plugin.processMessage({ + type: 'browsertime.pageSummary', + data: { info: { url: 'https://example.com' } } + }); + + t.true(called); +}); + +test('processMessage ignores other message types', async t => { + const plugin = new OpenSearchPlugin({}, createContext()); + + let called = false; + plugin.sendMetricsToOpenSearch = async () => { + called = true; + }; + + await plugin.processMessage({ + type: 'other.message', + data: {} + }); + + t.false(called); +});