Data export to opensearch (#3)

* chore(server): add dep to opensearch client

* feature(server): introduce opensearch data-export

* chore(module): set module name to lowercase

* test(module): update name

* test(module): update name

* chore(module): clean new output bundles correctly

* feature(server): add export ending log

* feature(server): add export starting log

* doc(README): update documentation for data export

* doc(TODO/FIXME): update lists

* doc(README): add opensearch graph

* doc(README): fix CI branch

* doc(TODO): update list

* chore(eslint): enable singlequotes rule

* chore(eslint): fix warnings

* refactor(helper): clean exporter code up

* doc(TODO/FIXME): update contents

* refactor(helper): typo

* test(helper): add unit tests

* test(helper): attempts to fix date UTs

* refactor(helper): attempts to build TZ independent dates

* refactor(scripts): remove deprecated data exporter script

* doc(TODO/FIXME): update lists

* chore(release): update release version

.eslintrc.cjs

@@ -3,4 +3,7 @@ module.exports = {
   parser: '@typescript-eslint/parser',
   plugins: ['@typescript-eslint', 'react', 'react-hooks'],
   root: true,
+  rules: {
+    quotes: ['warn', 'single'],
+  }
 };

.gitignore

@@ -107,8 +107,7 @@ dist
 .vscode/
 coverage/
 node_modules/
-MMM-*.js
-MMM-*.js.LICENSE.txt
+mmm-*.js*
 node_helper.js
 stats.json
 styles.css

README.md

@@ -72,6 +72,28 @@ Per current day/month/year and total:
 - Supplied energy is detailed according to the chosen fare option (provided by Teleinfo data: ``BASE``, ``HC``, ``EJP``)
 - Estimated costs are computed in respect to current fare option and configured fare details (see configuration section below). Please note they don't include subscription and extra furniture costs.
 
+
+### Exports local data to Opensearch index
+
+More info about [Opensearch](https://opensearch.org/docs/latest/). This features requires a fully working  Opensearch node + Opensearch dashboards stack (using Docker containers is supported and recommended). 
+
+![Opensearch 1](https://github.com/djey47/MMM-LKY-TIC/blob/main/doc/shots/Opensearch1.png?raw=true)
+
+![Opensearch 2](https://github.com/djey47/MMM-LKY-TIC/blob/main/doc/shots/Opensearch2.png?raw=true)
+
+When enabled (see *Configuration -> Teleinfo* section below), collected data is exported to the Opensearch index of your choice. Using this, you'll be able to display data in Opensearch dashboards (see above screenshot).
+
+Examples of index mapping can be found here: `data-export/opensearch/mappings`.
+
+**Notes:**
+
+- For now, only per-day data is exported
+- Export is processed at following events:
+  - Start of current MagicMirror module
+  - When MagicMirror and this module are running, and a day change is detected, so at 12 a.m.
+- Make sure the Opensearch instance is accessible from you MagicMirror device, and ready to receive data; as no immediate retry is attempted on failure. The next export occurrence will be either the day after, or if you decide to restart MagicMirror 
+- The index might not exist already, it will be created when necessary.
+
 ## Install
 
 ### Prerequisites
@@ -112,6 +134,17 @@ Since this module relies on `serialport` npm dependency which uses native module
 {
   "baudRate": 1200,
   "dataBits": 7,
+  "dataExport": {
+    "target": "opensearch",
+    "settings": {
+      "opensearch": {
+        "index-name": "my-lky-per-day-index",
+        "instance": "https://my-nas:9200",
+        "user": "os-user",
+        "password": "os-password"
+      }
+    }
+  }
   "developer": {
     "serialPortMockEnabled": false,
     "mockRefreshRate": 2500
@@ -131,6 +164,13 @@ Since this module relies on `serialport` npm dependency which uses native module
 
 - `baudRate`: transfer speed for serial link
 - `dataBits` and `stopBits`: data encoding in bits number from the serial link
+- `dataExport`: provides configuration for automatic data export (not mandatory)
+  - `target`: system to export to
+  - `settings`: per-system configuration
+    - `opensearch`
+      - `indexName`: Opensearch index to send data documents to
+      - `instance`: URL of ready-to-use Opensearch instance
+      - `user` and `password`: instance credentials 
 - `developer`: advanced settings
   - `serialPortMockEnabled`: enables (true) or disables (false) serial port emulation
   - `mockRefreshRate`: interval in ms for the emulator to produce mock teleinfo
@@ -156,15 +196,6 @@ Small CLI program to diagnose reading of teleinfo data on serial input: `npm run
 
 Configuration is set via `tools/scripts/teleinfo-reader/config/teleinfo-reader.json` file; see 'Teleinfo section' above.
 
-### opensearch-data-export
-This script will push info from the persisted datastore to an opensearch index: `npm run tool:opensearch-data-export <data store file> <index name>`.
-
-- `data store file`: path to the data store JSON file
-- `index name`: index name to use on opensearch instance
-
-Configuration is set via `tools/scripts/opensearch-data-export/config/opensearch-data-export.json`.
-
-
 ### Diagnostics with picocom
 It's also possible to capture raw data on serial input, using `picocom` command line tool: https://linux.die.net/man/8/picocom
 

doc/TODO-FIXME.md

@@ -2,7 +2,13 @@ TODO-FIXME
 ==========
 
 # TODO
-- store indexes at configured hours for historization
+- [front-end] basic/advanced display modes
+  - Basic only shows most important info (status bar, current supply, daily costs...)
+  - Advanced shows more pages but with auto scroll carroussel-like
+  - Configuration: mode (basic default), page persistance time...
+- costs: take into account the subscribe fees
+  - annual fee to be set in configuration (or monthly?)
+  - compute global cost : add all fare period costs + fee / day, fee / month, fee / year, total fee accordingly 
 - quick status display with icons/labels
   - [x] connection status to TIC: disconnected (no data or too old data received) or connected
     => does not work for now as the component does not update till new data arrives (see withNotification HOC to trigger the refresh every 5 seconds via setInterval - ? requires a retry count in state ?)
@@ -27,10 +33,15 @@ TODO-FIXME
 - unit tests for teleinfo processing
 
 # FIXME
-- teleinfo-reader utility does not work anymore because of MM2 interfaces (node_helper and log)
+- [ ] cannot npm install on dev env anymore (WIN-WSL...) due to native modules
+- [ ] teleinfo-reader utility does not work anymore because of MM2 interfaces (node_helper and log)
   => needs to be remade without any link to MM2, datastore, advanced stats
 
 # DONE
+- [x] Data export to Opensearch dashboards
+- store indexes at configured hours for historization
+  - [x] Manual data store indexing
+  - [x] per-day auto indexing to opensearch
 - provide interfaces for stored items
 - SIGINT signal sent by PM2 does not seem to execute async stop call
   => data store persist is not called

package.json

@@ -1,6 +1,6 @@
 {
   "name": "mmm-lky-tic",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "MagicMirror² module to display info about home power supply - FRANCE TELEINFO ENEDIS.",
   "main": "index.js",
   "license": "MIT",
@@ -13,7 +13,7 @@
     "lint:styles": "stylelint \"src/**/*.scss\"",
     "test": "jest",
     "test:watch": "jest --watch",
-    "clean:all": "rm -f MMM-*.js* node_helper.js styles.css",
+    "clean:all": "rm -f MMM-*.js* mmm-*.js* node_helper.js styles.css",
     "build:module": "npm run clean:all && npm run build:client && npm run build:helper",
     "build:module-prod": "npm run clean:all && npm run build:client-prod && npm run build:helper-prod",
     "build:client": "webpack --mode=development --config tools/webpack/webpack.config.module.js",
@@ -23,7 +23,6 @@
     "prettier:write": "prettier --write .",
     "prettier:check": "prettier --check .",
     "tool:teleinfo-reader": "ts-node ./tools/scripts/teleinfo-reader/teleinfo-reader.ts",
-    "tool:opensearch-data-export": "ts-node ./tools/scripts/opensearch-data-export/opensearch-data-export.ts",
     "dev": "chokidar \"./src/**/*\" \"./support/**/*\" \"./tools/webpack/*\" -i \"./**/*.spec.*\" -c \"npm run build:client\""
   },
   "keywords": [
@@ -36,7 +35,6 @@
   ],
   "devDependencies": {
     "@electron/rebuild": "3.3.0",
-    "@opensearch-project/opensearch": "^2.4.0",
     "@types/app-root-dir": "0.1.1",
     "@types/edit-json-file": "1.7.0",
     "@types/jest": "29.2.5",
@@ -77,6 +75,7 @@
     "@fortawesome/free-regular-svg-icons": "6.3.0",
     "@fortawesome/free-solid-svg-icons": "6.3.0",
     "@fortawesome/react-fontawesome": "0.2.0",
+    "@opensearch-project/opensearch": "2.4.0",
     "@serialport/bindings-cpp": "12.0.1",
     "classnames": "2.3.2",
     "date-fns": "2.29.3",

src/client/hooks/with-notifications/notification-catcher.spec.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/ban-ts-comment */
 
-import { NotificationCatcher } from "./notification-catcher";
+import { NotificationCatcher } from './notification-catcher';
 
 describe('NotificationCatcher class', () => {
   beforeAll(() => {

src/client/module.spec.ts

@@ -44,7 +44,7 @@ describe('MM2 Module client', () => {
     const { name, implementation } = checkAndExtractRegistration(
       mockModuleRegister.mock.lastCall
     );
-    expect(name).toBe('MMM-LKY-TIC');
+    expect(name).toBe('mmm-lky-tic');
     expect(implementation.defaults).toEqual({
       currencySymbol: '€',
       debug: false,

src/client/module.ts

@@ -11,7 +11,7 @@ import { NotificationCatcher } from './hooks/with-notifications/notification-cat
 /**
  * Custom MM2 module name
  */
-const MODULE_NAME = 'MMM-LKY-TIC';
+const MODULE_NAME = 'mmm-lky-tic';
 
 /**
  * @private

src/server/processing/teleinfo/data-export/helpers/opensearch-client.test.ts

@@ -0,0 +1,37 @@
+import { Client } from '@opensearch-project/opensearch';
+import { createOpenSearchClient } from './opensearch-client';
+
+import type { OpensearchConfiguration } from '../../../../../shared/domain/teleinfo-config';
+
+jest.mock('@opensearch-project/opensearch');
+const opensearchClientMock = Client as jest.Mock;
+
+describe('Opensearch client helper functions', () => {
+  describe('createOpenSearchClient function', () => {
+    beforeEach(() => {
+      opensearchClientMock.mockReset();
+    });
+
+    it('should return an initialized client', () => {
+      // given
+      const config: OpensearchConfiguration = {
+        indexName: '',
+        instance: 'http://osinstance:9200',
+        user: 'USER',
+        password: 'PASS'
+      };
+
+      // when
+      const actualClient = createOpenSearchClient(config);
+
+      // then
+      expect(actualClient).not.toBeUndefined();
+      expect(opensearchClientMock).toHaveBeenCalledWith({
+        node: 'http://USER:PASS@osinstance:9200',
+        ssl: {
+          rejectUnauthorized: false,
+        },
+      });
+    });
+  });
+});

src/server/processing/teleinfo/data-export/helpers/opensearch-client.ts

@@ -0,0 +1,16 @@
+import { Client } from '@opensearch-project/opensearch';
+import { OpensearchConfiguration } from '../../../../../shared/domain/teleinfo-config';
+
+export function createOpenSearchClient(config: OpensearchConfiguration) {
+  const { instance, user, password } = config;
+  const instanceURL = new URL(instance);
+
+  // Create a client with SSL/TLS enabled.
+  const client = new Client({
+    node: `${instanceURL.protocol}//${user}:${password}@${instanceURL.hostname}:${instanceURL.port}`,
+    ssl: {
+      rejectUnauthorized: false,
+    },
+  });
+  return client;
+}