[i18n] extract untracked translations and prettier logging

src/dev/i18n/extract_default_translations.js

@@ -62,11 +62,20 @@ See .i18nrc.json for the list of supported namespaces.`)
   }
 }
 
-export async function extractMessagesFromPathToMap(inputPath, targetMap, config, reporter) {
+export async function matchEntriesWithExctractors(inputPath, options = {}) {
+  const {
+    additionalIgnore = [],
+    mark = false,
+    absolute = false,
+  } = options;
+  const ignore = ['**/node_modules/**', '**/__tests__/**', '**/*.test.{js,jsx,ts,tsx}', '**/*.d.ts'].concat(additionalIgnore);
+
   const entries = await globAsync('*.{js,jsx,pug,ts,tsx,html}', {
     cwd: inputPath,
     matchBase: true,
-    ignore: ['**/node_modules/**', '**/__tests__/**', '**/*.test.{js,jsx,ts,tsx}', '**/*.d.ts'],
+    ignore,
+    mark,
+    absolute,
   });
 
   const { htmlEntries, codeEntries, pugEntries } = entries.reduce(
@@ -86,37 +95,43 @@ export async function extractMessagesFromPathToMap(inputPath, targetMap, config,
     { htmlEntries: [], codeEntries: [], pugEntries: [] }
   );
 
-  await Promise.all(
-    [
-      [htmlEntries, extractHtmlMessages],
-      [codeEntries, extractCodeMessages],
-      [pugEntries, extractPugMessages],
-    ].map(async ([entries, extractFunction]) => {
-      const files = await Promise.all(
-        filterEntries(entries, config.exclude).map(async entry => {
-          return {
-            name: entry,
-            content: await readFileAsync(entry),
-          };
-        })
-      );
-
-      for (const { name, content } of files) {
-        const reporterWithContext = reporter.withContext({ name });
-
-        try {
-          for (const [id, value] of extractFunction(content, reporterWithContext)) {
-            validateMessageNamespace(id, name, config.paths, reporterWithContext);
-            addMessageToMap(targetMap, id, value, reporterWithContext);
-          }
-        } catch (error) {
-          if (!isFailError(error)) {
-            throw error;
-          }
+  return [
+    [htmlEntries, extractHtmlMessages],
+    [codeEntries, extractCodeMessages],
+    [pugEntries, extractPugMessages],
+  ];
+}
 
-          reporterWithContext.report(error);
+export async function extractMessagesFromPathToMap(inputPath, targetMap, config, reporter) {
+  const categorizedEntries = await matchEntriesWithExctractors(inputPath);
+  return Promise.all(
+    categorizedEntries
+      .map(async ([entries, extractFunction]) => {
+        const files = await Promise.all(
+          filterEntries(entries, config.exclude).map(async entry => {
+            return {
+              name: entry,
+              content: await readFileAsync(entry),
+            };
+          })
+        );
+
+        for (const { name, content } of files) {
+          const reporterWithContext = reporter.withContext({ name });
+
+          try {
+            for (const [id, value] of extractFunction(content, reporterWithContext)) {
+              validateMessageNamespace(id, name, config.paths, reporterWithContext);
+              addMessageToMap(targetMap, id, value, reporterWithContext);
+            }
+          } catch (error) {
+            if (!isFailError(error)) {
+              throw error;
+            }
+
+            reporterWithContext.report(error);
+          }
         }
-      }
-    })
+      })
   );
 }

src/dev/i18n/index.ts

@@ -18,7 +18,11 @@
  */
 
 // @ts-ignore
-export { extractMessagesFromPathToMap } from './extract_default_translations';
+export {
+  extractMessagesFromPathToMap,
+  matchEntriesWithExctractors,
+} from './extract_default_translations';
+
 // @ts-ignore
 export { writeFileAsync, readFileAsync, normalizePath, ErrorReporter } from './utils';
 export { serializeToJson, serializeToJson5 } from './serializers';

src/dev/i18n/tasks/extract_default_translations.ts

@@ -30,7 +30,8 @@ export async function extractDefaultMessages({
   path?: string | string[];
   config: I18nConfig;
 }) {
-  const filteredPaths = filterConfigPaths(Array.isArray(path) ? path : [path || './'], config);
+  const inputPaths = Array.isArray(path) ? path : [path || './'];
+  const filteredPaths = filterConfigPaths(inputPaths, config);
   if (filteredPaths.length === 0) {
     throw createFailError(
       `${chalk.white.bgRed(
@@ -39,11 +40,15 @@ export async function extractDefaultMessages({
     );
   }
 
-  const reporter = new ErrorReporter();
-
-  const list = new Listr(
+  return new Listr(
     filteredPaths.map(filteredPath => ({
-      task: async (messages: Map<string, unknown>) => {
+      task: async ({
+        messages,
+        reporter,
+      }: {
+        messages: Map<string, unknown>;
+        reporter: ErrorReporter;
+      }) => {
         const initialErrorsNumber = reporter.errors.length;
 
         // Return result if no new errors were reported for this path.
@@ -61,14 +66,4 @@ export async function extractDefaultMessages({
       exitOnError: false,
     }
   );
-
-  try {
-    return await list.run(new Map());
-  } catch (error) {
-    if (error.name === 'ListrError' && reporter.errors.length) {
-      throw createFailError(reporter.errors.join('\n\n'));
-    }
-
-    throw error;
-  }
 }

src/dev/i18n/tasks/extract_untracked_translations.ts

@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  ErrorReporter,
+  I18nConfig,
+  matchEntriesWithExctractors,
+  normalizePath,
+  readFileAsync,
+} from '..';
+
+function filterEntries(entries: string[], exclude: string[]) {
+  return entries.filter((entry: string) =>
+    exclude.every((excludedPath: string) => !normalizePath(entry).startsWith(excludedPath))
+  );
+}
+
+export async function extractUntrackedMessages({
+  path,
+  config,
+}: {
+  path?: string | string[];
+  config: I18nConfig;
+}) {
+  const inputPaths = Array.isArray(path) ? path : [path || './'];
+  const availablePaths = Object.values(config.paths);
+  const ignore = availablePaths.concat([
+    '**/build/**',
+    '**/webpackShims/**',
+    '**/__fixtures__/**',
+    'utilities/**',
+    'built_assets/**',
+    'docs/**',
+    'optimize/**',
+    'data/**',
+    '**/target/**',
+    'tasks/**',
+    '**/test/**',
+    '**/scripts/**',
+    'src/dev/**',
+    '**/target/**',
+    'plugins/canvas/**',
+  ]);
+  const reporter = new ErrorReporter();
+  for (const inputPath of inputPaths) {
+    const categorizedEntries = await matchEntriesWithExctractors(inputPath, {
+      additionalIgnore: ignore,
+      mark: true,
+      absolute: true,
+    });
+
+    await Promise.all(
+      categorizedEntries.map(async ([entries, extractFunction]) => {
+        const files: any = await Promise.all(
+          filterEntries(entries, config.exclude)
+            .filter(entry => {
+              const normalizedEntry = normalizePath(entry);
+              return !availablePaths.some(
+                availablePath =>
+                  normalizedEntry.startsWith(`${normalizePath(availablePath)}/`) ||
+                  normalizePath(availablePath) === normalizedEntry
+              );
+            })
+            .map(async (entry: any) => ({
+              name: entry,
+              content: await readFileAsync(entry),
+            }))
+        );
+
+        for (const { name, content } of files) {
+          const reporterWithContext = reporter.withContext({ name });
+          for (const [id] of extractFunction(content, reporterWithContext)) {
+            reporterWithContext.report(`File ${name} contains i18n label (${id}).`);
+          }
+        }
+      })
+    );
+  }
+}

src/dev/i18n/tasks/index.ts

@@ -18,3 +18,4 @@
  */
 
 export { extractDefaultMessages } from './extract_default_translations';
+export { extractUntrackedMessages } from './extract_untracked_translations';

src/dev/run_i18n_check.ts

@@ -20,8 +20,8 @@
 import chalk from 'chalk';
 import Listr from 'listr';
 
-import { integrateLocaleFiles, mergeConfigs } from './i18n';
-import { extractDefaultMessages } from './i18n/tasks';
+import { ErrorReporter, integrateLocaleFiles, mergeConfigs } from './i18n';
+import { extractDefaultMessages, extractUntrackedMessages } from './i18n/tasks';
 import { createFailError, run } from './run';
 
 run(
@@ -31,6 +31,7 @@ run(
       'ignore-missing': ignoreMissing,
       'ignore-unused': ignoreUnused,
       'include-config': includeConfig,
+      'ignore-untracked': ignoreUntracked,
       fix = false,
       path,
     },
@@ -40,12 +41,13 @@ run(
       fix &&
       (ignoreIncompatible !== undefined ||
         ignoreUnused !== undefined ||
-        ignoreMissing !== undefined)
+        ignoreMissing !== undefined ||
+        ignoreUntracked !== undefined)
     ) {
       throw createFailError(
         `${chalk.white.bgRed(
           ' I18N ERROR '
-        )} none of the --ignore-incompatible, --ignore-unused or --ignore-missing is allowed when --fix is set.`
+        )} none of the --ignore-incompatible, --ignore-unused or --ignore-missing or --ignore-untracked is allowed when --fix is set.`
       );
     }
 
@@ -59,38 +61,77 @@ run(
       throw createFailError(`${chalk.white.bgRed(' I18N ERROR ')} --fix can't have a value`);
     }
 
+    if (typeof ignoreUntracked !== 'undefined' && typeof ignoreUntracked !== 'boolean') {
+      throw createFailError(
+        `${chalk.white.bgRed(' I18N ERROR ')} --ignore-untracked can't have a value`
+      );
+    }
+
     const config = await mergeConfigs(includeConfig);
-    const defaultMessages = await extractDefaultMessages({ path, config });
 
     if (config.translations.length === 0) {
       return;
     }
 
+    const extractDefaultMessagesTask = () => extractDefaultMessages({ path, config });
+
+    const compatibiltyChecksTask = () =>
+      new Listr(
+        config.translations.map(translationsPath => ({
+          task: async ({ messages }: { messages: Map<string, { message: string }> }) => {
+            // If `--fix` is set we should try apply all possible fixes and override translations file.
+            await integrateLocaleFiles(messages, {
+              sourceFileName: translationsPath,
+              targetFileName: fix ? translationsPath : undefined,
+              dryRun: !fix,
+              ignoreIncompatible: fix || !!ignoreIncompatible,
+              ignoreUnused: fix || !!ignoreUnused,
+              ignoreMissing: fix || !!ignoreMissing,
+              config,
+              log,
+            });
+          },
+          title: `Compatibility check with ${translationsPath}`,
+        })),
+        { concurrent: true, exitOnError: false }
+      );
+
+    const srcCodePaths = ['./src', './packages', './x-pack'];
+    const untrackedMessagesTask = async () =>
+      new Listr(
+        srcCodePaths.map(srcPath => ({
+          task: async () => {
+            await extractUntrackedMessages({ path: srcPath, config });
+          },
+          title: `Checking untracked messages in ${srcPath}`,
+        }))
+      );
+
     const list = new Listr(
-      config.translations.map(translationsPath => ({
-        task: async () => {
-          // If `--fix` is set we should try apply all possible fixes and override translations file.
-          await integrateLocaleFiles(defaultMessages, {
-            sourceFileName: translationsPath,
-            targetFileName: fix ? translationsPath : undefined,
-            dryRun: !fix,
-            ignoreIncompatible: fix || !!ignoreIncompatible,
-            ignoreUnused: fix || !!ignoreUnused,
-            ignoreMissing: fix || !!ignoreMissing,
-            config,
-            log,
-          });
+      [
+        {
+          title: 'Checking untracked messages',
+          enabled: () => !ignoreUntracked,
+          task: untrackedMessagesTask,
+        },
+        {
+          title: 'Extracting Default Messages',
+          task: extractDefaultMessagesTask,
         },
-        title: `Compatibility check with ${translationsPath}`,
-      })),
+        {
+          title: 'Compatibility Checks',
+          task: compatibiltyChecksTask,
+        },
+      ],
       {
-        concurrent: true,
-        exitOnError: false,
+        concurrent: false,
+        exitOnError: true,
       }
     );
 
+    const reporter = new ErrorReporter();
     try {
-      await list.run();
+      await list.run({ messages: new Map(), reporter });
     } catch (error) {
       process.exitCode = 1;
 
@@ -100,6 +141,10 @@ run(
         process.exit();
       }
 
+      if (error.name === 'ListrError' && reporter.errors.length) {
+        throw createFailError(reporter.errors.join('\n\n'));
+      }
+
       for (const e of error.errors) {
         log.error(e);
       }