[i18n] extract untracked translations and prettier logging

.i18nrc.json

@@ -10,6 +10,7 @@
     "interpreter": "src/legacy/core_plugins/interpreter",
     "kbn": "src/legacy/core_plugins/kibana",
     "kbnDocViews": "src/legacy/core_plugins/kbn_doc_views",
+    "kbnEmbeddables": "src/legacy/core_plugins/embeddable_api",
     "kbnVislibVisTypes": "src/legacy/core_plugins/kbn_vislib_vis_types",
     "markdownVis": "src/legacy/core_plugins/markdown_vis",
     "metricVis": "src/legacy/core_plugins/metric_vis",
@@ -25,6 +26,7 @@
     "xpack.apm": "x-pack/plugins/apm",
     "xpack.beatsManagement": "x-pack/plugins/beats_management",
     "xpack.canvas": "x-pack/plugins/canvas",
+    "xpack.code": "x-pack/plugins/code",
     "xpack.crossClusterReplication": "x-pack/plugins/cross_cluster_replication",
     "xpack.dashboardMode": "x-pack/plugins/dashboard_mode",
     "xpack.graph": "x-pack/plugins/graph",

src/dev/i18n/extract_default_translations.js

@@ -62,11 +62,20 @@ See .i18nrc.json for the list of supported namespaces.`)
   }
 }
 
-export async function extractMessagesFromPathToMap(inputPath, targetMap, config, reporter) {
+export async function matchEntriesWithExctractors(inputPath, options = {}) {
+  const {
+    additionalIgnore = [],
+    mark = false,
+    absolute = false,
+  } = options;
+  const ignore = ['**/node_modules/**', '**/__tests__/**', '**/*.test.{js,jsx,ts,tsx}', '**/*.d.ts'].concat(additionalIgnore);
+
   const entries = await globAsync('*.{js,jsx,pug,ts,tsx,html}', {
     cwd: inputPath,
     matchBase: true,
-    ignore: ['**/node_modules/**', '**/__tests__/**', '**/*.test.{js,jsx,ts,tsx}', '**/*.d.ts'],
+    ignore,
+    mark,
+    absolute,
   });
 
   const { htmlEntries, codeEntries, pugEntries } = entries.reduce(
@@ -86,37 +95,43 @@ export async function extractMessagesFromPathToMap(inputPath, targetMap, config,
     { htmlEntries: [], codeEntries: [], pugEntries: [] }
   );
 
-  await Promise.all(
-    [
-      [htmlEntries, extractHtmlMessages],
-      [codeEntries, extractCodeMessages],
-      [pugEntries, extractPugMessages],
-    ].map(async ([entries, extractFunction]) => {
-      const files = await Promise.all(
-        filterEntries(entries, config.exclude).map(async entry => {
-          return {
-            name: entry,
-            content: await readFileAsync(entry),
-          };
-        })
-      );
-
-      for (const { name, content } of files) {
-        const reporterWithContext = reporter.withContext({ name });
-
-        try {
-          for (const [id, value] of extractFunction(content, reporterWithContext)) {
-            validateMessageNamespace(id, name, config.paths, reporterWithContext);
-            addMessageToMap(targetMap, id, value, reporterWithContext);
-          }
-        } catch (error) {
-          if (!isFailError(error)) {
-            throw error;
-          }
+  return [
+    [htmlEntries, extractHtmlMessages],
+    [codeEntries, extractCodeMessages],
+    [pugEntries, extractPugMessages],
+  ];
+}
 
-          reporterWithContext.report(error);
+export async function extractMessagesFromPathToMap(inputPath, targetMap, config, reporter) {
+  const categorizedEntries = await matchEntriesWithExctractors(inputPath);
+  return Promise.all(
+    categorizedEntries
+      .map(async ([entries, extractFunction]) => {
+        const files = await Promise.all(
+          filterEntries(entries, config.exclude).map(async entry => {
+            return {
+              name: entry,
+              content: await readFileAsync(entry),
+            };
+          })
+        );
+
+        for (const { name, content } of files) {
+          const reporterWithContext = reporter.withContext({ name });
+
+          try {
+            for (const [id, value] of extractFunction(content, reporterWithContext)) {
+              validateMessageNamespace(id, name, config.paths, reporterWithContext);
+              addMessageToMap(targetMap, id, value, reporterWithContext);
+            }
+          } catch (error) {
+            if (!isFailError(error)) {
+              throw error;
+            }
+
+            reporterWithContext.report(error);
+          }
         }
-      }
-    })
+      })
   );
 }

src/dev/i18n/index.ts

@@ -18,7 +18,11 @@
  */
 
 // @ts-ignore
-export { extractMessagesFromPathToMap } from './extract_default_translations';
+export {
+  extractMessagesFromPathToMap,
+  matchEntriesWithExctractors,
+} from './extract_default_translations';
+
 // @ts-ignore
 export { writeFileAsync, readFileAsync, normalizePath, ErrorReporter } from './utils';
 export { serializeToJson, serializeToJson5 } from './serializers';

src/dev/i18n/integrate_locale_files.ts

@@ -37,7 +37,7 @@ import { createFailError } from '../run';
 import { I18nConfig } from './config';
 import { serializeToJson } from './serializers';
 
-interface IntegrateOptions {
+export interface IntegrateOptions {
   sourceFileName: string;
   targetFileName?: string;
   dryRun: boolean;

src/dev/i18n/tasks/extract_default_translations.ts

@@ -18,19 +18,18 @@
  */
 
 import chalk from 'chalk';
-import Listr from 'listr';
-
 import { ErrorReporter, extractMessagesFromPathToMap, filterConfigPaths, I18nConfig } from '..';
 import { createFailError } from '../../run';
 
-export async function extractDefaultMessages({
+export function extractDefaultMessages({
   path,
   config,
 }: {
   path?: string | string[];
   config: I18nConfig;
 }) {
-  const filteredPaths = filterConfigPaths(Array.isArray(path) ? path : [path || './'], config);
+  const inputPaths = Array.isArray(path) ? path : [path || './'];
+  const filteredPaths = filterConfigPaths(inputPaths, config) as string[];
   if (filteredPaths.length === 0) {
     throw createFailError(
       `${chalk.white.bgRed(
@@ -39,36 +38,24 @@ export async function extractDefaultMessages({
     );
   }
 
-  const reporter = new ErrorReporter();
-
-  const list = new Listr(
-    filteredPaths.map(filteredPath => ({
-      task: async (messages: Map<string, unknown>) => {
-        const initialErrorsNumber = reporter.errors.length;
-
-        // Return result if no new errors were reported for this path.
-        const result = await extractMessagesFromPathToMap(filteredPath, messages, config, reporter);
-        if (reporter.errors.length === initialErrorsNumber) {
-          return result;
-        }
-
-        // Throw an empty error to make Listr mark the task as failed without any message.
-        throw new Error('');
-      },
-      title: filteredPath,
-    })),
-    {
-      exitOnError: false,
-    }
-  );
-
-  try {
-    return await list.run(new Map());
-  } catch (error) {
-    if (error.name === 'ListrError' && reporter.errors.length) {
-      throw createFailError(reporter.errors.join('\n\n'));
-    }
-
-    throw error;
-  }
+  return filteredPaths.map(filteredPath => ({
+    task: async ({
+      messages,
+      reporter,
+    }: {
+      messages: Map<string, unknown>;
+      reporter: ErrorReporter;
+    }) => {
+      const initialErrorsNumber = reporter.errors.length;
+
+      // Return result if no new errors were reported for this path.
+      const result = await extractMessagesFromPathToMap(filteredPath, messages, config, reporter);
+      if (reporter.errors.length === initialErrorsNumber) {
+        return result;
+      }
+
+      throw reporter;
+    },
+    title: filteredPath,
+  }));
 }

src/dev/i18n/tasks/extract_untracked_translations.ts

@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { I18nConfig, matchEntriesWithExctractors, normalizePath, readFileAsync } from '..';
+import { createFailError } from '../../run';
+import { ErrorReporter, TaskError } from '..';
+
+function filterEntries(entries: string[], exclude: string[]) {
+  return entries.filter((entry: string) =>
+    exclude.every((excludedPath: string) => !normalizePath(entry).startsWith(excludedPath))
+  );
+}
+
+export async function extractUntrackedMessagesTask({
+  path,
+  config,
+  reporter,
+}: {
+  path?: string | string[];
+  config: I18nConfig;
+  reporter: any;
+}) {
+  const inputPaths = Array.isArray(path) ? path : [path || './'];
+  const availablePaths = Object.values(config.paths);
+  const ignore = availablePaths.concat([
+    '**/build/**',
+    '**/webpackShims/**',
+    '**/__fixtures__/**',
+    '**/packages/kbn-i18n/**',
+    '**/packages/kbn-plugin-generator/sao_template/**',
+    '**/packages/kbn-ui-framework/generator-kui/**',
+    '**/target/**',
+    '**/test/**',
+    '**/scripts/**',
+    '**/src/dev/**',
+    '**/target/**',
+    '**/dist/**',
+  ]);
+  for (const inputPath of inputPaths) {
+    const categorizedEntries = await matchEntriesWithExctractors(inputPath, {
+      additionalIgnore: ignore,
+      mark: true,
+      absolute: true,
+    });
+
+    for (const [entries, extractFunction] of categorizedEntries) {
+      const files = await Promise.all(
+        filterEntries(entries, config.exclude)
+          .filter(entry => {
+            const normalizedEntry = normalizePath(entry);
+            return !availablePaths.some(
+              availablePath =>
+                normalizedEntry.startsWith(`${normalizePath(availablePath)}/`) ||
+                normalizePath(availablePath) === normalizedEntry
+            );
+          })
+          .map(async (entry: any) => ({
+            name: entry,
+            content: await readFileAsync(entry),
+          }))
+      );
+
+      for (const { name, content } of files) {
+        const reporterWithContext = reporter.withContext({ name });
+        for (const [id] of extractFunction(content, reporterWithContext)) {
+          const errorMessage = `Untracked file contains i18n label (${id}).`;
+          reporterWithContext.report(createFailError(errorMessage));
+        }
+      }
+    }
+  }
+}
+
+export function extractUntrackedMessages(srcPaths: string[], config: I18nConfig) {
+  return srcPaths.map(srcPath => ({
+    title: `Checking untracked messages in ${srcPath}`,
+    task: async ({ reporter }: { reporter: ErrorReporter }) => {
+      const initialErrorsNumber = reporter.errors.length;
+      const result = await extractUntrackedMessagesTask({ path: srcPath, config, reporter });
+      if (reporter.errors.length === initialErrorsNumber) {
+        return result;
+      }
+
+      throw reporter;
+    },
+  }));
+}

src/dev/i18n/tasks/index.ts

@@ -18,3 +18,4 @@
  */
 
 export { extractDefaultMessages } from './extract_default_translations';
+export { extractUntrackedMessages } from './extract_untracked_translations';