Skip to content

Commit

Permalink
[i18n] extract untracked translations and prettier logging (#35171)
Browse files Browse the repository at this point in the history
* extract untracked translations and prettier logging

* self code review

* Update src/dev/run_i18n_check.ts

Co-Authored-By: Bamieh <ahmadbamieh@gmail.com>

* updating listR

* run new i18n_check and fix errors

* kbnEmbeddables -> embeddableApi

* remove any type

* Update src/legacy/core_plugins/embeddable_api/public/panel/panel_header/panel_actions/add_panel/add_panel_flyout.tsx

Co-Authored-By: Stacey Gammon <gammon@elastic.co>

* Update src/legacy/core_plugins/embeddable_api/public/panel/panel_header/panel_actions/add_panel/add_panel_action.tsx

Co-Authored-By: Stacey Gammon <gammon@elastic.co>

* self code review fixes

* fix extract and integrate scripts

* ts-ignore js file
  • Loading branch information
Bamieh authored Jun 17, 2019
1 parent 5b9c231 commit b36aad3
Show file tree
Hide file tree
Showing 26 changed files with 381 additions and 146 deletions.
2 changes: 2 additions & 0 deletions .i18nrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"interpreter": "src/legacy/core_plugins/interpreter",
"kbn": "src/legacy/core_plugins/kibana",
"kbnDocViews": "src/legacy/core_plugins/kbn_doc_views",
"embeddableApi": "src/legacy/core_plugins/embeddable_api",
"kbnVislibVisTypes": "src/legacy/core_plugins/kbn_vislib_vis_types",
"markdownVis": "src/legacy/core_plugins/markdown_vis",
"metricVis": "src/legacy/core_plugins/metric_vis",
Expand All @@ -25,6 +26,7 @@
"xpack.apm": "x-pack/plugins/apm",
"xpack.beatsManagement": "x-pack/plugins/beats_management",
"xpack.canvas": "x-pack/plugins/canvas",
"xpack.code": "x-pack/plugins/code",
"xpack.crossClusterReplication": "x-pack/plugins/cross_cluster_replication",
"xpack.dashboardMode": "x-pack/plugins/dashboard_mode",
"xpack.graph": "x-pack/plugins/graph",
Expand Down
79 changes: 47 additions & 32 deletions src/dev/i18n/extract_default_translations.js
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,20 @@ See .i18nrc.json for the list of supported namespaces.`)
}
}

export async function extractMessagesFromPathToMap(inputPath, targetMap, config, reporter) {
export async function matchEntriesWithExctractors(inputPath, options = {}) {
const {
additionalIgnore = [],
mark = false,
absolute = false,
} = options;
const ignore = ['**/node_modules/**', '**/__tests__/**', '**/*.test.{js,jsx,ts,tsx}', '**/*.d.ts'].concat(additionalIgnore);

const entries = await globAsync('*.{js,jsx,pug,ts,tsx,html}', {
cwd: inputPath,
matchBase: true,
ignore: ['**/node_modules/**', '**/__tests__/**', '**/*.test.{js,jsx,ts,tsx}', '**/*.d.ts'],
ignore,
mark,
absolute,
});

const { htmlEntries, codeEntries, pugEntries } = entries.reduce(
Expand All @@ -86,37 +95,43 @@ export async function extractMessagesFromPathToMap(inputPath, targetMap, config,
{ htmlEntries: [], codeEntries: [], pugEntries: [] }
);

await Promise.all(
[
[htmlEntries, extractHtmlMessages],
[codeEntries, extractCodeMessages],
[pugEntries, extractPugMessages],
].map(async ([entries, extractFunction]) => {
const files = await Promise.all(
filterEntries(entries, config.exclude).map(async entry => {
return {
name: entry,
content: await readFileAsync(entry),
};
})
);

for (const { name, content } of files) {
const reporterWithContext = reporter.withContext({ name });

try {
for (const [id, value] of extractFunction(content, reporterWithContext)) {
validateMessageNamespace(id, name, config.paths, reporterWithContext);
addMessageToMap(targetMap, id, value, reporterWithContext);
}
} catch (error) {
if (!isFailError(error)) {
throw error;
}
return [
[htmlEntries, extractHtmlMessages],
[codeEntries, extractCodeMessages],
[pugEntries, extractPugMessages],
];
}

reporterWithContext.report(error);
export async function extractMessagesFromPathToMap(inputPath, targetMap, config, reporter) {
const categorizedEntries = await matchEntriesWithExctractors(inputPath);
return Promise.all(
categorizedEntries
.map(async ([entries, extractFunction]) => {
const files = await Promise.all(
filterEntries(entries, config.exclude).map(async entry => {
return {
name: entry,
content: await readFileAsync(entry),
};
})
);

for (const { name, content } of files) {
const reporterWithContext = reporter.withContext({ name });

try {
for (const [id, value] of extractFunction(content, reporterWithContext)) {
validateMessageNamespace(id, name, config.paths, reporterWithContext);
addMessageToMap(targetMap, id, value, reporterWithContext);
}
} catch (error) {
if (!isFailError(error)) {
throw error;
}

reporterWithContext.report(error);
}
}
}
})
})
);
}
2 changes: 2 additions & 0 deletions src/dev/i18n/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
// @ts-ignore
export { extractMessagesFromPathToMap } from './extract_default_translations';
// @ts-ignore
export { matchEntriesWithExctractors } from './extract_default_translations';
// @ts-ignore
export { writeFileAsync, readFileAsync, normalizePath, ErrorReporter } from './utils';
export { serializeToJson, serializeToJson5 } from './serializers';
export { I18nConfig, filterConfigPaths, mergeConfigs } from './config';
Expand Down
2 changes: 1 addition & 1 deletion src/dev/i18n/integrate_locale_files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import { createFailError } from '../run';
import { I18nConfig } from './config';
import { serializeToJson } from './serializers';

interface IntegrateOptions {
export interface IntegrateOptions {
sourceFileName: string;
targetFileName?: string;
dryRun: boolean;
Expand Down
48 changes: 48 additions & 0 deletions src/dev/i18n/tasks/check_compatibility.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import { ToolingLog } from '@kbn/dev-utils';
import { integrateLocaleFiles, I18nConfig } from '..';

export interface I18nFlags {
fix: boolean;
ignoreIncompatible: boolean;
ignoreUnused: boolean;
ignoreMissing: boolean;
}

export function checkCompatibility(config: I18nConfig, flags: I18nFlags, log: ToolingLog) {
const { fix, ignoreIncompatible, ignoreUnused, ignoreMissing } = flags;
return config.translations.map(translationsPath => ({
task: async ({ messages }: { messages: Map<string, { message: string }> }) => {
// If `fix` is set we should try apply all possible fixes and override translations file.
await integrateLocaleFiles(messages, {
dryRun: !fix,
ignoreIncompatible: fix || ignoreIncompatible,
ignoreUnused: fix || ignoreUnused,
ignoreMissing: fix || ignoreMissing,
sourceFileName: translationsPath,
targetFileName: fix ? translationsPath : undefined,
config,
log,
});
},
title: `Compatibility check with ${translationsPath}`,
}));
}
57 changes: 21 additions & 36 deletions src/dev/i18n/tasks/extract_default_translations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,18 @@
*/

import chalk from 'chalk';
import Listr from 'listr';

import { ErrorReporter, extractMessagesFromPathToMap, filterConfigPaths, I18nConfig } from '..';
import { createFailError } from '../../run';

export async function extractDefaultMessages({
export function extractDefaultMessages({
path,
config,
}: {
path?: string | string[];
config: I18nConfig;
}) {
const filteredPaths = filterConfigPaths(Array.isArray(path) ? path : [path || './'], config);
const inputPaths = Array.isArray(path) ? path : [path || './'];
const filteredPaths = filterConfigPaths(inputPaths, config) as string[];
if (filteredPaths.length === 0) {
throw createFailError(
`${chalk.white.bgRed(
Expand All @@ -39,36 +38,22 @@ export async function extractDefaultMessages({
);
}

const reporter = new ErrorReporter();

const list = new Listr(
filteredPaths.map(filteredPath => ({
task: async (messages: Map<string, unknown>) => {
const initialErrorsNumber = reporter.errors.length;

// Return result if no new errors were reported for this path.
const result = await extractMessagesFromPathToMap(filteredPath, messages, config, reporter);
if (reporter.errors.length === initialErrorsNumber) {
return result;
}

// Throw an empty error to make Listr mark the task as failed without any message.
throw new Error('');
},
title: filteredPath,
})),
{
exitOnError: false,
}
);

try {
return await list.run(new Map());
} catch (error) {
if (error.name === 'ListrError' && reporter.errors.length) {
throw createFailError(reporter.errors.join('\n\n'));
}

throw error;
}
return filteredPaths.map(filteredPath => ({
task: async (context: {
messages: Map<string, { message: string }>;
reporter: ErrorReporter;
}) => {
const { messages, reporter } = context;
const initialErrorsNumber = reporter.errors.length;

// Return result if no new errors were reported for this path.
const result = await extractMessagesFromPathToMap(filteredPath, messages, config, reporter);
if (reporter.errors.length === initialErrorsNumber) {
return result;
}

throw reporter;
},
title: filteredPath,
}));
}
109 changes: 109 additions & 0 deletions src/dev/i18n/tasks/extract_untracked_translations.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import {
I18nConfig,
matchEntriesWithExctractors,
normalizePath,
readFileAsync,
ErrorReporter,
} from '..';
import { createFailError } from '../../run';

function filterEntries(entries: string[], exclude: string[]) {
return entries.filter((entry: string) =>
exclude.every((excludedPath: string) => !normalizePath(entry).startsWith(excludedPath))
);
}

export async function extractUntrackedMessagesTask({
path,
config,
reporter,
}: {
path?: string | string[];
config: I18nConfig;
reporter: any;
}) {
const inputPaths = Array.isArray(path) ? path : [path || './'];
const availablePaths = Object.values(config.paths);
const ignore = availablePaths.concat([
'**/build/**',
'**/webpackShims/**',
'**/__fixtures__/**',
'**/packages/kbn-i18n/**',
'**/packages/kbn-plugin-generator/sao_template/**',
'**/packages/kbn-ui-framework/generator-kui/**',
'**/target/**',
'**/test/**',
'**/scripts/**',
'**/src/dev/**',
'**/target/**',
'**/dist/**',
]);
for (const inputPath of inputPaths) {
const categorizedEntries = await matchEntriesWithExctractors(inputPath, {
additionalIgnore: ignore,
mark: true,
absolute: true,
});

for (const [entries, extractFunction] of categorizedEntries) {
const files = await Promise.all(
filterEntries(entries, config.exclude)
.filter(entry => {
const normalizedEntry = normalizePath(entry);
return !availablePaths.some(
availablePath =>
normalizedEntry.startsWith(`${normalizePath(availablePath)}/`) ||
normalizePath(availablePath) === normalizedEntry
);
})
.map(async (entry: any) => ({
name: entry,
content: await readFileAsync(entry),
}))
);

for (const { name, content } of files) {
const reporterWithContext = reporter.withContext({ name });
for (const [id] of extractFunction(content, reporterWithContext)) {
const errorMessage = `Untracked file contains i18n label (${id}).`;
reporterWithContext.report(createFailError(errorMessage));
}
}
}
}
}

export function extractUntrackedMessages(srcPaths: string[], config: I18nConfig) {
return srcPaths.map(srcPath => ({
title: `Checking untracked messages in ${srcPath}`,
task: async (context: { reporter: ErrorReporter }) => {
const { reporter } = context;
const initialErrorsNumber = reporter.errors.length;
const result = await extractUntrackedMessagesTask({ path: srcPath, config, reporter });
if (reporter.errors.length === initialErrorsNumber) {
return result;
}

throw reporter;
},
}));
}
2 changes: 2 additions & 0 deletions src/dev/i18n/tasks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,5 @@
*/

export { extractDefaultMessages } from './extract_default_translations';
export { extractUntrackedMessages } from './extract_untracked_translations';
export { checkCompatibility } from './check_compatibility';
Loading

0 comments on commit b36aad3

Please sign in to comment.