Skip to content

Commit

Permalink
Merge pull request #2551 from kuzzleio/feature/517-update-mapping-rei…
Browse files Browse the repository at this point in the history
…ndexCollection-flag

feat(elasticsearch): add flag to reindex collection after an update
  • Loading branch information
etrousset authored Oct 3, 2024
2 parents cde355a + ef744c0 commit eea5afe
Show file tree
Hide file tree
Showing 6 changed files with 115 additions and 25 deletions.
8 changes: 5 additions & 3 deletions doc/2/api/controllers/collection/update/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ Body:
}
}
},
"reindexCollection": true|false,
"settings": {
"analysis" : {
"analyzer":{
Expand Down Expand Up @@ -92,6 +93,7 @@ Body:
}
}
},
"reindexCollection": true|false,
"settings": {
"analysis" : {
"analyzer":{
Expand All @@ -117,8 +119,9 @@ Body:

## Body properties

* `settings`: Elasticsearch index [settings](https://www.elastic.co/guide/en/elasticsearch/reference/7.5/index-modules.html#index-modules-settings)
* `mappings`: [collection mappings](/core/2/guides/main-concepts/data-storage#mappings-properties)
- `settings`: Elasticsearch index [settings](https://www.elastic.co/guide/en/elasticsearch/reference/7.5/index-modules.html#index-modules-settings)
- `mappings`: [collection mappings](/core/2/guides/main-concepts/data-storage#mappings-properties)
- `reindexCollection`: boolean, if `true`, the collection will be reindexed after the update

---

Expand All @@ -143,4 +146,3 @@ Body:

- [Common errors](/core/2/api/errors/types#common-errors)
- [NotFoundError](/core/2/api/errors/types#notfounderror)

74 changes: 74 additions & 0 deletions jest/api/controller/collection/update.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { Kuzzle, WebSocket } from "kuzzle-sdk";

const kuzzle = new Kuzzle(new WebSocket("localhost"));
const index = "nyc-open-data";
const collection = "green-taxi";
const mappings = {
dynamic: "false" as const,
properties: {
name: {
type: "keyword",
},
},
};

beforeAll(async () => {
await kuzzle.connect();
if (await kuzzle.index.exists(index)) {
await kuzzle.index.delete(index);
}

await kuzzle.index.create(index);
await kuzzle.collection.create(index, collection, {
mappings,
});
});

afterAll(async () => {
await kuzzle.index.delete(index);
kuzzle.disconnect();
});

describe("collection:update", () => {
it("should reindex the collection if asked to", async () => {
await kuzzle.document.create(
index,
collection,
{ age: 42, name: "Bob" },
"document-1",
{ refresh: "wait_for" },
);

let result = await kuzzle.document.search(index, collection, {
query: {
range: {
age: {
gte: 40,
},
},
},
});

expect(result.hits.length).toEqual(0);

await kuzzle.collection.update(index, collection, {
mappings: { properties: { age: { type: "long" } } },
reindexCollection: true,
});

// Wait for the reindexing to complete
await new Promise((r) => setTimeout(r, 2000));

result = await kuzzle.document.search(index, collection, {
query: {
range: {
age: {
gte: 40,
},
},
},
});

expect(result.hits.length).toEqual(1);
});
});
26 changes: 17 additions & 9 deletions lib/service/storage/7/elasticsearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

import _ from "lodash";

import { ApiResponse, RequestParams, Client } from "sdk-es7";
import { ApiResponse, Client, RequestParams } from "sdk-es7";
import { Index, IndicesCreate } from "sdk-es7/api/requestParams";
import { TypeMapping } from "sdk-es7/api/types";
import {
Expand All @@ -39,16 +39,16 @@ import ms from "ms";
import semver from "semver";
import debug from "../../../util/debug";

import ESWrapper from "./esWrapper";
import { QueryTranslator } from "../commons/queryTranslator";
import didYouMean from "../../../util/didYouMean";
import { storeScopeEnum } from "../../../core/storage/storeScopeEnum";
import * as kerror from "../../../kerror";
import { assertIsObject } from "../../../util/requestAssertions";
import { isPlainObject } from "../../../util/safeObject";
import didYouMean from "../../../util/didYouMean";
import extractFields from "../../../util/extractFields";
import { Mutex } from "../../../util/mutex";
import { randomNumber } from "../../../util/name-generator";
import { storeScopeEnum } from "../../../core/storage/storeScopeEnum";
import { assertIsObject } from "../../../util/requestAssertions";
import { isPlainObject } from "../../../util/safeObject";
import { QueryTranslator } from "../commons/queryTranslator";
import ESWrapper from "./esWrapper";

debug("kuzzle:services:elasticsearch");

Expand Down Expand Up @@ -1659,8 +1659,13 @@ export class ES7 {
collection: string,
{
mappings = {},
reindexCollection = false,
settings = {},
}: { mappings?: TypeMapping; settings?: Record<string, any> } = {},
}: {
mappings?: TypeMapping;
reindexCollection?: boolean;
settings?: Record<string, any>;
} = {},
) {
const esRequest = {
index: await this._getIndice(index, collection),
Expand Down Expand Up @@ -1690,7 +1695,10 @@ export class ES7 {

await this.updateMapping(index, collection, mappings);

if (this._dynamicChanges(previousMappings, mappings)) {
if (
reindexCollection ||
this._dynamicChanges(previousMappings, mappings)
) {
await this.updateSearchIndex(index, collection);
}
}
Expand Down
21 changes: 13 additions & 8 deletions lib/service/storage/8/elasticsearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,21 +36,21 @@ import {

import assert from "assert";

import ms from "ms";
import Bluebird from "bluebird";
import ms from "ms";
import semver from "semver";

import { storeScopeEnum } from "../../../core/storage/storeScopeEnum";
import * as kerror from "../../../kerror";
import debug from "../../../util/debug";
import ESWrapper from "./esWrapper";
import { QueryTranslator } from "../commons/queryTranslator";
import didYouMean from "../../../util/didYouMean";
import * as kerror from "../../../kerror";
import { assertIsObject } from "../../../util/requestAssertions";
import { isPlainObject } from "../../../util/safeObject";
import { storeScopeEnum } from "../../../core/storage/storeScopeEnum";
import extractFields from "../../../util/extractFields";
import { Mutex } from "../../../util/mutex";
import { randomNumber } from "../../../util/name-generator";
import { assertIsObject } from "../../../util/requestAssertions";
import { isPlainObject } from "../../../util/safeObject";
import { QueryTranslator } from "../commons/queryTranslator";
import ESWrapper from "./esWrapper";

debug("kuzzle:services:elasticsearch");

Expand Down Expand Up @@ -1666,9 +1666,11 @@ export class ES8 {
collection: string,
{
mappings = {},
reindexCollection = false,
settings = {},
}: {
mappings?: estypes.MappingTypeMapping;
reindexCollection?: boolean;
settings?: Record<string, any>;
} = {},
) {
Expand Down Expand Up @@ -1700,7 +1702,10 @@ export class ES8 {

await this.updateMapping(index, collection, mappings);

if (this._dynamicChanges(previousMappings, mappings)) {
if (
reindexCollection ||
this._dynamicChanges(previousMappings, mappings)
) {
await this.updateSearchIndex(index, collection);
}
}
Expand Down
9 changes: 5 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"koncorde": "4.3.0",
"kuzzle-plugin-auth-passport-local": "6.4.1",
"kuzzle-plugin-logger": "3.0.3",
"kuzzle-sdk": "^7.11.3",
"kuzzle-sdk": "^7.13.0",
"kuzzle-vault": "2.0.4",
"lodash": "4.17.21",
"long": "5.2.3",
Expand Down

0 comments on commit eea5afe

Please sign in to comment.