Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OPSEXP-1369 Add elasticsearch connector #693

Merged
merged 44 commits into from
May 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
1e997aa
first stub
gionn Apr 13, 2022
d9e915d
pod running
gionn Apr 13, 2022
7765bc6
elastic
gionn Apr 14, 2022
065d56a
enable elasticsearch in repo
gionn Apr 14, 2022
be0bbc3
fixme
gionn Apr 14, 2022
8fc0f5b
disable sync
gionn Apr 14, 2022
2840423
fix license and solr shared secret
gionn Apr 14, 2022
7c76fbe
fix securecomm
gionn Apr 15, 2022
a505b1f
cleanup
gionn Apr 15, 2022
95f24f3
fix pre-commit checks
gionn Apr 15, 2022
0c38fc6
fixup
gionn Apr 15, 2022
2cff507
globals for database secret
gionn Apr 15, 2022
2dd30c1
less hardcoded url
gionn Apr 15, 2022
b281777
remove unnecessary ingress and service
gionn Apr 15, 2022
61e49b3
revert
gionn Apr 15, 2022
957f26e
cleanup
gionn Apr 15, 2022
8d7ae33
simplify
gionn Apr 15, 2022
9ea1009
cleanup values
gionn Apr 15, 2022
f9b1bb2
fix helm-docs
gionn Apr 19, 2022
f86109e
cleanup
gionn Apr 19, 2022
32381d5
invert verifies suites and fail fast
gionn Apr 20, 2022
024301f
really exit when pod not ready
gionn Apr 20, 2022
ca801ce
print events of failed pods
gionn Apr 20, 2022
637be07
limit pod logs to 1024
gionn Apr 20, 2022
d71ad82
fix deployment secret name
gionn Apr 20, 2022
c9ec98c
retry newman run before giving up
gionn Apr 20, 2022
aea2309
faster namespace deletion during tests
gionn Apr 20, 2022
07088f5
test persistence failover only for latest suite
gionn Apr 20, 2022
86504b1
make sure newman return error when retries expire
gionn Apr 20, 2022
9dfbe3f
use upstream checkov hook and test elasticsearch connector
gionn Apr 20, 2022
ff050ba
fixup graceperiod
gionn Apr 20, 2022
390adf6
reuse content-services helper and avoid globals
gionn Apr 21, 2022
6fcff23
enforce secret for solr by default
gionn Apr 21, 2022
12a2f02
variable for elasticsearch host
gionn Apr 21, 2022
b18fcb3
reuse template helper for content and filestore
gionn Apr 21, 2022
4380f23
inject postgresql details from parent
gionn Apr 21, 2022
50b3584
minor move in chart dependencies
gionn Apr 22, 2022
aac4a19
refactor in a common library chart
gionn Apr 22, 2022
7639e9d
continue building the matrix to avoid rerun every suite
gionn Apr 22, 2022
f3be84f
early exit for persistence failover
gionn Apr 22, 2022
cf41b9f
expose support for full postgres url
gionn Apr 22, 2022
d7c510a
rename subchart to alfresco-content-common
gionn Apr 26, 2022
cd481e2
unbox image values for each service
gionn Apr 26, 2022
312f297
Merge branch 'master' into OPSEXP-1369-elasticsearch-connector
gionn Apr 27, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions .github/scripts/helm_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,12 @@ pod_status() {

# failed pods logs
failed_pod_logs() {
pod_status | grep False | awk '{print $1}' | while read pod; do echo -e '\e[1;31m' "${pod}" '\e[0m' && kubectl logs "${pod}" --namespace "${namespace}"; done
pod_status | grep False | awk '{print $1}' | \
while read pod; do
echo -e '\e[1;31m' "${pod}" '\e[0m' && \
kubectl get event --namespace "${namespace}" --field-selector involvedObject.name="${pod}"
kubectl logs "${pod}" --namespace "${namespace}" --tail 1024
done
}

wait_for_connection() {
Expand Down Expand Up @@ -81,20 +86,25 @@ pods_ready() {

if [ "${PODS_COUNTER}" -ge "${PODS_COUNTER_MAX}" ]; then
pod_status
echo "Pods did not start - exit 1"
echo "Pods did not start - failing build"
failed_pod_logs
if [[ "${COMMIT_MESSAGE}" != *"[keep env]"* ]]; then
helm delete "${release_name_ingress}" "${release_name_acs}" -n "${namespace}"
kubectl delete namespace "${namespace}"
kubectl delete namespace "${namespace}" --grace-period=1
fi
exit 1
return 1
fi
}

newman() {
# shellcheck disable=SC2048
# shellcheck disable=SC2086
docker run -t -v "${PWD}/test/postman:/etc/newman" postman/newman:5.3 $*
for i in {1..5}; do
docker run -t -v "${PWD}/test/postman:/etc/newman" postman/newman:5.3 $* && return 0
echo "newman run failed, trying again ($i run)"
sleep 10
done
return 1
}
prepare_namespace() {
cat <<EOF | kubectl apply -f -
Expand Down Expand Up @@ -232,7 +242,7 @@ done

[ "${DNS_PROPAGATED}" -ne 1 ] && echo "DNS entry for ${HOST} did not propagate within expected time" && exit 1

pods_ready
pods_ready || exit 1

# Delay running the tests to give ingress & SOLR a chance to fully initialise
echo "Waiting 3 minutes from $(date) before running tests..."
Expand All @@ -253,7 +263,7 @@ if [[ "${TEST_RESULT}" == "0" ]]; then
echo "TEST_RESULT=${TEST_RESULT}"
fi

if [[ "${TEST_RESULT}" == "0" ]]; then
if [[ "${TEST_RESULT}" == "0" ]] && [[ ${ACS_VERSION} == "latest" ]]; then
# For checking if persistence failover is correctly working with our deployments
# in the next phase we delete the acs and postgresql pods,
# wait for k8s to recreate them, then check if the data created in the first test run is still there
Expand All @@ -266,7 +276,7 @@ if [[ "${TEST_RESULT}" == "0" ]]; then
--namespace="${namespace}"

# check pods
pods_ready
pods_ready || exit 1

# run checks after pod deletion
wait_for_connection
Expand All @@ -277,7 +287,7 @@ if [[ "${TEST_RESULT}" == "0" ]]; then
fi
if [[ "${COMMIT_MESSAGE}" != *"[keep env]"* ]]; then
helm delete "${release_name_ingress}" "${release_name_acs}" -n "${namespace}"
kubectl delete namespace "${namespace}"
kubectl delete namespace "${namespace}" --grace-period=1
fi

if [[ "${TEST_RESULT}" == "1" ]]; then
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ jobs:
helm:
needs: [ pre-commit ]
strategy:
max-parallel: 1 #cant add more parallel, problem with loadbalancers
max-parallel: 1 # otherwise flaky problems with aws ELB
fail-fast: false
matrix:
acs_version: [ 6.2.N, 7.0.N, 7.1.N, community, latest ]
acs_version: [ latest, 7.1.N, 7.0.N, 6.2.N, community ]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what is the reason for changing the order here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

since they are executed sequentially and now latest is the only one executing the persistence failover test (that I've disabled on the remaining jobs since it's slowing all the builds without really any added value), it's better to build latest first that is more comprehensive

runs-on: ubuntu-latest
env:
DOMAIN: dev.envalfresco.com
Expand Down Expand Up @@ -60,9 +60,9 @@ jobs:
compose:
needs: [ pre-commit ]
strategy:
fail-fast: false
fail-fast: true
matrix:
acs_version: [ 6.2.N, 7.0.N, 7.1.N, community, latest ]
acs_version: [ latest, 7.1.N, 7.0.N, 6.2.N, community ]
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand Down
2 changes: 2 additions & 0 deletions helm/alfresco-content-services/.helmignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
10 changes: 10 additions & 0 deletions helm/alfresco-content-services/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ home: https://www.alfresco.com
sources:
- https://github.com/Alfresco/acs-deployment
dependencies:
- name: alfresco-content-common
version: 0.1.0
- name: postgresql
version: 10.16.2
repository: https://charts.bitnami.com/bitnami
Expand All @@ -34,6 +36,14 @@ dependencies:
- name: alfresco-search
version: 1.0.4
condition: alfresco-search.enabled
- name: alfresco-elasticsearch-connector
version: 0.1.0
condition: alfresco-elasticsearch-connector.enabled
- name: alfresco-sync-service
version: 3.0.9
- name: elasticsearch
alias: elasticsearch
repository: https://helm.elastic.co
version: 7.10.1
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jesty check, is there a newer tested version?

condition: elasticsearch.enabled
icon: https://avatars0.githubusercontent.com/u/391127?s=200&v=4
16 changes: 15 additions & 1 deletion helm/alfresco-content-services/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@ A Helm chart for deploying Alfresco Content Services
| Repository | Name | Version |
|------------|------|---------|
| | activemq | 2.1.0 |
| | alfresco-content-common | 0.1.0 |
| | alfresco-elasticsearch-connector | 0.1.0 |
| | alfresco-search | 1.0.4 |
| | alfresco-sync-service | 3.0.9 |
| https://activiti.github.io/activiti-cloud-helm-charts | alfresco-digital-workspace(common) | 7.2.0 |
| https://charts.bitnami.com/bitnami | postgresql | 10.16.2 |
| https://charts.bitnami.com/bitnami | postgresql-syncservice(postgresql) | 10.16.2 |
| https://charts.bitnami.com/bitnami | postgresql | 10.16.2 |
| https://helm.elastic.co | elasticsearch(elasticsearch) | 7.10.1 |

## Values

Expand Down Expand Up @@ -76,6 +79,13 @@ Hence, setting up explicit Container memory and then assigning a percentage of i
| alfresco-digital-workspace.resources.requests.cpu | string | `"150m"` | |
| alfresco-digital-workspace.resources.requests.memory | string | `"256Mi"` | |
| alfresco-digital-workspace.service.envType | string | `"frontend"` | |
| alfresco-elasticsearch-connector.elasticsearch.host | string | `"elasticsearch-master"` | |
| alfresco-elasticsearch-connector.elasticsearch.port | int | `9200` | |
| alfresco-elasticsearch-connector.enabled | bool | `false` | |
| alfresco-elasticsearch-connector.reindexing.enabled | bool | `true` | |
| alfresco-elasticsearch-connector.reindexing.postgresql.database | string | `"alfresco"` | |
| alfresco-elasticsearch-connector.reindexing.postgresql.hostname | string | `"postgresql-acs"` | |
| alfresco-elasticsearch-connector.reindexing.postgresql.url | string | `nil` | |
| alfresco-search.alfresco-insight-zeppelin.insightzeppelin.enabled | bool | `false` | |
| alfresco-search.alfresco-insight-zeppelin.repository.host | string | `"alfresco-cs"` | |
| alfresco-search.alfresco-insight-zeppelin.repository.port | int | `80` | |
Expand All @@ -99,6 +109,10 @@ Hence, setting up explicit Container memory and then assigning a percentage of i
| database.password | string | `nil` | ex: alfresco |
| database.url | string | `nil` | ex: jdbc:postgresql://oldfashioned-mule-postgresql-acs:5432/alfresco |
| database.user | string | `nil` | ex: alfresco |
| elasticsearch.clusterHealthCheckParams | string | `"wait_for_status=yellow&timeout=1s"` | |
| elasticsearch.enabled | bool | `false` | |
| elasticsearch.image | string | `"docker.elastic.co/elasticsearch/elasticsearch-oss"` | |
| elasticsearch.replicas | int | `1` | |
| email | object | `{"handler":{"folder":{"overwriteDuplicates":true}},"inbound":{"emailContributorsAuthority":"EMAIL_CONTRIBUTORS","enabled":false,"unknownUser":"anonymous"},"initContainers":{"pemToKeystore":{"image":{"pullPolicy":"IfNotPresent","repository":"registry.access.redhat.com/redhat-sso-7/sso71-openshift","tag":"1.1-16"}},"pemToTruststore":{"image":{"pullPolicy":"IfNotPresent","repository":"registry.access.redhat.com/redhat-sso-7/sso71-openshift","tag":"1.1-16"}},"setPerms":{"image":{"pullPolicy":"IfNotPresent","repository":"busybox","tag":"1.35.0"}}},"server":{"allowed":{"senders":".*"},"auth":{"enabled":true},"blocked":{"senders":null},"connections":{"max":3},"domain":null,"enableTLS":true,"enabled":false,"hideTLS":false,"port":1125,"requireTLS":false},"ssl":{"secretName":null}}` | For a full information of properties on the email configuration, please view: https://docs.alfresco.com/6.2/concepts/email.html |
| email.server.enabled | bool | `false` | Enables the email server - see https://docs.alfresco.com/6.2/concepts/email-inboundsmtp-props.html |
| filestore | object | `{"environment":{"JAVA_OPTS":" -XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=80","scheduler.cleanup.interval":"86400000","scheduler.content.age.millis":"86400000"},"image":{"internalPort":8099,"pullPolicy":"IfNotPresent","repository":"quay.io/alfresco/alfresco-shared-file-store","tag":"0.16.1"},"initContainer":{"image":{"pullPolicy":"IfNotPresent","repository":"busybox","tag":"1.35.0"},"resources":{"limits":{"memory":"10Mi"},"requests":{"memory":"5Mi"}}},"livenessProbe":{"initialDelaySeconds":10,"livenessPercent":150,"livenessSavePeriodSeconds":600,"periodSeconds":20,"timeoutSeconds":10},"nodeSelector":{},"readinessProbe":{"initialDelaySeconds":20,"periodSeconds":60,"timeoutSeconds":10},"replicaCount":1,"resources":{"limits":{"memory":"1000Mi"},"requests":{"memory":"1000Mi"}},"service":{"externalPort":80,"name":"filestore","type":"ClusterIP"}}` | Declares the alfresco-shared-file-store used by the content repository and transform service |
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
apiVersion: v2
name: alfresco-content-common
description: A helper subchart to avoid duplication in alfresco-content-services chart and subcharts
type: library
version: 0.1.0
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{{- define "content-services.shortname" -}}
{{- $name := (.Values.NameOverride | default (printf "%s" "alfresco-cs")) -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{- define "alfresco.shortname" -}}
{{- $name := (.Values.NameOverride | default (printf "%s" "alfresco-")) -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apiVersion: v2
name: alfresco-elasticsearch-connector
description: A Helm chart for deploying Alfresco Elasticsearch connector
type: application
version: 0.1.0
appVersion: "3.0.0"
dependencies:
- name: alfresco-content-common
version: 0.1.0
repository: "file://../alfresco-content-common"
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# alfresco-elasticsearch-connector

![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square)

A Helm chart for deploying Alfresco Elasticsearch connector

## Requirements

| Repository | Name | Version |
|------------|------|---------|
| file://../alfresco-content-common | alfresco-content-common | 0.1.0 |

## Values

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| contentMediaTypeCache.enabled | bool | `true` | |
| contentMediaTypeCache.refreshTime | string | `"0 0 * * * *"` | |
| elasticsearch.host | string | `"elasticsearch-master"` | |
| elasticsearch.port | int | `9200` | |
| fullnameOverride | string | `""` | |
| global.alfrescoRegistryPullSecrets | string | `"quay-registry-secret"` | |
| imagePullSecrets | list | `[]` | |
| indexName | string | `"alfresco"` | |
| liveIndexing.content.image.pullPolicy | string | `"IfNotPresent"` | |
| liveIndexing.content.image.repository | string | `"quay.io/alfresco/alfresco-elasticsearch-live-indexing-content"` | |
| liveIndexing.content.image.tag | string | `"3.0.0"` | |
| liveIndexing.content.replicaCount | int | `1` | |
| liveIndexing.mediation.image.pullPolicy | string | `"IfNotPresent"` | |
| liveIndexing.mediation.image.repository | string | `"quay.io/alfresco/alfresco-elasticsearch-live-indexing-mediation"` | |
| liveIndexing.mediation.image.tag | string | `"3.0.0"` | |
| liveIndexing.metadata.image.pullPolicy | string | `"IfNotPresent"` | |
| liveIndexing.metadata.image.repository | string | `"quay.io/alfresco/alfresco-elasticsearch-live-indexing-metadata"` | |
| liveIndexing.metadata.image.tag | string | `"3.0.0"` | |
| liveIndexing.metadata.replicaCount | int | `1` | |
| liveIndexing.path.image.pullPolicy | string | `"IfNotPresent"` | |
| liveIndexing.path.image.repository | string | `"quay.io/alfresco/alfresco-elasticsearch-live-indexing-path"` | |
| liveIndexing.path.image.tag | string | `"3.0.0"` | |
| liveIndexing.path.replicaCount | int | `1` | |
| nameOverride | string | `""` | |
| nodeSelector | object | `{}` | |
| parentNameOverride | string | `""` | |
| pathIndexingComponent.enabled | bool | `true` | |
| podAnnotations | object | `{}` | |
| podSecurityContext | object | `{}` | |
| reindexing.enabled | bool | `true` | |
| reindexing.image.pullPolicy | string | `"IfNotPresent"` | |
| reindexing.image.repository | string | `"quay.io/alfresco/alfresco-elasticsearch-reindexing"` | |
| reindexing.image.tag | string | `"3.0.0"` | |
| reindexing.pathIndexingEnabled | bool | `true` | |
| reindexing.postgresql.database | string | `"alfresco"` | |
| reindexing.postgresql.hostname | string | `"postgresql-acs"` | |
| reindexing.postgresql.port | int | `5432` | |
| reindexing.postgresql.url | string | `nil` | |
| resources.limits.memory | string | `"2048Mi"` | |
| resources.requests.memory | string | `"256Mi"` | |
| securityContext | object | `{}` | |
| tolerations | list | `[]` | |

----------------------------------------------
Autogenerated from chart metadata using [helm-docs v1.8.1](https://github.com/norwoodj/helm-docs/releases/v1.8.1)
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "alfresco-elasticsearch-connector.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "alfresco-elasticsearch-connector.fullName" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "alfresco-elasticsearch-connector.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}

{{/*
Common labels
*/}}
{{- define "alfresco-elasticsearch-connector.labels" -}}
helm.sh/chart: {{ include "alfresco-elasticsearch-connector.chart" . }}
{{ include "alfresco-elasticsearch-connector.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}

{{/*
Selector labels
*/}}
{{- define "alfresco-elasticsearch-connector.selectorLabels" -}}
app.kubernetes.io/name: {{ include "alfresco-elasticsearch-connector.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}

{{/*
Create the name of the service account to use
*/}}
{{- define "alfresco-elasticsearch-connector.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "alfresco-elasticsearch-connector.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "alfresco-elasticsearch-connector.fullName" . }}-configmap
labels:
heritage: {{ .Release.Service }}
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
component: search
data:
ELASTICSEARCH_INDEXNAME: "{{ .Values.indexName }}"
SPRING_ELASTICSEARCH_REST_URIS: http://{{ .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }}
SPRING_ACTIVEMQ_BROKERURL: nio://{{ .Release.Name }}-activemq-broker:61616
ALFRESCO_SHAREDFILESTORE_BASEURL: http://{{ template "alfresco.shortname" . }}-filestore:80/alfresco/api/-default-/private/sfs/versions/1/file/
ALFRESCO_ACCEPTEDCONTENTMEDIATYPESCACHE_BASEURL: http://{{ template "content-services.shortname" . }}-transform-misc/transform/config
ALFRESCO_ACCEPTEDCONTENTMEDIATYPESCACHE_REFRESHTIME: "{{ .Values.contentMediaTypeCache.refreshTime }}"
ALFRESCO_ACCEPTEDCONTENTMEDIATYPESCACHE_ENABLED: "{{ .Values.contentMediaTypeCache.enabled }}"
ALFRESCO_PATHINDEXINGCOMPONENT.ENABLED: "{{ .Values.pathIndexingComponent.enabled }}"
{{- if .Values.liveIndexing.environment }}
{{- range $key, $val := .Values.liveIndexing.environment }}
{{ $key }}: {{ $val | quote }}
{{- end }}
{{- end }}
Loading