-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yaml
153 lines (140 loc) · 3.84 KB
/
docker-compose.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
version: "3.8"
services:
# Architecture
zookeeper:
image: confluentinc/cp-zookeeper:7.5.3
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
healthcheck:
test: [ "CMD", "bash", "-c", "echo 'ruok' | nc localhost 2181" ]
interval: 10s
timeout: 5s
retries: 5
broker:
image: confluentinc/cp-kafka:7.5.3
hostname: broker
container_name: broker
depends_on:
zookeeper:
condition: service_healthy
ports:
- "9092:9092"
- "9101:9101"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_JMX_PORT: 9101
KAFKA_JMX_HOSTNAME: localhost
healthcheck:
test: [ "CMD", "bash", "-c", 'nc -z localhost 9092' ]
interval: 10s
timeout: 5s
retries: 5
postgres:
image: postgres:latest
container_name: postgres
ports:
- "5432:5432"
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
healthcheck:
test: [ "CMD", "pg_isready", "-U", "postgres" ]
interval: 10s
timeout: 5s
retries: 5
elasticsearch:
image: elasticsearch:7.17.18
container_name: elasticsearch
environment:
xpack.security.enabled: false
discovery.type: single-node
ports:
- "9200:9200"
volumes:
- ./kibana-dashboards:/usr/share/elasticsearch/data
kibana:
image: kibana:7.17.18
container_name: kibana
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
depends_on:
- elasticsearch
ports:
- "5601:5601"
# Flink
jobmanager:
image: flink:1.17-java11
depends_on:
broker:
condition: service_healthy
kibana:
condition: service_started
jobbuilder:
condition: service_completed_successfully
historicdata:
condition: service_started
container_name: jobmanager
ports:
- "8081:8081"
command: standalone-job --job-classname FlinkStreaming.DataStreamJob /opt/flink/usrlib/flink-streaming-1.0-SNAPSHOT.jar --target broker:9092
volumes:
- ./flink-streaming/target:/opt/flink/usrlib
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
parallelism.default: 2
restart: always
taskmanager:
image: flink:1.17-java11
depends_on:
- jobmanager
container_name: taskmanager
command: taskmanager
volumes:
- ./flink-streaming/target:/opt/flink/usrlib
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
taskmanager.numberOfTaskSlots: 4
jobbuilder:
image: maven:3.8.4-openjdk-17-slim
container_name: mavenbuild
command: mvn clean package
volumes:
- ./flink-streaming:/opt/flink/project
working_dir: /opt/flink/project
historicdata:
build:
context: ./kafka-streaming
dockerfile: Dockerfile_hd
depends_on:
broker:
condition: service_healthy
container_name: historicdata
realtimedata:
build:
context: ./kafka-streaming
dockerfile: Dockerfile_rtd
depends_on:
broker:
condition: service_healthy
historicdata:
condition: service_completed_successfully
container_name: realtimedata