Orion v2 docker setup

.env

@@ -1,10 +1,11 @@
 COMPOSE_PROJECT_NAME=joystream
 PROJECT_NAME=query_node
 
-# We will use a single postgres service with multiple databases
-# The env variables below are by default used by all services and should be
-# overriden in local env files
-# DB config
+# We use a single postgres service for both the query node indexer and processor.
+# The default `DB_*` environment variables point to query node processor's database.
+# The DB_NAME env is overriden with INDEXER_DB_NAME in the indexer's service docker-compose config.
+# When running other services, those values should be overriden by their local environment configs.
+# Query node DB config:
 INDEXER_DB_NAME=query_node_indexer
 DB_NAME=query_node_processor
 DB_USER=postgres
@@ -90,6 +91,38 @@ HCAPTCHA_SECRET=
 BALANCE_CREDIT=300000000000
 BALANCE_LOCKED=300000000000
 
+# ====================================== ORION ======================================
+# Orion db config
+# Overrides `DB_*` environment variables for orion services in docker-compose.yml
+# We use a single postgres service for both:
+# - Subsquid archive's database (squid-archive, squid-archive-gateway) and
+# - Orion's database (orion_processor, orion_graphql-server).
+# ORION_DB_NAME is the name of the Orion's database, while SQUID_ARCHIVE_DB_NAME
+# is the name of the Subsquid archive database.
+SQUID_ARCHIVE_DB_NAME=squid_archive
+ORION_DB_NAME=orion
+ORION_DB_USER=postgres
+ORION_DB_PASS=postgres
+ORION_DB_PORT=5433
+
+# Orion / Subsquid archive service ports
+ORION_API_PORT=4350
+PROCESSOR_PROMETHEUS_PORT=3337
+SQUID_ARCHIVE_GATEWAY_PORT=8000
+
+# Other Orion configs
+SUPPORT_NO_CATEGORY_VIDEOS=true
+SUPPORT_NEW_CATEGORIES=true
+KILL_SWITCH_ON=false
+VIDEO_VIEW_PER_IP_TIME_LIMIT=30
+VIDEO_RELEVANCE_VIEWS_TICK=50
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5]"
+OPERATOR_SECRET=this-is-not-so-secret-change-it
+MAX_CACHED_ENTITIES=1000
+APP_PRIVATE_KEY=this-is-not-so-secret-change-it
+TRUSTED_REVERSE_PROXIES=1
+# =====================================================================================
+
 # joystream/node docker image tag
 # We do not provide a default value - scripts that startup a joystream-node service
 # Should be explicit about what version to use.

devops/ansible/roles/rpc/templates/Caddyfile.j2

@@ -40,7 +40,7 @@ wss://{{ hostname }}/query-node/server* {
 
 {{ hostname }}/orion* {
     uri strip_prefix /orion
-    reverse_proxy localhost:6116
+    reverse_proxy localhost:4350
 }
 
 {{ hostname }}/member-faucet* {

devops/ansible/templates/Playground-Caddyfile.j2

@@ -50,7 +50,12 @@ wss://{{ nip_domain }}/query-node/server* {
 
 {{ nip_domain }}/orion* {
     uri strip_prefix /orion
-    reverse_proxy localhost:6116
+    reverse_proxy localhost:4350
+}
+
+{{ nip_domain }}/archive* {
+    uri strip_prefix /archive
+    reverse_proxy localhost:8000
 }
 
 {{ nip_domain }}/member-faucet* {

docker-compose.yml

@@ -163,6 +163,23 @@ services:
       POSTGRES_DB: ${INDEXER_DB_NAME}
       PGPORT: ${DB_PORT}
 
+  orion-db:
+    image: postgres:14
+    container_name: orion-db
+    restart: unless-stopped
+    ports:
+      - '127.0.0.1:${ORION_DB_PORT}:${ORION_DB_PORT}'
+    volumes:
+      - orion-db-data:/var/lib/postgresql/data
+    env_file:
+      # relative to working directory where docker-compose was run from
+      - .env
+    environment:
+      POSTGRES_USER: ${ORION_DB_USER}
+      POSTGRES_PASSWORD: ${ORION_DB_PASS}
+      POSTGRES_DB: ${ORION_DB_NAME}
+      PGPORT: ${ORION_DB_PORT}
+
   graphql-server:
     image: node:14
     container_name: graphql-server
@@ -281,28 +298,92 @@ services:
     ports:
       - "3002:3002"
 
-  orion:
-    container_name: orion
-    image: joystream/orion
+  squid-archive:
+    container_name: squid-archive
+    image: subsquid/substrate-ingest:firesquid
+    depends_on:
+      - orion-db
+    restart: unless-stopped
+    env_file:
+      - .env
+    environment:
+      - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME}
+      - DB_USER=${ORION_DB_USER}
+      - DB_PASS=${ORION_DB_PASS}
+      - DB_HOST=orion-db
+      - DB_PORT=${ORION_DB_PORT}
+    command: [
+        '-e',
+        '${JOYSTREAM_NODE_WS}',
+        '-c',
+        '20',
+        '--out',
+        'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}',
+      ]
+
+  squid-archive-gateway:
+    container_name: squid-archive-gateway
+    image: subsquid/substrate-gateway:firesquid
+    depends_on:
+      - orion-db
+    restart: unless-stopped
+    env_file:
+      - .env
     environment:
-      - ORION_PORT=6116
-      - ORION_MONGO_HOSTNAME=mongo
-      - ORION_FEATURED_CONTENT_SECRET=password123
-      - ORION_ADMIN_SECRET=password123
-      - ORION_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
-      - APP_PRIVATE_KEY=secret123
+      - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME}
+      - DB_USER=${ORION_DB_USER}
+      - DB_PASS=${ORION_DB_PASS}
+      - DB_HOST=orion-db
+      - DB_PORT=${ORION_DB_PORT}
+      - DATABASE_MAX_CONNECTIONS=5
+      - RUST_LOG=actix_web=info,actix_server=info
+    command: [
+        '--database-url',
+        'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}',
+      ]
     ports:
-      - "6116:6116"
+      - '127.0.0.1:${SQUID_ARCHIVE_GATEWAY_PORT}:8000'
+
+  orion-processor:
+    container_name: orion-processor
+    image: joystream/orion:latest
+    restart: unless-stopped
+    env_file:
+      - .env
     depends_on:
-      - mongo
-    restart: always
+      - orion-db
+    environment:
+      - DB_NAME=${ORION_DB_NAME}
+      - DB_USER=${ORION_DB_USER}
+      - DB_PASS=${ORION_DB_PASS}
+      - DB_HOST=orion-db
+      - DB_PORT=${ORION_DB_PORT}
+      - GQL_PORT=${ORION_API_PORT}
+      - ARCHIVE_GATEWAY_URL=http://squid-archive-gateway:8000/graphql
+    command: >
+      sh -c "npx squid-typeorm-migration apply && \
+             node lib/processor.js"
+    ports:
+      - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}'
 
-  mongo:
-    restart: always
-    container_name: mongo
-    image: library/mongo:4.4
-    volumes:
-      - orion-mongo-data:/data/db
+  orion-api:
+    container_name: orion-api
+    image: joystream/orion:latest
+    restart: unless-stopped
+    env_file:
+      - .env
+    depends_on:
+      - orion-db
+    environment:
+      - DB_NAME=${ORION_DB_NAME}
+      - DB_USER=${ORION_DB_USER}
+      - DB_PASS=${ORION_DB_PASS}
+      - DB_HOST=orion-db
+      - DB_PORT=${ORION_DB_PORT}
+      - GQL_PORT=${ORION_API_PORT}
+    command: ["npm", "run", "query-node-start"]
+    ports:
+      - '${ORION_API_PORT}:${ORION_API_PORT}'
 
 volumes:
   chain-data:
@@ -333,5 +414,5 @@ volumes:
     driver: local
   distributor-2-data:
     driver: local
-  orion-mongo-data:
+  orion-db-data:
     driver: local

start-multistorage.sh

@@ -29,7 +29,7 @@ fi
 ./query-node/start.sh
 
 ## Orion
-docker-compose up -d orion
+./start-orion.sh
 
 ## Init the chain with some state
 if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then

start-orion.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -e
+
+docker-compose up -d orion-db
+docker-compose up -d squid-archive
+docker-compose up -d squid-archive-gateway
+docker-compose up -d orion-processor
+docker-compose up -d orion-api

start-qn-orion-faucet.sh

@@ -5,7 +5,7 @@ set -e
 ./query-node/start.sh
 
 ## Orion
-docker-compose up -d orion
+./start-orion.sh
 
 ## Faucet
 export SKIP_QUERY_NODE_CHECKS=true

start.sh

@@ -29,7 +29,7 @@ fi
 ./query-node/start.sh
 
 ## Orion
-docker-compose up -d orion
+./start-orion.sh
 
 ## Init the chain with some state
 if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then