[Merged by Bors] - Decouple cluster components from Fluvio cluster

.github/workflows/publish_latest_fluvio.yml

@@ -54,7 +54,9 @@ jobs:
       - name: Build and publish using fluvio-packages
         env:
           RUST_LOG: debug
-        run: cargo make -l verbose publish-fluvio-latest
+        run: |
+          cargo make -l verbose publish-fluvio-latest
+          cargo make -l verbose publish-fluvio-runner-latest
 
   # Bump the latest version of the Fluvio CLI on the package registry
   # This must be a distinct job than publish_fluvio_cli because this job requires

Cargo.toml

@@ -27,7 +27,7 @@ members = [
     "src/types",
     "src/utils",
     "src/extension-common",
-    "src/extension-runner-local",
+    "src/runner",
     "src/smartstream-wasm",
     "tests/runner"
 ]

DEVELOPER.md

@@ -1,19 +1,25 @@
 # Developing Fluvio
 
-Thank you for joining Fluvio community.  The goal of this document is to provide everything you need to get started with developing Fluvio.
+Thank you for joining Fluvio community. The goal of this document is to provide everything you need to get started with developing Fluvio.
 
 ## Assumptions
 
 Familiarity with
+
 - [Rust](https://www.rust-lang.org)
 - [Kubernetes](https://kubernetes.io)
 
-Developer guide and examples should work with the following platforms:
-- macOS X
+This Developer's guide and examples should work with the following platforms:
+
+- MacOS X
 - Linux  
+
 Other platforms such as Windows can be made to work, but we haven't tried them yet.
 
-To test and run services,  you need to get access to development Kubernetes cluster.  Our guide uses Minikube as examples because it is easy to it get it started, but you can use other Kubernetes cluster as well.  Please see  [Kubernetes](https://kubernetes.io) for setting up a development cluster.
+To test and run services, you need to get access to development Kubernetes cluster.
+Our guide uses Minikube as examples because it is easy to it get it started,
+but you can use another Kubernetes cluster as well.
+Please see [Kubernetes](https://kubernetes.io) for setting up a development cluster.
 
 Please read [doc](www.fluvio.io) for technical arch and operation guide.
 
@@ -23,12 +29,10 @@ Please read [doc](www.fluvio.io) for technical arch and operation guide.
 
 Please follow [setup](https://www.rust-lang.org/tools/install) instructions to install Rust and Cargo.
 
-
 ## Install minikube
 
 Please follow [minikube](https://minikube.sigs.k8s.io/docs/start/) to install minikube.
 
-
 ## Install Helm
 
 Please follow [helm setup](https://helm.sh/docs/intro/quickstart/) to install hel
@@ -80,38 +84,38 @@ In this case, we run `sc` and `spu` individually, allowing development testing.
 
 ## Starting SC
 
-The Streaming Controller (SC) is controller for fluvio cluster.  You only start a single SC for
-a single Fluvio cluster.
+The Streaming Controller (SC) is the controller for Fluvio cluster.
+You only start a single SC for a single Fluvio cluster.
 
-Following command, will start sc with default port (9003) and rust log level:
+To run the SC, you'll need to build and run the `fluvio-run` executable:
 
 ```
-$ RUST_LOG=fluvio=debug flvd cluster run sc 
+$ RUST_LOG=fluvio=debug cargo run --bin fluvio-run -- sc
 ```
 
 ## Starting SPU
 
-After SC is started, you can start adding unmanaged (custom SPU).
+After SC is started, you can start adding unmanaged (custom) SPUs.
 
-For each SPU, register spu.  For example, following registere spu with 5001 with public and private ports. 
-Normally, you only need to register SPU once.
+For each SPU, first register the SPU. For example, the following registers a SPU with ID 5001 with public and private ports. 
+Normally, you only need to register a SPU once.
 
 ```
-$ flvd custom-spu register --id 5001 --public-server 0.0.0.0:9010 --private-server  0.0.0.0:9011
+$ flvd cluster spu register --id 5001 --public-server 0.0.0.0:9010 --private-server  0.0.0.0:9011
 ```
 
 Then you can start SPU 5001
 
 ```
-$ RUST_LOG=fluvio=debug flvd cluster run spu -i 5001 -p 0.0.0.0:9010 -v 0.0.0.0:9011 > /tmp/spu_5001.log
+$ cargo run --bin fluvio-run -- spu -i 5001 -p 0.0.0.0:9010 -v 0.0.0.0:9011 > /tmp/spu_5001.log
 ```
 
-The logs can be founded in `/tmp/spu_5001`.log.
+The logs can be found in `/tmp/spu_5001.log`.
 
 Now, you should see SPU being active:
 
 ```
-$ flvd spu list
+$ flvd cluster spu list
  ID    NAME             STATUS  TYPE      RACK  PUBLIC        PRIVATE 
  5001  custom-spu-5001  Online  "custom"   -    0.0.0.0:9010  0.0.0.0:9011 
 ```
@@ -130,19 +134,17 @@ Ok!
 
 $ flvd consume topic -B -d
 hello world
-
 ```
 
 You can launch additional SPU as needed, just ensure that ports doesn't conflict with each other.
 For example, to add 2nd:
 
 ```
-$ flvd custom-spu register --id 5001 --public-server 0.0.0.0:9020 --private-server  0.0.0.0:9021
-$ flvd run spu -i 5001 -p 0.0.0.0:9020 -v 0.0.0.0:9021
+$ flvd cluster spu register --id 5001 --public-server 0.0.0.0:9020 --private-server  0.0.0.0:9021
+$ cargo run --bin fluvio-run -- spu -i 5001 -p 0.0.0.0:9020 -v 0.0.0.0:9021
 ```
 
 
-
 # Compiling for K8
 
 In order to deploy to minikube.  Docker image version must be built.
@@ -167,33 +169,38 @@ export TARGET_CC=x86_64-linux-musl-gcc
 For Linux, please see [musl wiki](https://wiki.musl-libc.org) for the installation of musl-gcc.
 
 For ubuntu:
+
 ```
 sudo apt install -y musl-tools
 export TARGET_CC=musl-gcc
 sudo ln -s /usr/bin/musl-gcc /usr/local/bin/x86_64-linux-musl-gcc
 ```
+
 ## To build docker image
 
 Run following command to build image
+
 ```
 $ make minikube_image
 ```
 
 Make sure you uninstall previous clusters for local and k8:
+
 ```
 $ flvd cluster delete --local
 $ flvd cluster delete
 ```
 
 Run command below now to run install with image just built
+
 ```
 $ fluvio cluster start --develop
 ```
 
 Topic creation, product and consumer can now be tested as with `local` cluster.
 
-
 You can remove fluvio cluster by
+
 ```
 fluvio cluster delete
 ```
@@ -209,6 +216,7 @@ Note that when you uninstall cluster, CLI will remove all related objects such a
 First install fluvio k8 cluster as normally.
 
 Then delete deployment:
+
 ```
 kubectl delete deployment fluvio-sc
 ```
@@ -219,8 +227,8 @@ Then, can run sc directly
 cargo run --bin fluvio-sc-k8
 ```
 
-
 ## Troubleshooting
+
 This guide helps users to solve issues they might face during the setup process. 
 
 ###### Cross-compilation errors
@@ -235,6 +243,7 @@ error: linker `x86_64-linux-musl-gcc` not found
 error: aborting due to previous error
 error: could not compile `fluvio-spu`.
 ```
+
 This is indicative that you need to add standard library for the target platform:
 
 ```
@@ -300,7 +309,7 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
 Rebuilding minikube cluster sometimes doesnt remove the storage class. Hence the sys chart installation throws an error. Make sure the storage class is deleted.
 
 ```
- kubectl delete storageclass fluvio-spu
+kubectl delete storageclass fluvio-spu
  ```
 
 #### Deleting partition

Makefile

@@ -23,16 +23,20 @@ SC_AUTH_CONFIG=./src/sc/test-data/auth_config
 SKIP_CHECK=--skip-checks
 EXTRA_ARG=
 
+TEST_RELEASE_PATH=$(if $(RELEASE),target/release,target/debug)
+export PATH := $(shell pwd)/${TEST_RELEASE_PATH}:${PATH}
+
 
 # install all tools required
 install_tools_mac:
 	brew install yq
 	brew install helm
 
-build_test:	TEST_RELEASE_FLAG=$(if $(RELEASE),--release,)
-build_test:	TEST_TARGET=$(if $(TARGET),--target $(TARGET),)
+build_test: TEST_RELEASE_FLAG=$(if $(RELEASE),--release,)
+build_test: TEST_TARGET=$(if $(TARGET),--target $(TARGET),)
 build_test:	install_test_target
 	cargo build $(TEST_RELEASE_FLAG) $(TEST_TARGET) --bin fluvio $(VERBOSE)
+	cargo build $(TEST_RELEASE_FLAG) $(TEST_TARGET) --bin fluvio-run $(VERBOSE)
 	cargo build $(TEST_RELEASE_FLAG) $(TEST_TARGET) --bin flv-test $(VERBOSE)
 
 install_test_target:
@@ -57,7 +61,7 @@ smoke-test-tls:	test-clean-up build_test
 smoke-test-tls-policy:	test-clean-up build_test
 	AUTH_POLICY=$(SC_AUTH_CONFIG)/policy.json X509_AUTH_SCOPES=$(SC_AUTH_CONFIG)/scopes.json  \
 	FLV_SPU_DELAY=$(SPU_DELAY) \
-	$(TEST_BIN) smoke --spu ${DEFAULT_SPU} --tls --local ${TEST_LOG} ${SKIP_CHECK} --keep-cluster -- --producer-iteration=${DEFAULT_ITERATION}
+	$(TEST_BIN) smoke --spu ${DEFAULT_SPU} --tls --local ${TEST_LOG} ${SKIP_CHECK} -- --producer-iteration=${DEFAULT_ITERATION}
 
 # test rbac with ROOT user
 smoke-test-tls-root:	smoke-test-tls-policy test-permission-user1
@@ -225,7 +229,7 @@ fluvio_image: fluvio_bin_linux
 fluvio_bin_linux: RELEASE_FLAG=$(if $(RELEASE),--release,)
 fluvio_bin_linux: install_musl
 	cargo build $(RELEASE_FLAG)   \
-		--bin fluvio_runner_local_cli --target $(TARGET_LINUX)
+		--bin fluvio-run --target $(TARGET_LINUX)
 
 make publish_fluvio_image:
 	curl \

install.sh

@@ -546,6 +546,8 @@ main() {
     # Let fluvio know it is invoked from installer
     say "☁️ Installing Fluvio Cloud..."
     FLUVIO_BOOTSTRAP=true "${FLUVIO_BIN}/fluvio" install fluvio/fluvio-cloud
+    say "☁️ Installing Fluvio Runner..."
+    FLUVIO_BOOTSTRAP=true "${FLUVIO_BIN}/fluvio" install fluvio/fluvio-run
 
     say "🎉 Install complete!"
     remind_path

k8-util/docker/build.sh

@@ -6,7 +6,7 @@ if [ -n "$MINIKUBE_DOCKER_ENV" ]; then
 fi
 
 tmp_dir=$(mktemp -d -t fluvio-docker-image-XXXXXX)
-cp target/x86_64-unknown-linux-musl/$CARGO_PROFILE/fluvio_runner_local_cli $tmp_dir/fluvio
+cp target/x86_64-unknown-linux-musl/$CARGO_PROFILE/fluvio-run $tmp_dir/fluvio-run
 cp $(dirname $0)/fluvio.Dockerfile $tmp_dir/Dockerfile
 cd $tmp_dir
 docker build -t infinyon/fluvio:$DOCKER_TAG .

k8-util/docker/fluvio.Dockerfile

@@ -1,2 +1,2 @@
 FROM alpine:3.12
-COPY fluvio fluvio
+COPY fluvio-run fluvio-run

k8-util/helm/fluvio-app/templates/sc-deployment.yaml

@@ -26,7 +26,7 @@ spec:
           env:
             - name: RUST_LOG
               value: {{ .Values.scLog }}
-          command: ["/fluvio", "run", "sc"]
+          command: ["/fluvio-run", "sc"]
       {{ if .Values.tls }}
           args:
             - --tls