Rewrite in go

Handle #1
jvassev · Mar 12, 2019 · b2f3119 · b2f3119
1 parent c7a8306
commit b2f3119
Show file tree

Hide file tree

Showing 331 changed files with 223,465 additions and 585 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,4 +6,5 @@ build/
 *.pyc
 .idea/
 *.iml
+.vscode/
 image2ipfs/git-revision
diff --git a/image2ipfs-server/Dockerfile → Dockerfile b/image2ipfs-server/Dockerfile → Dockerfile
@@ -2,9 +2,10 @@ FROM golang:1.10 as builder
 
 WORKDIR /go/src/github.com/jvassev/image2ipfs
 COPY . .
-RUN make install-server
+ARG GIT_VERSION
+RUN make nested-build GIT_VERSION=$GIT_VERSION
 
 # Speed up local builds where vendor is populated
 FROM busybox
-ENTRYPOINT ["image2ipfs-server"]
-COPY --from=builder /go/bin/image2ipfs-server /bin
+ENTRYPOINT ["image2ipfs", "server"]
+COPY --from=builder /go/bin/image2ipfs /bin
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/Makefile b/Makefile
@@ -1,55 +1,57 @@
 TAG               ?= latest
 IMAGE             ?= jvassev/ipfs-registry
 DOCKER_BUILD_ARGS ?=
+PKG               := github.com/jvassev/image2ipfs
+IPFS_GATEWAY      ?= http://localhost:8080
 
-IPFS_GATEWAY ?= http://localhost:8080
-
-dist: version
-	python setup.py sdist
-
-lint:
-	cd image2ipfs && pylint --rcfile pylint.rc *.py
+VERSION           := 0.1.0
+GIT_VERSION       ?= $(shell git rev-parse HEAD)
+LDFLAGS           := -X github.com/jvassev/image2ipfs/util.Version=$(VERSION)/$(GIT_VERSION) -w -s
 
 test:
-	cd image2ipfs && python -m unittest discover
-
-
-clean:
-	rm -fr image2ipfs/*.pyc
-	rm -fr dist/ build/ *.egg-info
-
-install: clean version
-	python setup.py install
-	image2ipfs --version
+	go test -v $(PKG)
 
-version:
-	@git describe --match 'v[0-9]*' --dirty='.m' --always > image2ipfs/git-revision
-	@cat image2ipfs/git-revision
+install: test
+	CGO_ENABLED=0 go install -v -ldflags "$(LDFLAGS)" $(PKG)
 
-upload: clean
-	python setup.py sdist upload
+build-image: test
+	CGO_ENABLED=0 docker build $(DOCKER_BUILD_ARGS) -t $(IMAGE):$(TAG) --build-arg GIT_VERSION=$(GIT_VERSION) .
 
-
-build-image:
-	docker build $(DOCKER_BUILD_ARGS) -t $(IMAGE):$(TAG) -f image2ipfs-server/Dockerfile .
-	docker tag    $(IMAGE):$(TAG) $(IMAGE):latest
+nested-build: test
+	CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go install -v -ldflags "$(LDFLAGS)" $(PKG)
 
 shell:
-	docker run --rm -ti -u root -v `pwd`:/workspace --entrypoint=/bin/sh $(IMAGE)
+	docker run --rm -ti -u root -v `pwd`:/workspace --entrypoint=/bin/sh $(IMAGE):$(TAG)
 
 push-image: build-image
 	docker push $(IMAGE):$(TAG)
 
-attach:
-	@docker exec -ti $(CONT)  /bin/bash
-
 guess-tag:
 	@echo TAG=`git describe --match 'v[0-9]*' --dirty='.m' --always`
 
-install-server:
-	CGO_ENABLED=0 go install github.com/jvassev/image2ipfs/image2ipfs-server
-
 run-server: build-image
 	docker run -ti --rm \
 		--net=host \
 		$(IMAGE):$(TAG)
+
+clean:
+	rm -fr dist
+
+dist: clean test dist-linux dist-mac dist-win
+
+dist-linux:
+	@mkdir -p dist
+	GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -o dist/image2ipfs -v -ldflags "$(LDFLAGS)" $(PKG)
+	gzip -S -$(VERSION)_amd64_linux.gz dist/image2ipfs
+
+dist-mac:
+	@mkdir -p dist
+	GOARCH=amd64 GOOS=darwin CGO_ENABLED=0 go build -o dist/image2ipfs -v -ldflags "$(LDFLAGS)" $(PKG)
+	gzip -S -$(VERSION)_amd64_darwin.gz dist/image2ipfs
+
+dist-win:
+	@mkdir -p dist
+	GOARCH=amd64 GOOS=windows CGO_ENABLED=0 go build -o dist/image2ipfs.exe -v -ldflags "$(LDFLAGS)" $(PKG)
+	gzip -cvf dist/image2ipfs.exe > dist/image2ipfs-$(VERSION)_amd64_windows.gz
+	rm dist/image2ipfs.exe
+
diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@ image2ipfs works only with images prduced docker >= 1.10. On the bright side, al
 image2ipfs takes an image archive produced using `docker save`.
 The archive is extracted to a temp location then processed a bit. Finally, it is added to IPFS using `ipfs add -r`
 (you need to have the ipfs binary in your `PATH`). For a simple busybox image
-image2ipfs will produce a work dir like this:
+image2ipfs will produce something like this:
 ```
 busybox
 ├── blobs
@@ -23,11 +23,10 @@ busybox
 _v1 Manifests are not supported starting with version 0.0.6_
 But how do you get from something like QmQSF1oN4TXeU2kRvmjerEs62ZYfKPyRCqPvW1XTTc4fLS to a working `docker pull busybox`?
 
-The answer is simple: using url rewriting. In the `registry/` subdirectory of the project
-there is a trivial Go application that speaks the Registry v2 protocol but instead of serving the blobs and manifests it redirects
+The answer is simple: using url rewriting.There is a trivial Go application that speaks the Registry v2 protocol but instead of serving the blobs and manifests it redirects
 to an IPFS gateway of your choice.
 
-When pulling REPO:latest (with REPO=busybox in this example), Docker daemon will issue these requests:
+When pulling REPO:latest (with REPO=busybox in this example), Docker daemon will issue (roughly) these requests:
 ```
 GET /v2/
 GET /v2/REPO/manifest/latest
@@ -36,7 +35,7 @@ GET /v2/REPO/blobs/sha256:193bda8d9ac77416619eb556391a9c5447adb2abf12aab515d1b0c
 GET /v2/REPO/blobs/sha256:a1b1b81d3d1afdb8fe119b002318c12c20934713b9754a40f702adb18a2540b9
 ```
 
-All the Go app does is produce IPFS links to a an IPFS gateway and redirect docker there:
+The Go app serves redirects to an IPFS gateway like so:
 ```
 GET /ipfs/HASH/manifest/latest
 GET /ipfs/HASH/blobs/sha256:47bcc53f74dc94b1920f0b34f6036096526296767650f223433fe65c35f149eb
@@ -48,21 +47,12 @@ GET /ipfs/HASH/blobs/sha256:a1b1b81d3d1afdb8fe119b002318c12c20934713b9754a40f702
 # Installation
 ```bash
 $ git clone https://github.com/jvassev/image2ipfs
-$ cd image2ipfs
 
 # this will install a python command so pyenv or virtualenv is recommended rather than sudo
 $ make install
 
-$ image2ipfs -v
-0.0.6
-```
-
-You can also install using pip
-```bash
-$ pip install image2ipfs
-
-$ image2ipfs -v
-0.0.6
+$ image2ipfs --version
+0.1.0
 ```
 
 # Running the registry
@@ -72,18 +62,58 @@ You can pull the official image from dockerhub. This example assumes the gateway
 docker run -td --name ipfs-registry -e IPFS_GATEWAY=http://localhost:8080 --net host jvassev/ipfs-registry
 ```
 
-Or build from source:
+Or build from source and skip Docker:
 ```bash
-# build image locally
-make build-image
+$ make install
+
+$ image2ipfs server
+2019/03/12 15:43:58 Using IPFS gateway http://127.0.0.1:8080
+2019/03/12 15:43:58 Serving on :5000
+```
+
+# Synopsis
+
+The client and server live in the same binary. You can find prebuilt releases for Linux, Mac and Windows on the Releases page. The command flags are compatible with the legacy Python version.
+
+```
+usage: image2ipfs [<flags>] <command> [<args> ...]
 
-# or install to $GOPATH/bin
-make install
+Flags:
+      --help       Show context-sensitive help (also try --help-long and --help-man).
+      --version    Show application version.
+  -r, --registry="http://localhost:5000"
+                   Registry to use when generating pull URL
+  -n, --no-add     Don`t add to IPFS, just print directory
+  -q, --quiet      Produce less output - just the final pullable image name
+  -d, --debug      Leave workdir intact (useful for debugging)
+  -i, --input="-"  Docker image archive to process, defaults to stdin. Use - to explicitly set stdin
+
+Commands:
+  help [<command>...]
+    Show help.
+
+
+  client [<flags>]
+    Populate IPFS node with image data
+
+    -r, --registry="http://localhost:5000"
+                     Registry to use when generating pull URL
+    -n, --no-add     Don`t add to IPFS, just print directory
+    -q, --quiet      Produce less output - just the final pullable image name
+    -d, --debug      Leave workdir intact (useful for debugging)
+    -i, --input="-"  Docker image archive to process, defaults to stdin. Use - to explicitly set stdin
+
+  server [<flags>]
+    Run an IFPS-backed registry
+
+    --gateway="http://127.0.0.1:8080"
+                    IPFS gateway. It must be reachable by pulling clients
+    --addr=":5000"  Listen address.
 ```
 
 # Configure Docker
 
-As usual add `--insecure-registry localhost:5000` to your docker daemon args
+As usual add `--insecure-registry localhost:5000` to your docker daemon args. You need to put the `image2ipfs server` behind a reverse proxy if you want to do proper TLS termination.
 
 # Demo
 Assuming you have completed the steps above, let's publish a centos:7 image to IPFS!
@@ -105,74 +135,63 @@ Server:
  Built:        Sun May  1 20:27:17 2016
  OS/Arch:      linux/amd64
 
-$ image2ipfs -v
-0.0.6
+$ image2ipfs --version
+0.1.0
 
 $ docker pull centos:7
 7: Pulling from library/centos
-fa5be2806d4c: Pull complete
-2ebc6e0c744d: Pull complete
-044c0f15c4d9: Pull complete
-28e524afdd05: Pull complete
-Digest: sha256:b3da5267165bbaa9a75d8ee21a11728c6fba98c0944dfa28f15c092877bb4391
+a02a4930cb5d: Pull complete
+Digest: sha256:184e5f35598e333bfa7de10d8fb1cebb5ee4df5bc0f970bf2b1e7c7345136426
 Status: Downloaded newer image for centos:7
 
 $ docker save centos:7 | image2ipfs
-Extracting to /tmp/tmpluDoZF
-Preparing image in /tmp/tmp3iBMaF
-	Processing centos@sha256:49dccac9d468cc1d3d9a3eafb835d79ed56b99c931ab232774d32b75d220d241
-	Compressing layer /tmp/tmpluDoZF/768d4f50f65f00831244703e57f64134771289e3de919a576441c9140e037ea2/layer.tar
-	Compressing layer /tmp/tmpluDoZF/da060eb693ac47689ca545355a060197bd2aadad76ca67535e95838de0737302/layer.tar
-	Compressing layer /tmp/tmpluDoZF/a3f571afcd5241f02ca23fd50e45a403437d20ae3e215413d602e2deae3f86bc/layer.tar
-	Compressing layer /tmp/tmpluDoZF/49dccac9d468cc1d3d9a3eafb835d79ed56b99c931ab232774d32b75d220d241/layer.tar
-	Compressing layer /tmp/tmpluDoZF/49dccac9d468cc1d3d9a3eafb835d79ed56b99c931ab232774d32b75d220d241/layer.tar
-	Compressing layer /tmp/tmpluDoZF/a3f571afcd5241f02ca23fd50e45a403437d20ae3e215413d602e2deae3f86bc/layer.tar
-	Compressing layer /tmp/tmpluDoZF/da060eb693ac47689ca545355a060197bd2aadad76ca67535e95838de0737302/layer.tar
-	Compressing layer /tmp/tmpluDoZF/768d4f50f65f00831244703e57f64134771289e3de919a576441c9140e037ea2/layer.tar
-Image ready: QmdLmBErojQctCn9MHMV61BNA6ue2fyDZZAvFRF32cccTT
-	Browse image at http://localhost:8080/ipfs/QmdLmBErojQctCn9MHMV61BNA6ue2fyDZZAvFRF32cccTT
-	Dockerized hash ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq
-	You can pull using localhost:5000/ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq/centos
-localhost:5000/ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq/centos
+docker save centos:7 | /home/jvassev/more/gohome/bin/image2ipfs
+2019/03/12 15:38:59 workdir is /tmp/image2ipfs887013748
+2019/03/12 15:39:01 processing centos:7
+2019/03/12 15:39:01 processing layer /tmp/image2ipfs887013748/9841c41d4b0d8fe3bf22b7c3c12e7633870218fffec04d84e7d62993ac175a19/layer.tar
+2019/03/12 15:39:11 using ipfs in /home/jvassev/.bin/ipfs
+2019/03/12 15:39:12 image ready Qmcsr3naQWG6YzeWTCa7Ee55JUqDdAwvo3j8VyAuRcrHvM
+2019/03/12 15:39:12 browse image at http://localhost:8080/ipfs/Qmcsr3naQWG6YzeWTCa7Ee55JUqDdAwvo3j8VyAuRcrHvM
+2019/03/12 15:39:12 dockerized hash ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq
+2019/03/12 15:39:12 you can pull using localhost:5000/ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq/centos
+localhost:5000/ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq/centos
 
 # delete the image to make docker pull layers again
 $ docker rmi centos:7
 ```
 
-The last line contains the image name (you can get only it by passing `-q` to image2ipfs). If you have the IPFS registry
-running you should be able to pull:
+`image2ipfs` will produce a single line to stdout - the pullable docker image name. The rest is debug output you can suppress with `-q`.
 
 ```bash
-$ docker pull localhost:5000/ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq/centos
+$ docker pull localhost:5000/ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq/centos
 Using default tag: latest
-latest: Pulling from ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq/centos
-Digest: sha256:2c863897110a2aa59287df9e4544fb4d15f83b41e44ed578872e6314b1025a1e
-Status: Image is up to date for localhost:5000/ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq/centos:latest
+latest: Pulling from ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq/centos
+Digest: sha256:4c7a24018edbcb72ec0e6a7ff6809db4aa2f306784bd79ab971e0497954ed4e2
+Status: Downloaded newer image for localhost:5000/ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq/centos:latest
 
-$ docker history localhost:5000/ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq/centos
+$ docker history localhost:5000/ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq/centos
 IMAGE               CREATED             CREATED BY                                      SIZE                COMMENT
 778a53015523        4 weeks ago         /bin/sh -c #(nop) CMD ["/bin/bash"]             0 B
 <missing>           4 weeks ago         /bin/sh -c #(nop) LABEL name=CentOS Base Imag   0 B
 <missing>           4 weeks ago         /bin/sh -c #(nop) ADD file:6dd89087d4d418ca0c   196.7 MB
 <missing>           7 months ago        /bin/sh -c #(nop) MAINTAINER The CentOS Proje   0 B
 ```
 
-Depending on how you have started the ipfs-registry the docker daemon will be redirected to an IPFS gateway of your choice. By default the registry
+Depending on how you have started the image2ipfs the docker daemon will be redirected to an IPFS gateway of your choice. By default the registry
 will redirect to the ipfs daemon running locally (http://localhost:8080).
 
-But what is this "Dockerized hash ciqn5zu57ciucp3gw2hwxymuwz3tgjlvfdfwg3xhwx456y4xxydkhmq" in the output?
+But what is this "dockerized hash ciqnqalrqxnqzubb2jllburc4e6wt2j7crjx2nxsdfbiw4sceq367lq" in the output?
 Docker requires image names to be all lowercase which doesn't play nicely with base58-encoded binary. A dockerized IPFS hash
-is just a base-32 of the same binary value. This is the reason why the ipfs-registry is not a simple nginx+rewrite rules: you need
-to do base-32 to base-58 conversions. If you see a string starting with `ciq` that's probably a dockerized ipfs hash.
-(OK, you can also do this with nginx_lua)
+is just a base-32 of the same binary value. If you see a string starting with `ciq..` that's probably a dockerized IPFS hash.
 
 In automated scenarios you'd probably want to run image2ipfs like this:
 ```bash
 
 # build whatever images
 $ docker built -t $TAG ...
-$ docker save $TAG | image2ipfs -q -r my-gateway.local:9090 | tee pull-url.txt
-my-gateway.local:9090/ciq..../centos
+
+# save the resulting image name
+$ docker save $TAG | image2ipfs -q -r my-gateway.local:8080 > pull-url.txt
 ```
 
 `-r my-gateway.local` instructs image2ipfs what pull url to produce. In a CI/CD you can distribute this generated url to downstream jobs that need to pull it.
@@ -183,23 +202,6 @@ Not sure. It would be great if an IPFS gateway could speak the Registry v2 proto
 
 The Dockerized hash can be shortened a bit if base-36 is used instead of base-32/hex.
 
-# Synopsis
-
-```
-usage: image2ipfs [-h] [--quiet] [--version] [--input INPUT] [--no-add]
-                  [--registry REGISTRY]
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --quiet, -q           produce less output
-  --version, -v         prints version
-  --input INPUT, -i INPUT
-                        Docker image archive to process, defaults to stdin
-  --no-add, -n          Don`t add to IPFS, just print directory
-  --registry REGISTRY, -r REGISTRY
-                        Registry to use when generating pull URL
-
-```
 
 # Changelog
 
@@ -209,6 +211,7 @@ optional arguments:
 * 0.0.4: Support for schema version 2 and docker > 1.10
 * 0.0.5: image2ipfs: stop handling archives produced by docker < 1.10, ipfs-registry can still work with all post-1.6.x dockers
 * 0.0.6: Deprecate manifest v1. Rewrite image2ipfs-server in golang
+* 0.1.0: Rewrite image2ipfs in go: Single binary for client and server
 
 # FAQ