feat mysql and mariadb

Signed-off-by: yvonnexyang <yvonnexyang@tencent.com>
goharbor · Apr 24, 2022 · 56b85c9 · 56b85c9
1 parent 39a5ea7
commit 56b85c9
Show file tree

Hide file tree

Showing 46 changed files with 2,417 additions and 107 deletions.
diff --git a/.github/workflows/CI-mariadb.yml b/.github/workflows/CI-mariadb.yml
@@ -0,0 +1,95 @@
+name: CI
+env:
+   DATABASE_TYPE: mariadb
+   MYSQL_HOST: localhost
+   MYSQL_PORT: 3306
+   MYSQL_USERNAME: root
+   MYSQL_PASSWORD: root123
+   MYSQL_DATABASE: registry
+   POSTGRESQL_HOST: localhost
+   POSTGRESQL_PORT: 5432
+   POSTGRESQL_USR: postgres
+   POSTGRESQL_PWD: root123
+   POSTGRESQL_DATABASE: registry
+   DOCKER_COMPOSE_VERSION: 1.23.0
+   HARBOR_ADMIN: admin
+   HARBOR_ADMIN_PASSWD: Harbor12345
+   CORE_SECRET: tempString
+   KEY_PATH: "/data/secret/keys/secretkey"
+   REDIS_HOST: localhost
+   REG_VERSION: v2.7.1-patch-2819-2553
+   UI_BUILDER_VERSION: 1.6.0
+
+on:
+  pull_request:
+  push:
+    paths-ignore:
+    - 'docs/**'
+
+jobs:
+  UTTEST4MARIADB:
+    env:
+      UTTEST: true
+    runs-on:
+      #- self-hosted
+      - ubuntu-latest
+    timeout-minutes: 100
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.17.7
+        id: go
+      - name: setup Docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+        with:
+          docker_version: 20.04
+          docker_channel: stable
+      - uses: actions/checkout@v2
+        with:
+          path: src/github.com/goharbor/harbor
+      - name: setup env
+        run: |
+          cd src/github.com/goharbor/harbor
+          pwd
+          go env
+          echo "GOPATH=$(go env GOPATH):$GITHUB_WORKSPACE" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+          echo "TOKEN_PRIVATE_KEY_PATH=${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem" >> $GITHUB_ENV
+        shell: bash
+      - name: before_install
+        run: |
+          set -x
+          cd src/github.com/goharbor/harbor
+          pwd
+          env
+          #sudo apt install -y xvfb
+          #xvfb-run ls
+          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          chmod +x docker-compose
+          sudo mv docker-compose /usr/local/bin
+          IP=`hostname -I | awk '{print $1}'`
+          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
+          echo "IP=$IP" >> $GITHUB_ENV
+          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+          sudo service docker restart
+      - name: install
+        run: |
+          cd src/github.com/goharbor/harbor
+          env
+          df -h
+          bash ./tests/showtime.sh ./tests/ci/ut_install.sh
+      - name: script
+        run: |
+          echo IP: $IP
+          df -h
+          cd src/github.com/goharbor/harbor
+          bash ./tests/showtime.sh ./tests/ci/ut_run.sh $IP
+          df -h
+      - name: Codecov For BackEnd
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./src/github.com/goharbor/harbor/profile.cov
+          flags: unittests
+
diff --git a/.github/workflows/CI-mysql.yml b/.github/workflows/CI-mysql.yml
@@ -0,0 +1,95 @@
+name: CI
+env:
+   DATABASE_TYPE: mysql
+   MYSQL_HOST: localhost
+   MYSQL_PORT: 3308
+   MYSQL_USERNAME: root
+   MYSQL_PASSWORD: root123
+   MYSQL_DATABASE: registry
+   POSTGRESQL_HOST: localhost
+   POSTGRESQL_PORT: 5432
+   POSTGRESQL_USR: postgres
+   POSTGRESQL_PWD: root123
+   POSTGRESQL_DATABASE: registry
+   DOCKER_COMPOSE_VERSION: 1.23.0
+   HARBOR_ADMIN: admin
+   HARBOR_ADMIN_PASSWD: Harbor12345
+   CORE_SECRET: tempString
+   KEY_PATH: "/data/secret/keys/secretkey"
+   REDIS_HOST: localhost
+   REG_VERSION: v2.7.1-patch-2819-2553
+   UI_BUILDER_VERSION: 1.6.0
+
+on:
+  pull_request:
+  push:
+    paths-ignore:
+    - 'docs/**'
+
+jobs:
+  UTTEST4MARIADB:
+    env:
+      UTTEST: true
+    runs-on:
+      #- self-hosted
+      - ubuntu-latest
+    timeout-minutes: 100
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.17.7
+        id: go
+      - name: setup Docker
+        uses: docker-practice/actions-setup-docker@0.0.1
+        with:
+          docker_version: 20.04
+          docker_channel: stable
+      - uses: actions/checkout@v2
+        with:
+          path: src/github.com/goharbor/harbor
+      - name: setup env
+        run: |
+          cd src/github.com/goharbor/harbor
+          pwd
+          go env
+          echo "GOPATH=$(go env GOPATH):$GITHUB_WORKSPACE" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+          echo "TOKEN_PRIVATE_KEY_PATH=${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem" >> $GITHUB_ENV
+        shell: bash
+      - name: before_install
+        run: |
+          set -x
+          cd src/github.com/goharbor/harbor
+          pwd
+          env
+          #sudo apt install -y xvfb
+          #xvfb-run ls
+          curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+          chmod +x docker-compose
+          sudo mv docker-compose /usr/local/bin
+          IP=`hostname -I | awk '{print $1}'`
+          echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
+          echo "IP=$IP" >> $GITHUB_ENV
+          sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
+          sudo update-ca-certificates
+          sudo service docker restart
+      - name: install
+        run: |
+          cd src/github.com/goharbor/harbor
+          env
+          df -h
+          bash ./tests/showtime.sh ./tests/ci/ut_install.sh
+      - name: script
+        run: |
+          echo IP: $IP
+          df -h
+          cd src/github.com/goharbor/harbor
+          bash ./tests/showtime.sh ./tests/ci/ut_run.sh $IP
+          df -h
+      - name: Codecov For BackEnd
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./src/github.com/goharbor/harbor/profile.cov
+          flags: unittests
+
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -1,10 +1,16 @@
 name: CI
 env:
+   DATABASE_TYPE: postgresql
    POSTGRESQL_HOST: localhost
    POSTGRESQL_PORT: 5432
    POSTGRESQL_USR: postgres
    POSTGRESQL_PWD: root123
    POSTGRESQL_DATABASE: registry
+   MYSQL_HOST: localhost
+   MYSQL_PORT: 3306
+   MYSQL_USERNAME: root
+   MYSQL_PASSWORD: root123
+   MYSQL_DATABASE: registry
    DOCKER_COMPOSE_VERSION: 1.23.0
    HARBOR_ADMIN: admin
    HARBOR_ADMIN_PASSWD: Harbor12345

diff --git a/make/harbor.yml.tmpl b/make/harbor.yml.tmpl
@@ -142,6 +142,8 @@ _version: 2.5.0
 # Uncomment external_database if using external database.
 # external_database:
 #   harbor:
+#     # database type, default is postgresql, options include postgresql, mariadb and mysql
+#     type: harbor_db_type
 #     host: harbor_db_host
 #     port: harbor_db_port
 #     db_name: harbor_db_name

diff --git a/make/migrations/mysql/0015_1.10.0_schema.up.sql b/make/migrations/mysql/0015_1.10.0_schema.up.sql
@@ -34,7 +34,8 @@ CREATE TABLE scan_report
     report JSON,
     start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-    UNIQUE(digest, registration_uuid, mime_type)
+    UNIQUE(digest, registration_uuid, mime_type),
+    CHECK (report is null or JSON_VALID (report))
 );
 
 /** Add table for immutable tag  **/

diff --git a/make/migrations/mysql/0030_2.0.0_schema.up.sql b/make/migrations/mysql/0030_2.0.0_schema.up.sql
@@ -92,6 +92,7 @@ JOIN (
 ) AS ordered_art ON art.repository_name=ordered_art.repository_name AND art.digest=ordered_art.digest
 WHERE ordered_art.seq=1;
 
+ALTER TABLE artifact DROP INDEX unique_artifact;
 ALTER TABLE artifact DROP COLUMN tag;
 
 /*remove the duplicate artifact rows*/
@@ -102,7 +103,6 @@ WHERE id NOT IN (
 );
 
 SET sql_mode = '';
-ALTER TABLE artifact DROP INDEX unique_artifact;
 ALTER TABLE artifact ADD CONSTRAINT unique_artifact UNIQUE (repository_id, digest);
 
 /*set artifact size*/
@@ -127,7 +127,8 @@ CREATE TABLE artifact_reference
   annotations json,
   FOREIGN KEY (parent_id) REFERENCES artifact(id),
   FOREIGN KEY (child_id) REFERENCES artifact(id),
-  CONSTRAINT  unique_reference UNIQUE (parent_id, child_id)
+  CONSTRAINT  unique_reference UNIQUE (parent_id, child_id),
+  CHECK (annotations is null or JSON_VALID (annotations))
 );
 
 /* artifact_trash records deleted artifact */

diff --git a/make/migrations/mysql/0040_2.1.0_schema.up.sql b/make/migrations/mysql/0040_2.1.0_schema.up.sql
@@ -1,4 +1,4 @@
-ALTER TABLE project ADD COLUMN registry_id int;
+ALTER TABLE project ADD COLUMN IF NOT EXISTS registry_id int;
 ALTER TABLE cve_whitelist RENAME TO cve_allowlist;
 UPDATE role SET name='maintainer' WHERE name='master';
 UPDATE project_metadata SET name='reuse_sys_cve_allowlist' WHERE name='reuse_sys_cve_whitelist';
@@ -14,7 +14,8 @@ CREATE TABLE IF NOT EXISTS execution (
     start_time timestamp DEFAULT CURRENT_TIMESTAMP,
     end_time timestamp,
     revision int,
-    PRIMARY KEY (id)
+    PRIMARY KEY (id),
+    CHECK (extra_attrs is null or JSON_VALID (extra_attrs))
 );
 
 CREATE TABLE IF NOT EXISTS task (
@@ -31,16 +32,17 @@ CREATE TABLE IF NOT EXISTS task (
     start_time timestamp,
     update_time timestamp,
     end_time timestamp,
-    FOREIGN KEY (execution_id) REFERENCES execution(id)
+    FOREIGN KEY (execution_id) REFERENCES execution(id),
+    CHECK (extra_attrs is null or JSON_VALID (extra_attrs))
 );
 
 ALTER TABLE `blob` ADD COLUMN update_time timestamp default CURRENT_TIMESTAMP;
 ALTER TABLE `blob` ADD COLUMN status varchar(255) default 'none';
 ALTER TABLE `blob` ADD COLUMN version BIGINT default 0;
-CREATE INDEX idx_status ON `blob` (status);
-CREATE INDEX idx_version ON `blob` (version);
+CREATE INDEX IF NOT EXISTS idx_status ON `blob` (status);
+CREATE INDEX IF NOT EXISTS idx_version ON `blob` (version);
 
-CREATE TABLE p2p_preheat_instance (
+CREATE TABLE IF NOT EXISTS p2p_preheat_instance (
   id          SERIAL PRIMARY KEY NOT NULL,
   name        varchar(255) NOT NULL,
   description varchar(255),
@@ -69,22 +71,23 @@ CREATE TABLE IF NOT EXISTS p2p_preheat_policy (
     UNIQUE (name, project_id)
 );
 
-ALTER TABLE schedule ADD COLUMN vendor_type varchar(16);
-ALTER TABLE schedule ADD COLUMN vendor_id int;
-ALTER TABLE schedule ADD COLUMN cron varchar(64);
-ALTER TABLE schedule ADD COLUMN callback_func_name varchar(128);
-ALTER TABLE schedule ADD COLUMN callback_func_param text;
+ALTER TABLE schedule ADD COLUMN IF NOT EXISTS vendor_type varchar(16);
+ALTER TABLE schedule ADD COLUMN IF NOT EXISTS vendor_id int;
+ALTER TABLE schedule ADD COLUMN IF NOT EXISTS cron varchar(64);
+ALTER TABLE schedule ADD COLUMN IF NOT EXISTS callback_func_name varchar(128);
+ALTER TABLE schedule ADD COLUMN IF NOT EXISTS callback_func_param text;
 
 /*abstract the cron, callback function parameters from table retention_policy*/
 UPDATE schedule, (
-    SELECT id, data->>'$.trigger.references.job_id' AS schedule_id,
-        data->>'$.trigger.settings.cron' AS cron
+    SELECT id, replace(json_extract(data,'$.trigger.references.job_id'),'"','') AS schedule_id,
+        replace(json_extract(data,'$.trigger.settings.cron'),'"','') AS cron
         FROM retention_policy
     ) AS retention
 SET vendor_type= 'RETENTION', vendor_id=retention.id, schedule.cron = retention.cron,
     callback_func_name = 'RETENTION', callback_func_param=concat('{"PolicyID":', retention.id, ',"Trigger":"Schedule"}')
 WHERE schedule.id=retention.schedule_id;
 
+
 /*create new execution and task record for each schedule*/
 CREATE PROCEDURE PROC_UPDATE_EXECUTION_TASK ( ) BEGIN
 INSERT INTO execution ( vendor_type, vendor_id, `trigger` ) SELECT
@@ -120,13 +123,13 @@ END;
 
 CALL PROC_UPDATE_EXECUTION_TASK();
 
-ALTER TABLE schedule DROP COLUMN job_id;
-ALTER TABLE schedule DROP COLUMN status;
+ALTER TABLE schedule DROP COLUMN IF EXISTS job_id;
+ALTER TABLE schedule DROP COLUMN IF EXISTS status;
 
 UPDATE registry SET type = 'quay' WHERE type = 'quay-io';
 
 
-ALTER TABLE artifact ADD COLUMN icon varchar(255);
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS icon varchar(255);
 
 /*remove the constraint for name in table 'notification_policy'*/
 /*ALTER TABLE notification_policy DROP CONSTRAINT notification_policy_name_key;*/
@@ -146,7 +149,7 @@ INSERT INTO data_migrations (version) VALUES (
         ELSE 0
     END
 );
-ALTER TABLE schema_migrations DROP COLUMN data_version;
+ALTER TABLE schema_migrations DROP COLUMN IF EXISTS data_version;
 
 UPDATE artifact
 SET icon=(