Unify harbor DB environment variables

.github/workflows/CI-mariadb.yml

@@ -1,16 +1,11 @@
 name: CI
 env:
    DATABASE_TYPE: mariadb
-   MYSQL_HOST: localhost
-   MYSQL_PORT: 3306
-   MYSQL_USERNAME: root
-   MYSQL_PASSWORD: root123
-   MYSQL_DATABASE: registry
-   POSTGRESQL_HOST: localhost
-   POSTGRESQL_PORT: 5432
-   POSTGRESQL_USR: postgres
-   POSTGRESQL_PWD: root123
-   POSTGRESQL_DATABASE: registry
+   DB_HOST: localhost
+   DB_PORT: 3306
+   DB_USERNAME: root
+   DB_PASSWORD: root123
+   DB_DATABASE: registry
    DOCKER_COMPOSE_VERSION: 1.23.0
    HARBOR_ADMIN: admin
    HARBOR_ADMIN_PASSWD: Harbor12345

.github/workflows/CI-mysql.yml

@@ -1,16 +1,11 @@
 name: CI
 env:
    DATABASE_TYPE: mysql
-   MYSQL_HOST: localhost
-   MYSQL_PORT: 3308
-   MYSQL_USERNAME: root
-   MYSQL_PASSWORD: root123
-   MYSQL_DATABASE: registry
-   POSTGRESQL_HOST: localhost
-   POSTGRESQL_PORT: 5432
-   POSTGRESQL_USR: postgres
-   POSTGRESQL_PWD: root123
-   POSTGRESQL_DATABASE: registry
+   DB_HOST: localhost
+   DB_PORT: 3308
+   DB_USERNAME: root
+   DB_PASSWORD: root123
+   DB_DATABASE: registry
    DOCKER_COMPOSE_VERSION: 1.23.0
    HARBOR_ADMIN: admin
    HARBOR_ADMIN_PASSWD: Harbor12345
@@ -27,7 +22,7 @@ on:
     - 'docs/**'
 
 jobs:
-  UTTEST4MARIADB:
+  UTTEST4MYSQL:
     env:
       UTTEST: true
     runs-on:

.github/workflows/CI.yml

@@ -1,16 +1,11 @@
 name: CI
 env:
    DATABASE_TYPE: postgresql
-   POSTGRESQL_HOST: localhost
-   POSTGRESQL_PORT: 5432
-   POSTGRESQL_USR: postgres
-   POSTGRESQL_PWD: root123
-   POSTGRESQL_DATABASE: registry
-   MYSQL_HOST: localhost
-   MYSQL_PORT: 3306
-   MYSQL_USERNAME: root
-   MYSQL_PASSWORD: root123
-   MYSQL_DATABASE: registry
+   DB_HOST: localhost
+   DB_PORT: 5432
+   DB_USERNAME: postgres
+   DB_PASSWORD: root123
+   DB_DATABASE: registry
    DOCKER_COMPOSE_VERSION: 1.23.0
    HARBOR_ADMIN: admin
    HARBOR_ADMIN_PASSWD: Harbor12345

make/migrations/mysql/0001_initial_schema.up.sql

@@ -65,8 +65,9 @@ create table project (
 );
 
 
-insert into project (owner_id, name, creation_time, update_time) values
-(1, 'library', NOW(), NOW());
+
+insert into project (owner_id, name, creation_time, update_time)
+select user_id , 'library', NOW(), NOW() from harbor_user where username = 'admin';
 
 create table project_member (
  id SERIAL NOT NULL,

make/migrations/mysql/0040_2.1.0_schema.up.sql

@@ -1,4 +1,19 @@
-ALTER TABLE project ADD COLUMN IF NOT EXISTS registry_id int;
+CREATE PROCEDURE PROC_ADD_COLUMN_IF_NOT_EXISTS (in TB_NAME varchar(64), in CL_NAME varchar(64), in CL_TYPE varchar(64)) BEGIN
+SELECT count(*) INTO @EXIST_CL
+FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA = database()
+  AND TABLE_NAME = TB_NAME
+  AND COLUMN_NAME = CL_NAME LIMIT 1;
+
+SET @sql_cl = IF (@EXIST_CL <= 0, CONCAT('ALTER TABLE `', TB_NAME, '` ADD COLUMN `',CL_NAME, '` ', CL_TYPE),
+    'select \' COLUMN EXISTS\' status');
+PREPARE stmt_cl FROM @sql_cl;
+EXECUTE stmt_cl;
+DEALLOCATE PREPARE stmt_cl;
+END;
+
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('project', 'registry_id', 'int');
+
 ALTER TABLE cve_whitelist RENAME TO cve_allowlist;
 UPDATE role SET name='maintainer' WHERE name='master';
 UPDATE project_metadata SET name='reuse_sys_cve_allowlist' WHERE name='reuse_sys_cve_whitelist';
@@ -36,11 +51,26 @@ CREATE TABLE IF NOT EXISTS task (
     CHECK (extra_attrs is null or JSON_VALID (extra_attrs))
 );
 
-ALTER TABLE `blob` ADD COLUMN update_time timestamp default CURRENT_TIMESTAMP;
-ALTER TABLE `blob` ADD COLUMN status varchar(255) default 'none';
-ALTER TABLE `blob` ADD COLUMN version BIGINT default 0;
-CREATE INDEX IF NOT EXISTS idx_status ON `blob` (status);
-CREATE INDEX IF NOT EXISTS idx_version ON `blob` (version);
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('blob', 'update_time', 'timestamp default CURRENT_TIMESTAMP');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('blob', 'status', 'varchar(255) default \'none\'');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('blob', 'version', 'BIGINT default 0');
+
+CREATE PROCEDURE PROC_CREATE_INDEX_IF_NOT_EXISTS (in TB_NAME varchar(64), in CL_NAME varchar(64), in IND_NAME varchar(64)) BEGIN
+SELECT count(*) INTO @EXIST_IND
+FROM INFORMATION_SCHEMA.STATISTICS
+WHERE TABLE_SCHEMA = database()
+  AND TABLE_NAME = TB_NAME
+  AND INDEX_NAME = IND_NAME LIMIT 1;
+
+SET @sql_ind = IF (@EXIST_IND <= 0, CONCAT('CREATE INDEX `', IND_NAME, '` ON `', TB_NAME, '` (', CL_NAME, ')'),
+    'select \' INDEX EXISTS\' status');
+PREPARE stmt_ind FROM @sql_ind;
+EXECUTE stmt_ind;
+DEALLOCATE PREPARE stmt_ind;
+END;
+
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('blob', 'status', 'idx_status');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('blob', 'version', 'idx_version');
 
 CREATE TABLE IF NOT EXISTS p2p_preheat_instance (
   id          SERIAL PRIMARY KEY NOT NULL,
@@ -71,11 +101,11 @@ CREATE TABLE IF NOT EXISTS p2p_preheat_policy (
     UNIQUE (name, project_id)
 );
 
-ALTER TABLE schedule ADD COLUMN IF NOT EXISTS vendor_type varchar(16);
-ALTER TABLE schedule ADD COLUMN IF NOT EXISTS vendor_id int;
-ALTER TABLE schedule ADD COLUMN IF NOT EXISTS cron varchar(64);
-ALTER TABLE schedule ADD COLUMN IF NOT EXISTS callback_func_name varchar(128);
-ALTER TABLE schedule ADD COLUMN IF NOT EXISTS callback_func_param text;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'vendor_type', 'varchar(16)');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'vendor_id', 'int');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'cron', 'varchar(64)');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'callback_func_name', 'varchar(128)');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'callback_func_param', 'text');
 
 /*abstract the cron, callback function parameters from table retention_policy*/
 UPDATE schedule, (
@@ -123,13 +153,12 @@ END;
 
 CALL PROC_UPDATE_EXECUTION_TASK();
 
-ALTER TABLE schedule DROP COLUMN IF EXISTS job_id;
-ALTER TABLE schedule DROP COLUMN IF EXISTS status;
+ALTER TABLE schedule DROP COLUMN job_id;
+ALTER TABLE schedule DROP COLUMN status;
 
 UPDATE registry SET type = 'quay' WHERE type = 'quay-io';
 
-
-ALTER TABLE artifact ADD COLUMN IF NOT EXISTS icon varchar(255);
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('artifact', 'icon', 'varchar(255)');
 
 /*remove the constraint for name in table 'notification_policy'*/
 /*ALTER TABLE notification_policy DROP CONSTRAINT notification_policy_name_key;*/
@@ -149,7 +178,7 @@ INSERT INTO data_migrations (version) VALUES (
         ELSE 0
     END
 );
-ALTER TABLE schema_migrations DROP COLUMN IF EXISTS data_version;
+ALTER TABLE schema_migrations DROP COLUMN data_version;
 
 UPDATE artifact
 SET icon=(

make/migrations/mysql/0050_2.2.0_schema.up.sql

@@ -18,21 +18,21 @@ Clean the dirty data in quota/quota_usage
 DELETE FROM quota WHERE reference='project' AND reference_id NOT IN (SELECT project_id FROM project WHERE deleted=FALSE);
 DELETE FROM quota_usage WHERE reference='project' AND reference_id NOT IN (SELECT project_id FROM project WHERE deleted=FALSE);
 
-ALTER TABLE schedule ADD COLUMN cron_type varchar(64);
-ALTER TABLE robot ADD COLUMN secret varchar(2048);
-ALTER TABLE robot ADD COLUMN salt varchar(64);
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'cron_type', 'varchar(64)');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('robot', 'secret', 'varchar(2048)');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('robot', 'salt', 'varchar(64)');
 
 SET sql_mode = '';
-ALTER TABLE task ADD COLUMN vendor_type varchar(16);
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('task', 'vendor_type', 'varchar(16)');
 UPDATE task, execution SET task.vendor_type = execution.vendor_type WHERE task.execution_id = execution.id;
 ALTER TABLE task MODIFY COLUMN vendor_type varchar(16) NOT NULL;
 
-ALTER TABLE execution ADD COLUMN update_time timestamp;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('execution', 'update_time', 'timestamp');
 
 UPDATE artifact AS art
 SET size = ( SELECT sum( size )  FROM `blob` WHERE digest IN ( SELECT digest_blob FROM artifact_blob WHERE digest_af = art.digest ) );
 
-ALTER TABLE robot ADD COLUMN duration int;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('robot', 'duration', 'int');
 
 CREATE TABLE  IF NOT EXISTS role_permission (
  id SERIAL PRIMARY KEY NOT NULL,
@@ -188,7 +188,7 @@ END;
 CALL PROC_UPDATE_REPLICATION_EXECUTION();
 
 /*move the replication execution records into the new execution table*/
-ALTER TABLE replication_execution ADD COLUMN new_execution_id int;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('replication_execution', 'new_execution_id', 'int');
 INSERT INTO execution ( vendor_type, vendor_id, status, status_message, revision, `trigger`, start_time, end_time ) SELECT
 'REPLICATION',
 rep_exec.policy_id,
@@ -357,6 +357,7 @@ SET sql_mode = '';
 ALTER TABLE execution MODIFY COLUMN vendor_type varchar(64) NOT NULL;
 ALTER TABLE `schedule` MODIFY COLUMN vendor_type varchar(64) DEFAULT NULL;
 ALTER TABLE `schedule` ADD COLUMN extra_attrs JSON;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'extra_attrs', 'JSON');
 ALTER TABLE task MODIFY COLUMN vendor_type varchar(64) NOT NULL;
 
 /* Remove these columns in scan_report because execution-task pattern will handle them */
@@ -783,7 +784,7 @@ DELETE rt FROM retention_task AS rt LEFT JOIN retention_execution re ON rt.execu
 WHERE re.id IS NULL;
 
 /*move the replication execution records into the new execution table*/
-ALTER TABLE retention_execution ADD COLUMN IF NOT EXISTS new_execution_id int;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('retention_execution', 'new_execution_id', 'int');
 
 CREATE PROCEDURE PROC_UPDATE_EXECUTION_AND_RETENTION_EXECUTION ( ) BEGIN
 	DECLARE

make/migrations/mysql/0052_2.2.2_schema.up.sql

@@ -1,2 +1,2 @@
-ALTER TABLE schedule ADD COLUMN revision integer;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('schedule', 'revision', 'integer');
 UPDATE schedule set revision = 0;

make/migrations/mysql/0053_2.2.3_schema.up.sql

@@ -1,4 +1,4 @@
-CREATE INDEX idx_artifact_push_time ON artifact (push_time);
-CREATE INDEX idx_tag_push_time ON tag (push_time);
-CREATE INDEX idx_tag_artifact_id ON tag (artifact_id);
-CREATE INDEX idx_artifact_reference_child_id ON artifact_reference (child_id);
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('artifact', 'push_time', 'idx_artifact_push_time');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('tag', 'push_time', 'idx_tag_push_time');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('tag', 'artifact_id', 'idx_tag_artifact_id');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('artifact_reference', 'child_id', 'idx_artifact_reference_child_id');

make/migrations/mysql/0060_2.3.0_schema.up.sql

@@ -1,8 +1,8 @@
-ALTER TABLE replication_policy ADD COLUMN dest_namespace_replace_count int;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('replication_policy', 'dest_namespace_replace_count', 'int');
 UPDATE replication_policy SET dest_namespace_replace_count=-1 WHERE dest_namespace IS NOT NULL;
 
-CREATE INDEX idx_artifact_push_time ON artifact (push_time);
-CREATE INDEX idx_tag_push_time ON tag (push_time);
-CREATE INDEX idx_tag_artifact_id ON tag (artifact_id);
-CREATE INDEX idx_artifact_reference_child_id ON artifact_reference (child_id);
-CREATE INDEX idx_audit_log_op_time ON audit_log (op_time);
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('artifact', 'push_time', 'idx_artifact_push_time');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('tag', 'push_time', 'idx_tag_push_time');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('tag', 'artifact_id', 'idx_tag_artifact_id');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('artifact_reference', 'child_id', 'idx_artifact_reference_child_id');
+CALL PROC_CREATE_INDEX_IF_NOT_EXISTS('audit_log', 'op_time', 'idx_audit_log_op_time');

make/migrations/mysql/0070_2.4.0_schema.up.sql

@@ -1,11 +1,13 @@
 /* cleanup deleted user project members */
 DELETE FROM project_member WHERE project_member.entity_type = 'u' AND EXISTS (SELECT NULL FROM harbor_user WHERE project_member.entity_id = harbor_user.user_id AND harbor_user.deleted = true );
 
-ALTER TABLE replication_policy ADD COLUMN speed_kb int;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('replication_policy', 'speed_kb', 'int');
 
 /* add version fields for lock free quota */
 ALTER TABLE quota ADD COLUMN version bigint DEFAULT 0;
 ALTER TABLE quota_usage ADD COLUMN version bigint DEFAULT 0;
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('quota', 'version', 'bigint DEFAULT 0');
+CALL PROC_ADD_COLUMN_IF_NOT_EXISTS('quota_usage', 'version', 'bigint DEFAULT 0');
 
 /* convert Negligible to None for the severity of the vulnerability record */
 UPDATE vulnerability_record SET severity='None' WHERE severity='Negligible';

make/photon/prepare/templates/core/env.jinja

@@ -8,24 +8,14 @@ _REDIS_URL_REG={{redis_url_reg}}
 LOG_LEVEL={{log_level}}
 EXT_ENDPOINT={{public_url}}
 DATABASE_TYPE={{harbor_db_type}}
-{% if ( harbor_db_type == "mysql" or harbor_db_type == "mariadb" ) %}
-MYSQL_HOST={{harbor_db_host}}
-MYSQL_PORT={{harbor_db_port}}
-MYSQL_USERNAME={{harbor_db_username}}
-MYSQL_PASSWORD={{harbor_db_password}}
-MYSQL_DATABASE={{harbor_db_name}}
-MYSQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
-MYSQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
-{% else %}
-POSTGRESQL_HOST={{harbor_db_host}}
-POSTGRESQL_PORT={{harbor_db_port}}
-POSTGRESQL_USERNAME={{harbor_db_username}}
-POSTGRESQL_PASSWORD={{harbor_db_password}}
-POSTGRESQL_DATABASE={{harbor_db_name}}
-POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
-POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
-POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
-{% endif %}
+DB_HOST={{harbor_db_host}}
+DB_PORT={{harbor_db_port}}
+DB_USERNAME={{harbor_db_username}}
+DB_PASSWORD={{harbor_db_password}}
+DB_DATABASE={{harbor_db_name}}
+DB_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
+DB_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
+DB_SSLMODE={{harbor_db_sslmode}}
 REGISTRY_URL={{registry_url}}
 PORTAL_URL={{portal_url}}
 TOKEN_SERVICE_URL={{token_service_url}}

src/common/const.go

@@ -51,21 +51,14 @@ const (
 	ExtEndpoint                      = "ext_endpoint"
 	AUTHMode                         = "auth_mode"
 	DatabaseType                     = "database_type"
-	PostGreSQLHOST                   = "postgresql_host"
-	PostGreSQLPort                   = "postgresql_port"
-	PostGreSQLUsername               = "postgresql_username"
-	PostGreSQLPassword               = "postgresql_password"
-	PostGreSQLDatabase               = "postgresql_database"
-	PostGreSQLSSLMode                = "postgresql_sslmode"
-	PostGreSQLMaxIdleConns           = "postgresql_max_idle_conns"
-	PostGreSQLMaxOpenConns           = "postgresql_max_open_conns"
-	MySQLHOST                        = "mysql_host"
-	MySQLPort                        = "mysql_port"
-	MySQLUsername                    = "mysql_username"
-	MySQLPassword                    = "mysql_password"
-	MySQLDatabase                    = "mysql_database"
-	MySQLMaxIdleConns                = "mysql_max_idle_conns"
-	MySQLMaxOpenConns                = "mysql_max_open_conns"
+	DBHOST                           = "db_host"
+	DBPort                           = "db_port"
+	DBUsername                       = "db_username"
+	DBPassword                       = "db_password"
+	DBDatabase                       = "db_database"
+	DBSSLMode                        = "db_sslmode"
+	DBMaxIdleConns                   = "db_max_idle_conns"
+	DBMaxOpenConns                   = "db_max_open_conns"
 	SelfRegistration                 = "self_registration"
 	CoreURL                          = "core_url"
 	CoreLocalURL                     = "core_local_url"

src/common/dao/dao_test.go

@@ -97,12 +97,13 @@ const password string = "Abc12345"
 const projectName string = "test_project"
 
 func TestMain(m *testing.M) {
-	databases := []string{"postgresql", "mysql", "mariadb"}
+	dbType := os.Getenv("DATABASE_TYPE")
+	databases := []string{dbType}
 	for _, database := range databases {
 		log.Infof("run test cases for database: %s", database)
 		result := 1
 		switch database {
-		case "postgresql":
+		case "", "postgresql":
 			PrepareTestForPostgresSQL()
 		case "mysql", "mariadb":
 			PrepareTestForMySQL()