From 6a2d430427e4770bf97ac4e6eaac74bc56ab83b3 Mon Sep 17 00:00:00 2001 From: Brian McLaughlin Date: Thu, 18 Apr 2024 16:49:55 -0400 Subject: [PATCH 1/5] cherry picked #2122 and limited galaxy-importer range No-Issue --- galaxy_ng/tests/integration/api/test_artifact_upload.py | 5 ----- setup.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/galaxy_ng/tests/integration/api/test_artifact_upload.py b/galaxy_ng/tests/integration/api/test_artifact_upload.py index 00de5375bd..f8e0bcbce6 100644 --- a/galaxy_ng/tests/integration/api/test_artifact_upload.py +++ b/galaxy_ng/tests/integration/api/test_artifact_upload.py @@ -453,11 +453,6 @@ def test_ansible_lint_exception_AAH_2606(ansible_config, upload_artifact, hub_ve "meta/runtime.yml:1: yaml[new-line-at-end-of-file]:" + " No new line character at the end of file" ), - ( - "tests/sanity/ignore-2.10.txt:1: sanity[cannot-ignore]:" - + " Ignore file contains validate-modules:use-run-command-not-popen at line 1," - + " which is not a permitted ignore." - ) ] artifact = bc( diff --git a/setup.py b/setup.py index ea94fe81f2..c8da14a414 100644 --- a/setup.py +++ b/setup.py @@ -111,7 +111,7 @@ def _format_pulp_requirement(plugin, specifier=None, ref=None, gh_namespace="pul requirements = [ - "galaxy-importer>=0.4.16,<0.5.0", + "galaxy-importer>=0.4.16,<=0.4.19", "pulpcore>=3.28.21,<3.29.0", "pulp_ansible>=0.20.3,<0.21.0", "django-prometheus>=2.0.0", From 0e6041fbf2f7709ef9ef00b20982e661b0e92cf1 Mon Sep 17 00:00:00 2001 From: Brian McLaughlin Date: Fri, 19 Apr 2024 07:55:25 -0400 Subject: [PATCH 2/5] Pin galaxy-importer No-Issue --- requirements/requirements.common.txt | 31 +++--------------------- requirements/requirements.insights.txt | 31 +++--------------------- requirements/requirements.standalone.txt | 31 +++--------------------- 3 files changed, 9 insertions(+), 84 deletions(-) diff --git a/requirements/requirements.common.txt b/requirements/requirements.common.txt index aea2b0e0f2..8f40eda57f 100644 --- a/requirements/requirements.common.txt +++ b/requirements/requirements.common.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile --output-file=requirements/requirements.common.txt setup.py # @@ -24,11 +24,6 @@ asgiref==3.7.2 # via django async-lru==2.0.4 # via pulp-ansible -async-timeout==4.0.3 - # via - # aiohttp - # pulpcore - # redis asyncio-throttle==1.0.2 # via pulpcore attrs==22.2.0 @@ -42,11 +37,6 @@ backoff==2.2.1 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # pulpcore -backports-zoneinfo==0.2.1 ; python_version < "3.9" - # via - # django - # galaxy-ng (setup.py) - # psycopg bindep==2.11.0 # via ansible-builder black==24.1.0 @@ -183,13 +173,8 @@ idna==3.6 # yarl importlib-metadata==6.0.1 # via - # markdown # opentelemetry-api # pulpcore -importlib-resources==6.1.1 - # via - # jsonschema - # pulp-glue inflection==0.5.1 # via drf-spectacular insights-analytics-collector==0.3.2 @@ -311,8 +296,6 @@ pbr==6.0.0 # via bindep pillow==10.0.1 # via pulp-ansible -pkgutil-resolve-name==1.3.10 - # via jsonschema platformdirs==4.1.0 # via black prometheus-client==0.19.0 @@ -438,18 +421,12 @@ subprocess-tee==0.4.1 # via ansible-lint tablib[html,ods,xls,xlsx,yaml]==3.5.0 # via django-import-export -tomli==2.0.1 - # via black types-setuptools==69.0.0.20240125 # via requirements-parser typing-extensions==4.9.0 # via - # asgiref - # async-lru - # black # opentelemetry-sdk # psycopg - # rich uritemplate==4.1.1 # via drf-spectacular url-normalize==1.4.3 @@ -481,9 +458,7 @@ yarl==1.9.4 # aiohttp # pulpcore zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/requirements.insights.txt b/requirements/requirements.insights.txt index c38f28ff75..d1f8dafd54 100644 --- a/requirements/requirements.insights.txt +++ b/requirements/requirements.insights.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile --output-file=requirements/requirements.insights.txt requirements/requirements.insights.in setup.py # @@ -26,11 +26,6 @@ asgiref==3.7.2 # via django async-lru==2.0.4 # via pulp-ansible -async-timeout==4.0.3 - # via - # aiohttp - # pulpcore - # redis asyncio-throttle==1.0.2 # via pulpcore attrs==22.2.0 @@ -44,11 +39,6 @@ backoff==2.2.1 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # pulpcore -backports-zoneinfo==0.2.1 ; python_version < "3.9" - # via - # django - # galaxy-ng (setup.py) - # psycopg bindep==2.11.0 # via ansible-builder black==24.1.0 @@ -192,13 +182,8 @@ idna==3.6 # yarl importlib-metadata==6.0.1 # via - # markdown # opentelemetry-api # pulpcore -importlib-resources==6.1.1 - # via - # jsonschema - # pulp-glue inflection==0.5.1 # via drf-spectacular insights-analytics-collector==0.3.2 @@ -322,8 +307,6 @@ pbr==6.0.0 # via bindep pillow==10.0.1 # via pulp-ansible -pkgutil-resolve-name==1.3.10 - # via jsonschema platformdirs==4.1.0 # via black prometheus-client==0.19.0 @@ -449,18 +432,12 @@ subprocess-tee==0.4.1 # via ansible-lint tablib[html,ods,xls,xlsx,yaml]==3.5.0 # via django-import-export -tomli==2.0.1 - # via black types-setuptools==69.0.0.20240125 # via requirements-parser typing-extensions==4.9.0 # via - # asgiref - # async-lru - # black # opentelemetry-sdk # psycopg - # rich uritemplate==4.1.1 # via drf-spectacular url-normalize==1.4.3 @@ -494,9 +471,7 @@ yarl==1.9.4 # aiohttp # pulpcore zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/requirements.standalone.txt b/requirements/requirements.standalone.txt index 4c605ac49a..d2a8440517 100644 --- a/requirements/requirements.standalone.txt +++ b/requirements/requirements.standalone.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # # pip-compile --output-file=requirements/requirements.standalone.txt requirements/requirements.standalone.in setup.py # @@ -24,11 +24,6 @@ asgiref==3.7.2 # via django async-lru==2.0.4 # via pulp-ansible -async-timeout==4.0.3 - # via - # aiohttp - # pulpcore - # redis asyncio-throttle==1.0.2 # via pulpcore attrs==22.2.0 @@ -42,11 +37,6 @@ backoff==2.2.1 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # pulpcore -backports-zoneinfo==0.2.1 ; python_version < "3.9" - # via - # django - # galaxy-ng (setup.py) - # psycopg bindep==2.11.0 # via ansible-builder black==24.1.0 @@ -183,13 +173,8 @@ idna==3.6 # yarl importlib-metadata==6.0.1 # via - # markdown # opentelemetry-api # pulpcore -importlib-resources==6.1.1 - # via - # jsonschema - # pulp-glue inflection==0.5.1 # via drf-spectacular insights-analytics-collector==0.3.2 @@ -311,8 +296,6 @@ pbr==6.0.0 # via bindep pillow==10.0.1 # via pulp-ansible -pkgutil-resolve-name==1.3.10 - # via jsonschema platformdirs==4.1.0 # via black prometheus-client==0.19.0 @@ -438,18 +421,12 @@ subprocess-tee==0.4.1 # via ansible-lint tablib[html,ods,xls,xlsx,yaml]==3.5.0 # via django-import-export -tomli==2.0.1 - # via black types-setuptools==69.0.0.20240125 # via requirements-parser typing-extensions==4.9.0 # via - # asgiref - # async-lru - # black # opentelemetry-sdk # psycopg - # rich uritemplate==4.1.1 # via drf-spectacular url-normalize==1.4.3 @@ -481,9 +458,7 @@ yarl==1.9.4 # aiohttp # pulpcore zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 8166409c92bb3a713ae483fabe33aba8d3964de3 Mon Sep 17 00:00:00 2001 From: Brian McLaughlin Date: Fri, 19 Apr 2024 08:08:22 -0400 Subject: [PATCH 3/5] Update rh-certified url to match https://github.com/ansible/galaxy_ng/commit/891656c3c9c6300ed88c6a4885b544f598500ace No-Issue --- galaxy_ng/tests/unit/api/test_api_ui_collection_viewsets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy_ng/tests/unit/api/test_api_ui_collection_viewsets.py b/galaxy_ng/tests/unit/api/test_api_ui_collection_viewsets.py index ebd6a9efc7..dc2039927b 100644 --- a/galaxy_ng/tests/unit/api/test_api_ui_collection_viewsets.py +++ b/galaxy_ng/tests/unit/api/test_api_ui_collection_viewsets.py @@ -247,7 +247,7 @@ def setUp(self): super().setUp() self.remote_data = { "name": "rh-certified", - "url": "https://console.redhat.com/api/automation-hub/", + "url": "https://console.redhat.com/api/automation-hub/content/published/", } self.remote = CollectionRemote.objects.get(name=self.remote_data["name"]) self.repository = AnsibleRepository.objects.get(name=self.remote_data["name"]) From dd2ef48da791000ed08d9325665d5448f6506224 Mon Sep 17 00:00:00 2001 From: Brian McLaughlin Date: Tue, 30 Apr 2024 09:03:26 -0400 Subject: [PATCH 4/5] Add sleeps and wait_for_all_tasks_gk calls per 2112, 2101 No-Issue --- galaxy_ng/tests/integration/api/test_namespace_management.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/galaxy_ng/tests/integration/api/test_namespace_management.py b/galaxy_ng/tests/integration/api/test_namespace_management.py index ef2933c4e5..b645468b7f 100644 --- a/galaxy_ng/tests/integration/api/test_namespace_management.py +++ b/galaxy_ng/tests/integration/api/test_namespace_management.py @@ -188,6 +188,8 @@ def test_namespace_edit_logo(galaxy_client): } my_namespace = gc.post("_ui/v1/my-namespaces/", body=payload) assert my_namespace["avatar_url"] == '' + sleep(60) + wait_for_all_tasks_gk(gc) namespaces = gc.get('_ui/v1/my-namespaces/') name = my_namespace["name"] @@ -198,6 +200,7 @@ def test_namespace_edit_logo(galaxy_client): "avatar_url": "https://avatars.githubusercontent.com/u/1869705?v=4" } gc.put(f"_ui/v1/my-namespaces/{name}/", body=payload) + sleep(60) wait_for_all_tasks_gk(gc) updated_namespace = gc.get(f'_ui/v1/my-namespaces/{name}/') assert updated_namespace["avatar_url"] != "" From 48f3da7b559bd77d23de63c3cc9777487ec04efc Mon Sep 17 00:00:00 2001 From: James Tanner Date: Tue, 9 Apr 2024 20:32:03 -0400 Subject: [PATCH 5/5] Need to template out the nginx port also. No-Issue Signed-off-by: James Tanner --- profiles/base/Dockerfile | 4 + profiles/base/nginx/nginx.conf.j2 | 146 ++++++++++++++++++++++++++ profiles/base/nginx/template_nginx.py | 39 +++++++ 3 files changed, 189 insertions(+) create mode 100644 profiles/base/nginx/nginx.conf.j2 create mode 100755 profiles/base/nginx/template_nginx.py diff --git a/profiles/base/Dockerfile b/profiles/base/Dockerfile index b854082ac1..133ea7f15c 100644 --- a/profiles/base/Dockerfile +++ b/profiles/base/Dockerfile @@ -6,6 +6,10 @@ WORKDIR /opt/galaxy_ng/ RUN switch_python 3.11 +# override the image's nginx templating scripts ... +RUN cp -p /opt/galaxy_ng/profiles/base/nginx/template_nginx.py /nginx/. +RUN cp /opt/galaxy_ng/profiles/base/nginx/nginx.conf.j2 /nginx/. + # preinstall galaxy_ng in thebase image RUN python3.11 -m pip install . diff --git a/profiles/base/nginx/nginx.conf.j2 b/profiles/base/nginx/nginx.conf.j2 new file mode 100644 index 0000000000..c4570d1a04 --- /dev/null +++ b/profiles/base/nginx/nginx.conf.j2 @@ -0,0 +1,146 @@ +# TODO: Support IPv6. +# TODO: Maybe serve multiple `location`s, not just one. + +# The "nginx" package on fedora creates this user and group. +user nginx nginx; +# Gunicorn docs suggest this value. +worker_processes 1; +daemon off; +events { + worker_connections 1024; # increase if you have lots of clients + accept_mutex off; # set to 'on' if nginx worker_processes > 1 +} + +http { + include mime.types; + # fallback in case we can't determine a type + default_type application/octet-stream; + sendfile on; + + # If left at the default of 1024, nginx emits a warning about being unable + # to build optimal hash types. + types_hash_max_size 4096; + + upstream pulp-content { + server 127.0.0.1:24816; + } + + upstream pulp-api { + server 127.0.0.1:24817; + } + + server { + # Gunicorn docs suggest the use of the "deferred" directive on Linux. + {% if https | default(false) -%} + listen 443 default_server deferred ssl; + + ssl_certificate /etc/pulp/certs/pulp_webserver.crt; + ssl_certificate_key /etc/pulp/certs/pulp_webserver.key; + ssl_session_cache shared:SSL:50m; + ssl_session_timeout 1d; + ssl_session_tickets off; + + # intermediate configuration + ssl_protocols TLSv1.2; + ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + ssl_prefer_server_ciphers on; + + # HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months) + add_header Strict-Transport-Security max-age=15768000; + {%- else -%} + listen {{ NGINX_PORT }} default_server deferred; + {%- endif %} + server_name $hostname; + + # The default client_max_body_size is 1m. Clients uploading + # files larger than this will need to chunk said files. + client_max_body_size 10m; + + # Gunicorn docs suggest this value. + keepalive_timeout 5; + + #location {{ content_path }} { + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # proxy_set_header Host $http_host; + # # we don't want nginx trying to do something clever with + # # redirects, we set the Host: header above already. + # proxy_redirect off; + # proxy_pass http://pulp-content; + #} + + location /pulp/content/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-content; + } + + location {{ api_root }}api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + client_max_body_size 0; + } + + {%- if domain_enabled | default(false) %} + location ~ {{ api_root }}.+/api/v3/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + client_max_body_size 0; + } + {%- endif %} + + location /auth/login/ { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + } + + include pulp/*.conf; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://pulp-api; + # most pulp static files are served through whitenoise + # http://whitenoise.evans.io/en/stable/ + } + + {%- if https | default(false) %} + # ACME http-01 tokens, i.e, for Let's Encrypt + location /.well-known/ { + try_files $uri $uri/ =404; + } + {%- endif %} + } + {%- if https | default(false) %} + server { + listen 55001 default_server; + server_name _; + return 301 https://$host$request_uri; + } + {%- endif %} +} + + diff --git a/profiles/base/nginx/template_nginx.py b/profiles/base/nginx/template_nginx.py new file mode 100755 index 0000000000..ca90bb39a2 --- /dev/null +++ b/profiles/base/nginx/template_nginx.py @@ -0,0 +1,39 @@ +import argparse +import os +import django +from django.core.exceptions import AppRegistryNotReady, ImproperlyConfigured + +from jinja2 import Template + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Create Pulp's nginx conf file based on current settings.", + ) + parser.add_argument("template_file", type=open) + parser.add_argument("output_file", type=argparse.FileType("w")) + args = parser.parse_args() + + https = os.getenv("PULP_HTTPS", "false") + values = { + "https": https.lower() == "true", + "api_root": "/pulp/", + "content_path": "/pulp/content/", + "domain_enabled": False, + } + + try: + django.setup() + from django.conf import settings + except (AppRegistryNotReady, ImproperlyConfigured): + print("Failed to find settings for nginx template, using defaults") + else: + values["api_root"] = settings.API_ROOT + values["content_path"] = settings.CONTENT_PATH_PREFIX + values["domain_enabled"] = getattr(settings, "DOMAIN_ENABLED", False) + + values['NGINX_PORT'] = os.environ.get('NGINX_PORT', '55001') + + template = Template(args.template_file.read()) + output = template.render(**values) + args.output_file.write(output)