Skip to content

Commit

Permalink
Issue/603 forecast client 2 (#607)
Browse files Browse the repository at this point in the history
* add secret for huggingface #604

* add TODO

* add ecs task for forecast ad

* refactor ecs-task, so we can pass in mulite secrets

* upgrade ecs task, multie secrets, update india development

* rename to container-secretsmanager_secrets

* rename again

* small fixes

* upgrade uk

* Revert "refactor ecs-task, so we can pass in mulite secrets"

This reverts commit a1f67f5.

* update forecast ad

* PR comments

* fix container-secret_vars

* tidy up documentation

* update

* tidy

* fix

* fix

* use `module.`

* add USE_SATELLITE

* fix

* PR comments
  • Loading branch information
peterdudfield committed Sep 9, 2024
1 parent 6a29dde commit 70e88cb
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 3 deletions.
63 changes: 60 additions & 3 deletions terraform/india/development/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
# 4.1 - ECS task definition for the GFS consumer
# 4.2 - ECS task definition for Collection RUVNL data
# 4.3 - Satellite Consumer
# 4.4 - ECS task definition for the Forecast
# 4.4 - ECS task definition for the Forecast - Client RU
# 4.5 - ECS task definition for the Forecast - Client AD
# 5.0 - Airflow EB Instance
# 5.1 - India API EB Instance
# 5.2 - India Analysis Dashboard
Expand Down Expand Up @@ -95,7 +96,7 @@ resource "aws_secretsmanager_secret" "satellite_consumer_secret" {
name = "${local.environment}/data/satellite-consumer"
}

# 3.1
# 3.2
resource "aws_secretsmanager_secret" "huggingface_consumer_secret" {
name = "${local.environment}/huggingface/token"
}
Expand Down Expand Up @@ -268,7 +269,7 @@ module "satellite_consumer_ecs" {



# 4.4 - Forecast
# 4.4 - Forecast - Client RU
module "forecast" {
source = "../../modules/services/forecast_generic"

Expand Down Expand Up @@ -307,6 +308,62 @@ module "forecast" {
sentry_dsn= var.sentry_dsn
}


# 4.5 - Forecast - Client AD
module "forecast-ad" {
source = "../../modules/services/ecs_task"

aws-region = var.region
aws-environment = local.environment

s3-buckets = [
{
id : module.s3-satellite-bucket.bucket_id,
access_policy_arn : module.s3-satellite-bucket.write_policy_arn
},
{
id : module.s3-nwp-bucket.bucket_id,
access_policy_arn : module.s3-nwp-bucket.write_policy_arn
}
]

ecs-task_name = "client-ad"
ecs-task_type = "forecast"
ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
ecs-task_size = {
memory = 3072
cpu = 1024
storage = 21
}

container-env_vars = [
{ "name" : "AWS_REGION", "value" : var.region },
{ "name" : "ENVIRONMENT", "value" : local.environment },
{ "name" : "LOGLEVEL", "value" : "DEBUG" },
{ "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/ecmwf/data/latest.zarr" },
{ "name" : "NWP_GFS_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/gfs/data/latest.zarr" },
{ "name" : "SATELLITE_ZARR_PATH", "value": "s3://${module.s3-satellite-bucket.bucket_id}/data/latest/iodc_latest.zarr.zip" },
{ "name" : "SENTRY_DSN", "value": var.sentry_dsn},
{ "name" : "USE_SATELLITE", "value": "True"},
{ "name" : "CLIENT_NAME", "value": "ad"}
]

container-secret_vars = [
{secret_policy_arn: aws_secretsmanager_secret.huggingface_consumer_secret.arn,
values: ["HUGGINGFACE_TOKEN"]
},
{secret_policy_arn: module.postgres-rds.secret.arn,
values: ["DB_URL"]
}
]

container-tag = var.version-forecast-ad
container-name = "india_forecast_app"
container-registry = "openclimatefix"
container-command = []
}


# 5.0
module "airflow" {
source = "../../modules/services/airflow"
Expand Down
6 changes: 6 additions & 0 deletions terraform/india/development/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ variable version-forecast {
description = "Container image tag of the forecast to use: openclimatefix/forecast"
}

variable version-forecast-ad {
type = string
default = "0.1.0"
description = "Container image tag of the forecast ad to use: openclimatefix/forecast"
}

variable version-runvl-consumer {
type = string
default = "0.0.1"
Expand Down

0 comments on commit 70e88cb

Please sign in to comment.