Allow Mod upload via S3

build.gradle

@@ -184,6 +184,7 @@ dependencies {
   implementation("org.jsoup:jsoup:1.21.1")
   implementation("com.github.jasminb:jsonapi-converter:0.14")
   implementation("commons-codec:commons-codec:1.18.0")
+  implementation("software.amazon.awssdk:s3:2.31.68")
 
   // Required library for FafTokenService approach (called by nimbus-jwt)
   runtimeOnly("org.bouncycastle:bcpkix-jdk15on:1.70")

compose.yaml

@@ -0,0 +1,60 @@
+services:
+  # Set up the faf db
+  faf-db:
+    image: mariadb:11
+    environment:
+      MARIADB_DATABASE: faf
+      MARIADB_USER: faf-api
+      MARIADB_PASSWORD: banana
+      MARIADB_ROOT_PASSWORD: banana
+    healthcheck:
+      test: [ "CMD", "healthcheck.sh", "--connect", "--innodb_initialized" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "3306:3306"
+
+  faf-db-migrations:
+    image: faforever/faf-db-migrations:v136
+    command: migrate
+    environment:
+      FLYWAY_URL: jdbc:mysql://faf-db/faf?useSSL=false
+      FLYWAY_USER: root
+      FLYWAY_PASSWORD: banana
+      FLYWAY_DATABASE: faf
+    depends_on:
+      faf-db:
+        condition: service_healthy
+
+  faf-db-testdata:
+    image: mariadb:11
+    entrypoint: sh -c "apt-get update && apt-get install -y curl && curl -s https://github.com/FAForever/db/refs/heads/develop/test-data.sql | mariadb -h faf-db -uroot -pbanana -D faf"
+    depends_on:
+      faf-db-migrations:
+        condition: service_completed_successfully
+
+  minio:
+    image: docker.io/bitnami/minio:2025
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    environment:
+      MINIO_ROOT_USER: admin
+      MINIO_ROOT_PASSWORD: banana123
+      MINIO_DEFAULT_BUCKETS: user-uploads
+      MINIO_SCHEME: http
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/ready" ]
+      interval: 2s
+      timeout: 2s
+      retries: 15
+
+  rabbitmq:
+    image: rabbitmq:3.13-management
+    environment:
+      RABBITMQ_DEFAULT_VHOST: /faf-core
+      RABBITMQ_DEFAULT_USER: faf-api
+      RABBITMQ_DEFAULT_PASS: banana
+    ports:
+      - "5672:5672"

src/main/java/com/faforever/api/config/FafApiProperties.java

@@ -42,6 +42,7 @@ public class FafApiProperties {
   private Recaptcha recaptcha = new Recaptcha();
   private Monitoring monitoring = new Monitoring();
   private Coturn coturn = new Coturn();
+  private S3 s3 = new S3();
 
   @Data
   public static class OAuth2 {
@@ -298,4 +299,12 @@ public static class Monitoring {
   public static class Coturn {
     private int tokenLifetimeSeconds = 86400;
   }
+
+  @Data
+  public static class S3 {
+    private String endpoint;
+    private String userUploadBucket;
+    private String accessKey;
+    private String secretKey;
+  }
 }

src/main/java/com/faforever/api/config/S3Config.java

@@ -0,0 +1,48 @@
+package com.faforever.api.config;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+
+import java.net.URI;
+
+@Configuration
+@RequiredArgsConstructor
+public class S3Config {
+
+  private final FafApiProperties properties;
+
+  @Bean
+  public S3Client s3Client() {
+    return S3Client.builder()
+      .endpointOverride(URI.create(properties.getS3().getEndpoint()))
+      .region(Region.EU_CENTRAL_1) // region must be non-null but is ignored by some S3-compatible services
+      .credentialsProvider(StaticCredentialsProvider.create(
+        AwsBasicCredentials.create(properties.getS3().getAccessKey(), properties.getS3().getSecretKey())
+      ))
+      .serviceConfiguration(S3Configuration.builder()
+        .pathStyleAccessEnabled(true) // prevents putting the bucket name as subdomain
+        .build())
+      .build();
+  }
+
+  @Bean
+  public S3Presigner s3Presigner() {
+    return S3Presigner.builder()
+      .endpointOverride(URI.create(properties.getS3().getEndpoint()))
+      .region(Region.EU_CENTRAL_1) // region must be non-null but is ignored by some S3-compatible services
+      .credentialsProvider(StaticCredentialsProvider.create(
+        AwsBasicCredentials.create(properties.getS3().getAccessKey(), properties.getS3().getSecretKey())
+      ))
+      .serviceConfiguration(S3Configuration.builder()
+        .pathStyleAccessEnabled(true) // prevents putting the bucket name as subdomain
+        .build())
+      .build();
+  }
+}

src/main/java/com/faforever/api/mod/ModService.java

@@ -31,6 +31,14 @@
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.client.HttpClientErrorException;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.presigner.S3Presigner;
+import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest;
+import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
@@ -40,11 +48,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
@@ -67,6 +77,55 @@ public class ModService {
   private final ModRepository modRepository;
   private final ModVersionRepository modVersionRepository;
   private final LicenseRepository licenseRepository;
+  private final S3Client s3Client;
+  private final S3Presigner s3Presigner;
+
+  private String getBucketKey(int userId, UUID requestId) {
+    return "%s-mod-%s".formatted(userId, requestId);
+  }
+
+  public String getPresignedS3Url(Player uploader, UUID requestId) {
+    log.info("User {} requested presigned url for mod upload, request id {}", uploader.getId(), requestId);
+
+    checkUploaderVaultBan(uploader);
+
+    PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+      .bucket(properties.getS3().getUserUploadBucket())
+      .key(getBucketKey(uploader.getId(), requestId))
+      .build();
+
+    PutObjectPresignRequest putObjectPresignRequest = PutObjectPresignRequest.builder()
+      .signatureDuration(Duration.ofHours(1))
+      .putObjectRequest(putObjectRequest)
+      .build();
+
+    PresignedPutObjectRequest presignedRequest = s3Presigner.presignPutObject(putObjectPresignRequest);
+
+    return presignedRequest.url().toString();
+  }
+
+  public Path getModFromS3Location(Player uploader, UUID requestId) throws IOException {
+    Path tempDir = Files.createTempDirectory("mod-download");
+    String bucketKey = getBucketKey(uploader.getId(), requestId);
+    Path tempFile = tempDir.resolve(bucketKey + ".zip");
+
+    checkUploaderVaultBan(uploader);
+
+    GetObjectRequest request = GetObjectRequest.builder()
+      .bucket(properties.getS3().getUserUploadBucket())
+      .key(bucketKey)
+      .build();
+
+    s3Client.getObject(request, ResponseTransformer.toFile(tempFile));
+
+    return tempFile;
+  }
+
+  public void deleteModFromS3Location(Player uploader, UUID requestId) throws IOException {
+    String bucketKey = getBucketKey(uploader.getId(), requestId);
+
+    s3Client.deleteObject(DeleteObjectRequest.builder().bucket(properties.getS3().getUserUploadBucket()).key(bucketKey).build());
+  }
 
   @SneakyThrows
   @Transactional
@@ -248,7 +307,7 @@ private void validateModInfo(com.faforever.commons.mod.Mod modInfo) {
       final Integer versionInt = Ints.tryParse(modVersion.toString());
       if (versionInt == null) {
         errors.add(new Error(ErrorCode.MOD_VERSION_NOT_A_NUMBER, modVersion.toString()));
-      } else if (!isModVersionValidRange(versionInt)){
+      } else if (!isModVersionValidRange(versionInt)) {
         errors.add(new Error(ErrorCode.MOD_VERSION_INVALID_RANGE, MOD_VERSION_MIN_VALUE, MOD_VERSION_MAX_VALUE));
       }
     }

src/main/java/com/faforever/api/mod/ModUploadMetadata.java

@@ -1,4 +1,10 @@
 package com.faforever.api.mod;
 
-public record ModUploadMetadata(Integer licenseId, String repositoryUrl) {
+import java.util.UUID;
+
+public record ModUploadMetadata(
+  UUID requestId,
+  Integer licenseId,
+  String repositoryUrl
+) {
 }

src/main/java/com/faforever/api/mod/ModsController.java

@@ -1,41 +1,78 @@
 package com.faforever.api.mod;
 
-import com.faforever.api.config.FafApiProperties;
+import com.faforever.api.data.domain.Player;
 import com.faforever.api.player.PlayerService;
 import com.faforever.api.security.OAuthScope;
 import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.security.core.Authentication;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.UUID;
 
-import static org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE;
+import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
 
 @RestController
 @RequestMapping(path = "/mods")
+@Slf4j
 @RequiredArgsConstructor
 public class ModsController {
 
   private final PlayerService playerService;
   private final ModService modService;
-  private final FafApiProperties fafApiProperties;
-
-  @Operation(summary = "Upload a mod")
-  @ApiResponses(value = {
-    @ApiResponse(responseCode = "200", description = "Success"),
-    @ApiResponse(responseCode = "401", description = "Unauthorized"),
-    @ApiResponse(responseCode = "500", description = "Failure")})
-  @RequestMapping(path = "/upload", method = RequestMethod.POST, produces = APPLICATION_JSON_UTF8_VALUE)
+
+  public record UploadUrlResponse(String uploadUrl, UUID requestId) {
+  }
+
+  @Operation(summary = "Begin process of uploading a mod (as a zip file)")
+  @GetMapping(path = "/upload/start", produces = APPLICATION_JSON_VALUE)
+  @PreAuthorize("hasScope('" + OAuthScope._UPLOAD_MOD + "')")
+  public UploadUrlResponse startUpload(Authentication authentication) {
+    UUID requestId = UUID.randomUUID();
+    String presignedUrl = modService.getPresignedS3Url(playerService.getPlayer(authentication), requestId);
+
+    return new UploadUrlResponse(presignedUrl, requestId);
+  }
+
+  @Operation(summary = "Notify about mod upload completion")
+  @PostMapping(path = "/upload/complete", produces = APPLICATION_JSON_VALUE)
+  @PreAuthorize("hasScope('" + OAuthScope._UPLOAD_MOD + "')")
+  public void completeUpload(@RequestBody ModUploadMetadata metadata,
+                             Authentication authentication) throws IOException {
+    Player uploader = playerService.getPlayer(authentication);
+
+    log.info("User {} reported completed mod upload, request id {}", uploader.getId(), metadata.requestId());
+    Path tempFile = modService.getModFromS3Location(uploader, metadata.requestId());
+
+    try {
+      log.debug("Process uploaded file @ {}", tempFile);
+      modService.processUploadedMod(
+        tempFile,
+        tempFile.getFileName().toString(),
+        playerService.getPlayer(authentication),
+        metadata.licenseId(),
+        metadata.repositoryUrl()
+      );
+    } finally {
+      log.debug("Delete uploaded file for request id {}", metadata.requestId());
+      modService.deleteModFromS3Location(uploader, metadata.requestId());
+    }
+  }
+
+  @Deprecated
+  @Operation(summary = "Upload a mod (as a zip file)")
+  @PostMapping(path = "/upload", produces = APPLICATION_JSON_VALUE)
   @PreAuthorize("hasScope('" + OAuthScope._UPLOAD_MOD + "')")
   public void uploadMod(
     @RequestParam("file") MultipartFile file,

src/main/resources/config/application-local.yml

@@ -56,12 +56,17 @@ faf-api:
   recaptcha:
     enabled: ${RECAPTCHA_ENABLED:false}
     secret: ${RECAPTCHA_SECRET}
+  s3:
+    endpoint: ${S3_ENDPOINT:http://localhost:9000}
+    access-key: ${S3_ACCESS_KEY:admin}
+    secret-key: ${S3_SECRET_KEY:banana123}
+    user-upload-bucket: ${S3_USER_UPLOADS_BUCKET:user-uploads}
 
 spring:
   datasource:
     url: jdbc:mariadb://${DATABASE_ADDRESS:127.0.0.1}/${DATABASE_NAME:faf}?useSSL=false
     name: faf
-    username: ${DATABASE_USERNAME:faf-java-api}
+    username: ${DATABASE_USERNAME:faf-api}
     password: ${DATABASE_PASSWORD:banana}
   league-datasource:
     url: jdbc:mariadb://${LEAGUE_DATABASE_ADDRESS:127.0.0.1}/${LEAGUE_DATABASE_NAME:faf-league}?useSSL=false
@@ -71,7 +76,7 @@ spring:
   rabbitmq:
     host: ${RABBIT_HOST:127.0.0.1}
     port: ${RABBIT_PORT:5672}
-    username: ${RABBIT_USERNAME:faf-java-api}
+    username: ${RABBIT_USERNAME:faf-api}
     password: ${RABBIT_PASSWORD:banana}
     virtual-host: ${RABBIT_VHOST:/faf-core}
   jpa:

src/main/resources/config/application.yml

@@ -128,7 +128,7 @@ spring:
     serialization:
       WRITE_DATES_AS_TIMESTAMPS: false
   profiles:
-    active: ${API_PROFILE}
+    active: ${API_PROFILE:prod}
   servlet:
     multipart:
       max-file-size: 350MB