Database summary statistics export

server/src/routes/applications/data-export/anonymous-csv-transform.ts

@@ -0,0 +1,46 @@
+import type { DisciplineOfStudy } from "@durhack/durhack-common/input/discipline-of-study"
+import type { DietaryRequirement } from "@durhack/durhack-common/types/application"
+import type { UserInfo } from "@/database"
+import type { IdAugments } from "@/routes/applications/data-export/anonymous-id-augmenting-transform"
+import type { AttendanceAugments } from "@/routes/applications/data-export/attendance-augmenting-transform"
+import type { ConsentAugments } from "@/routes/applications/data-export/consent-augmenting-transform"
+import { PickAttributesToCsvTransform } from "@/routes/applications/data-export/pick-attributes-to-csv-transform"
+import type { AgeAugments } from "@/routes/applications/data-export/user-age-augmenting-transform"
+import type { UserCvAugments } from "@/routes/applications/data-export/user-cv-augmenting-transform"
+import type { FlagAugments } from "@/routes/applications/data-export/user-flag-augmenting-transform"
+
+type Source = UserInfo &
+  IdAugments &
+  AgeAugments &
+  AttendanceAugments &
+  ConsentAugments<"media" | "dsuPrivacy"> &
+  UserCvAugments &
+  FlagAugments<"discipline-of-study", DisciplineOfStudy> &
+  FlagAugments<"dietary-requirement", DietaryRequirement>
+
+export class AnonymousCsvTransform extends PickAttributesToCsvTransform<Source> {
+  constructor() {
+    super({
+      attributes: [
+        { name: "id", label: "anon_id" },
+        { name: "countryOfResidence", label: "country_of_residence" },
+        { name: "university", label: "university" },
+        { name: "levelOfStudy", label: "level_of_study" },
+        { name: "tShirtSize", label: "t_shirt_size" },
+        { name: "gender", label: "gender" },
+        { name: "ethnicity", label: "ethnicity" },
+        { name: "hackathonExperience", label: "hackathon_experience" },
+        { name: "dietary-requirement", label: "dietary_requirements" },
+        { name: "discipline-of-study", label: "discipline_of_study" },
+        { name: "ageGroup", label: "age_group" },
+        { name: "dsuPrivacy", label: "dsu_privacy" },
+        { name: "media", label: "photo_media_consent" },
+        { name: "is_cv_uploaded", label: "did_upload_cv" },
+        { name: "cv_update_time", label: "cv_last_updated_at" },
+        { name: "applicationSubmittedAt", label: "application_submitted_at" },
+        { name: "applicationStatus", label: "application_status" },
+        { name: "isCheckedIn", label: "attendance" },
+      ],
+    })
+  }
+}

server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts

@@ -0,0 +1,40 @@
+import stream from "node:stream"
+import { isString } from "@/lib/type-guards"
+
+export type IdAugments = { id: number }
+
+type IdAugmentedUserInfo = IdAugments & { userId: string }
+
+export class AnonymousIdAugmentingTransform extends stream.Transform {
+  counter: number
+
+  constructor() {
+    super({
+      writableObjectMode: true,
+      readableObjectMode: true,
+    })
+
+    this.counter = 0
+  }
+
+  augmentUserInfo(userInfo: { userId: string; id?: number }): IdAugmentedUserInfo {
+    userInfo.id = this.counter++
+    return userInfo as IdAugmentedUserInfo
+  }
+
+  async augmentChunk(chunk: { userId: string }[]): Promise<IdAugmentedUserInfo[]> {
+    return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo)))
+  }
+
+  _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void {
+    this.augmentChunk(chunk)
+      .then((filteredChunk) => {
+        callback(null, filteredChunk satisfies IdAugmentedUserInfo[])
+      })
+      .catch((error: unknown) => {
+        if (error instanceof Error) return callback(error)
+        if (isString(error)) return callback(new Error(error))
+        return callback(new Error(`Something really strange happened. Error object: ${error}`))
+      })
+  }
+}

server/src/routes/applications/data-export/data-export-handlers.ts

@@ -13,10 +13,15 @@ import { Group, onlyGroups } from "@/decorators/authorise"
 import { KeycloakAugmentingTransform } from "@/lib/keycloak-augmenting-transform"
 import { getTempDir } from "@/lib/temp-dir"
 import { hasCode } from "@/lib/type-guards"
+import { AnonymousCsvTransform } from "@/routes/applications/data-export/anonymous-csv-transform"
+import { AnonymousIdAugmentingTransform } from "@/routes/applications/data-export/anonymous-id-augmenting-transform"
 import {
   ConsentAugmentingTransform,
   type ConsentAugments,
 } from "@/routes/applications/data-export/consent-augmenting-transform"
+import { UserAgeAugmentingTransform } from "@/routes/applications/data-export/user-age-augmenting-transform"
+import { UserCvAugmentingTransform } from "@/routes/applications/data-export/user-cv-augmenting-transform"
+import { UserFlagAugmentingTransform } from "@/routes/applications/data-export/user-flag-augmenting-transform"
 import type { Middleware } from "@/types"
 import { AttendanceAugmentingTransform, type AttendanceAugments } from "./attendance-augmenting-transform"
 import { CvExportingWritable } from "./cv-exporting-writable"
@@ -36,6 +41,7 @@ class DataExportHandlers {
         major_league_hacking_attendees_url: new URL("/applications/data-export/major-league-hacking?attendees", origin),
         hackathons_uk_applications_url: new URL("/applications/data-export/hackathons-uk", origin),
         hackathons_uk_attendees_url: new URL("/applications/data-export/hackathons-uk?attendees", origin),
+        anonymous_applications_url: new URL("/applications/data-export/anonymous", origin),
       })
     }
   }
@@ -159,6 +165,34 @@ class DataExportHandlers {
       }
     }
   }
+
+  @onlyGroups([Group.organisers, Group.admins])
+  getAnonymous(): Middleware {
+    return async (_request, response) => {
+      const tempDir = await getTempDir()
+      try {
+        const fileName = "anonymous-data-export.csv"
+        const fileDestination = pathJoin(tempDir, fileName)
+
+        await pipeline(
+          Readable.from(generateUserInfo()),
+          new AttendanceAugmentingTransform(),
+          new ConsentAugmentingTransform({ media: true, dsuPrivacy: true }),
+          new UserAgeAugmentingTransform(),
+          new UserFlagAugmentingTransform("dietary-requirement"),
+          new UserFlagAugmentingTransform("discipline-of-study"),
+          new UserCvAugmentingTransform(),
+          new AnonymousIdAugmentingTransform(),
+          new AnonymousCsvTransform(),
+          createWriteStream(fileDestination),
+        )
+
+        await response.download(fileDestination, fileName)
+      } finally {
+        await rm(tempDir, { recursive: true, force: true })
+      }
+    }
+  }
 }
 
 const dataExportHandlers = new DataExportHandlers()

server/src/routes/applications/data-export/index.ts

@@ -36,3 +36,10 @@ applicationsDataExportApp
   .all(authenticate())
   .get(dataExportHandlers.getCVArchive())
   .all(forbiddenOrUnauthorised())
+
+applicationsDataExportApp
+  .route("/anonymous")
+  .all(methodNotAllowed(["GET"]))
+  .all(authenticate())
+  .get(dataExportHandlers.getAnonymous())
+  .all(forbiddenOrUnauthorised())

server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts

@@ -23,6 +23,9 @@ export class PickAttributesToCsvTransform<Source extends Record<string, unknown>
   pickAttributesToCsv(item: Source): string {
     const values: string[] = []
     for (const attribute of this.attributes) {
+      function fail() {
+        throw new Error(`Unsupported value ${attributeValue} for ${String(attribute.name)}`)
+      }
       const attributeValue = item[attribute.name]
 
       if (attributeValue == null) {
@@ -45,7 +48,20 @@ export class PickAttributesToCsvTransform<Source extends Record<string, unknown>
         continue
       }
 
-      throw new Error(`Unsupported value ${attributeValue} for ${String(attribute.name)}`)
+      if (typeof attributeValue !== "object") fail()
+
+      if (Array.isArray(attributeValue) && attributeValue.every(isString)) {
+        values.push(JSON.stringify(attributeValue))
+        continue
+      }
+
+      if (attributeValue instanceof Date) {
+        const epochSeconds = Math.floor(attributeValue.getTime() / 1000)
+        values.push(String(epochSeconds))
+        continue
+      }
+
+      fail()
     }
 
     return values.join(";")

server/src/routes/applications/data-export/user-age-augmenting-transform.ts

@@ -0,0 +1,68 @@
+import assert from "node:assert/strict"
+import stream from "node:stream"
+import type { UserInfo } from "@/database"
+import { isString } from "@/lib/type-guards"
+
+type AgeGroup = "<18" | "18-21" | "22-25" | "26-29" | "30-39" | "40-59" | "60+"
+
+export type AgeAugments = {
+  ageGroup: AgeGroup
+}
+
+type AgeAugmentedUserInfo = { userId: string } & AgeAugments
+
+export class UserAgeAugmentingTransform extends stream.Transform {
+  constructor() {
+    super({
+      writableObjectMode: true,
+      readableObjectMode: true,
+    })
+  }
+
+  resolveAgeGroup(age: number): AgeGroup {
+    if (age < 18) {
+      return "<18"
+    }
+    if (age <= 21) {
+      return "18-21"
+    }
+    if (age <= 25) {
+      return "22-25"
+    }
+    if (age <= 29) {
+      return "26-29"
+    }
+    if (age <= 39) {
+      return "30-39"
+    }
+    if (age <= 59) {
+      return "40-59"
+    }
+    return "60+"
+  }
+
+  augmentUserInfo(userInfo: UserInfo): AgeAugmentedUserInfo {
+    assert(userInfo.applicationSubmittedAt)
+    assert(userInfo.age)
+
+    const ageGroup: AgeGroup = this.resolveAgeGroup(userInfo.age)
+
+    return { ...userInfo, ageGroup }
+  }
+
+  async augmentChunk(chunk: UserInfo[]): Promise<AgeAugmentedUserInfo[]> {
+    return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo)))
+  }
+
+  _transform(chunk: UserInfo[], _encoding: never, callback: stream.TransformCallback): void {
+    this.augmentChunk(chunk)
+      .then((filteredChunk) => {
+        callback(null, filteredChunk satisfies AgeAugmentedUserInfo[])
+      })
+      .catch((error: unknown) => {
+        if (error instanceof Error) return callback(error)
+        if (isString(error)) return callback(new Error(error))
+        return callback(new Error(`Something really strange happened. Error object: ${error}`))
+      })
+  }
+}

server/src/routes/applications/data-export/user-cv-augmenting-transform.ts

@@ -0,0 +1,46 @@
+import stream from "node:stream"
+import { prisma, type UserInfo } from "@/database"
+
+import { isString } from "@/lib/type-guards"
+
+export type UserCvAugments = {
+  is_cv_uploaded: boolean
+  cv_update_time: Date | null
+}
+
+type CvAugmentedUserInfo = { userId: string } & UserCvAugments
+
+export class UserCvAugmentingTransform extends stream.Transform {
+  constructor() {
+    super({ writableObjectMode: true, readableObjectMode: true })
+  }
+
+  async augmentUserInfo(userInfo: UserInfo): Promise<CvAugmentedUserInfo> {
+    const foundCV = await prisma.userCV.findUnique({
+      where: {
+        userId: userInfo.userId,
+      },
+      select: {
+        updatedAt: true,
+      },
+    })
+
+    return { ...userInfo, is_cv_uploaded: !!foundCV, cv_update_time: foundCV ? foundCV.updatedAt : null }
+  }
+
+  async augmentChunk(chunk: UserInfo[]): Promise<CvAugmentedUserInfo[]> {
+    return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo)))
+  }
+
+  _transform(chunk: UserInfo[], _encoding: never, callback: stream.TransformCallback): void {
+    this.augmentChunk(chunk)
+      .then((filteredChunk) => {
+        callback(null, filteredChunk satisfies CvAugmentedUserInfo[])
+      })
+      .catch((error: unknown) => {
+        if (error instanceof Error) return callback(error)
+        if (isString(error)) return callback(new Error(error))
+        return callback(new Error(`Something really strange happened. Error object: ${error}`))
+      })
+  }
+}

server/src/routes/applications/data-export/user-flag-augmenting-transform.ts

@@ -0,0 +1,58 @@
+import stream from "node:stream"
+
+import { prisma } from "@/database"
+import { isString } from "@/lib/type-guards"
+
+export type FlagAugments<Namespace extends string, Member extends string> = { [K in Namespace]: Member[] }
+
+type FlagAugmentedUserInfo<Namespace extends string, Member extends string> = FlagAugments<Namespace, Member> & {
+  userId: string
+}
+
+export class UserFlagAugmentingTransform<Namespace extends string, Member extends string> extends stream.Transform {
+  namespace: Namespace
+
+  constructor(namespace: Namespace) {
+    super({
+      writableObjectMode: true,
+      readableObjectMode: true,
+    })
+
+    this.namespace = namespace
+  }
+
+  async augmentUserInfo(userInfo: { userId: string }): Promise<FlagAugmentedUserInfo<Namespace, Member>> {
+    const flags = await prisma.userFlag.findMany({
+      where: { userId: userInfo.userId, flagName: { startsWith: `${this.namespace}:` } },
+    })
+
+    const members: Member[] = []
+
+    for (const flag of flags) {
+      const flagName = flag.flagName.substring(this.namespace.length + 1) as Member
+      members.push(flagName)
+    }
+
+    const result = { [this.namespace as Namespace]: members } satisfies FlagAugments<string, Member> as FlagAugments<
+      Namespace,
+      Member
+    >
+    return { ...userInfo, ...result }
+  }
+
+  async augmentChunk(chunk: { userId: string }[]): Promise<FlagAugmentedUserInfo<Namespace, Member>[]> {
+    return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo)))
+  }
+
+  _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void {
+    this.augmentChunk(chunk)
+      .then((filteredChunk) => {
+        callback(null, filteredChunk satisfies FlagAugmentedUserInfo<Namespace, Member>[])
+      })
+      .catch((error: unknown) => {
+        if (error instanceof Error) return callback(error)
+        if (isString(error)) return callback(new Error(error))
+        return callback(new Error(`Something really strange happened. Error object: ${error}`))
+      })
+  }
+}