diff --git a/server/src/routes/applications/data-export/anonymous-csv-transform.ts b/server/src/routes/applications/data-export/anonymous-csv-transform.ts new file mode 100644 index 00000000..937ec6f8 --- /dev/null +++ b/server/src/routes/applications/data-export/anonymous-csv-transform.ts @@ -0,0 +1,46 @@ +import type { DisciplineOfStudy } from "@durhack/durhack-common/input/discipline-of-study" +import type { DietaryRequirement } from "@durhack/durhack-common/types/application" +import type { UserInfo } from "@/database" +import type { IdAugments } from "@/routes/applications/data-export/anonymous-id-augmenting-transform" +import type { AttendanceAugments } from "@/routes/applications/data-export/attendance-augmenting-transform" +import type { ConsentAugments } from "@/routes/applications/data-export/consent-augmenting-transform" +import { PickAttributesToCsvTransform } from "@/routes/applications/data-export/pick-attributes-to-csv-transform" +import type { AgeAugments } from "@/routes/applications/data-export/user-age-augmenting-transform" +import type { UserCvAugments } from "@/routes/applications/data-export/user-cv-augmenting-transform" +import type { FlagAugments } from "@/routes/applications/data-export/user-flag-augmenting-transform" + +type Source = UserInfo & + IdAugments & + AgeAugments & + AttendanceAugments & + ConsentAugments<"media" | "dsuPrivacy"> & + UserCvAugments & + FlagAugments<"discipline-of-study", DisciplineOfStudy> & + FlagAugments<"dietary-requirement", DietaryRequirement> + +export class AnonymousCsvTransform extends PickAttributesToCsvTransform { + constructor() { + super({ + attributes: [ + { name: "id", label: "anon_id" }, + { name: "countryOfResidence", label: "country_of_residence" }, + { name: "university", label: "university" }, + { name: "levelOfStudy", label: "level_of_study" }, + { name: "tShirtSize", label: "t_shirt_size" }, + { name: "gender", label: "gender" }, + { name: "ethnicity", label: "ethnicity" }, + { name: "hackathonExperience", label: "hackathon_experience" }, + { name: "dietary-requirement", label: "dietary_requirements" }, + { name: "discipline-of-study", label: "discipline_of_study" }, + { name: "ageGroup", label: "age_group" }, + { name: "dsuPrivacy", label: "dsu_privacy" }, + { name: "media", label: "photo_media_consent" }, + { name: "is_cv_uploaded", label: "did_upload_cv" }, + { name: "cv_update_time", label: "cv_last_updated_at" }, + { name: "applicationSubmittedAt", label: "application_submitted_at" }, + { name: "applicationStatus", label: "application_status" }, + { name: "isCheckedIn", label: "attendance" }, + ], + }) + } +} diff --git a/server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts b/server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts new file mode 100644 index 00000000..55129616 --- /dev/null +++ b/server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts @@ -0,0 +1,40 @@ +import stream from "node:stream" +import { isString } from "@/lib/type-guards" + +export type IdAugments = { id: number } + +type IdAugmentedUserInfo = IdAugments & { userId: string } + +export class AnonymousIdAugmentingTransform extends stream.Transform { + counter: number + + constructor() { + super({ + writableObjectMode: true, + readableObjectMode: true, + }) + + this.counter = 0 + } + + augmentUserInfo(userInfo: { userId: string; id?: number }): IdAugmentedUserInfo { + userInfo.id = this.counter++ + return userInfo as IdAugmentedUserInfo + } + + async augmentChunk(chunk: { userId: string }[]): Promise { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies IdAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +} diff --git a/server/src/routes/applications/data-export/data-export-handlers.ts b/server/src/routes/applications/data-export/data-export-handlers.ts index 8e3d87b2..06ebc19c 100644 --- a/server/src/routes/applications/data-export/data-export-handlers.ts +++ b/server/src/routes/applications/data-export/data-export-handlers.ts @@ -13,10 +13,15 @@ import { Group, onlyGroups } from "@/decorators/authorise" import { KeycloakAugmentingTransform } from "@/lib/keycloak-augmenting-transform" import { getTempDir } from "@/lib/temp-dir" import { hasCode } from "@/lib/type-guards" +import { AnonymousCsvTransform } from "@/routes/applications/data-export/anonymous-csv-transform" +import { AnonymousIdAugmentingTransform } from "@/routes/applications/data-export/anonymous-id-augmenting-transform" import { ConsentAugmentingTransform, type ConsentAugments, } from "@/routes/applications/data-export/consent-augmenting-transform" +import { UserAgeAugmentingTransform } from "@/routes/applications/data-export/user-age-augmenting-transform" +import { UserCvAugmentingTransform } from "@/routes/applications/data-export/user-cv-augmenting-transform" +import { UserFlagAugmentingTransform } from "@/routes/applications/data-export/user-flag-augmenting-transform" import type { Middleware } from "@/types" import { AttendanceAugmentingTransform, type AttendanceAugments } from "./attendance-augmenting-transform" import { CvExportingWritable } from "./cv-exporting-writable" @@ -36,6 +41,7 @@ class DataExportHandlers { major_league_hacking_attendees_url: new URL("/applications/data-export/major-league-hacking?attendees", origin), hackathons_uk_applications_url: new URL("/applications/data-export/hackathons-uk", origin), hackathons_uk_attendees_url: new URL("/applications/data-export/hackathons-uk?attendees", origin), + anonymous_applications_url: new URL("/applications/data-export/anonymous", origin), }) } } @@ -159,6 +165,34 @@ class DataExportHandlers { } } } + + @onlyGroups([Group.organisers, Group.admins]) + getAnonymous(): Middleware { + return async (_request, response) => { + const tempDir = await getTempDir() + try { + const fileName = "anonymous-data-export.csv" + const fileDestination = pathJoin(tempDir, fileName) + + await pipeline( + Readable.from(generateUserInfo()), + new AttendanceAugmentingTransform(), + new ConsentAugmentingTransform({ media: true, dsuPrivacy: true }), + new UserAgeAugmentingTransform(), + new UserFlagAugmentingTransform("dietary-requirement"), + new UserFlagAugmentingTransform("discipline-of-study"), + new UserCvAugmentingTransform(), + new AnonymousIdAugmentingTransform(), + new AnonymousCsvTransform(), + createWriteStream(fileDestination), + ) + + await response.download(fileDestination, fileName) + } finally { + await rm(tempDir, { recursive: true, force: true }) + } + } + } } const dataExportHandlers = new DataExportHandlers() diff --git a/server/src/routes/applications/data-export/index.ts b/server/src/routes/applications/data-export/index.ts index 8a601fc0..83456ebe 100644 --- a/server/src/routes/applications/data-export/index.ts +++ b/server/src/routes/applications/data-export/index.ts @@ -36,3 +36,10 @@ applicationsDataExportApp .all(authenticate()) .get(dataExportHandlers.getCVArchive()) .all(forbiddenOrUnauthorised()) + +applicationsDataExportApp + .route("/anonymous") + .all(methodNotAllowed(["GET"])) + .all(authenticate()) + .get(dataExportHandlers.getAnonymous()) + .all(forbiddenOrUnauthorised()) diff --git a/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts b/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts index 7b35c056..efd3d177 100644 --- a/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts +++ b/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts @@ -23,6 +23,9 @@ export class PickAttributesToCsvTransform pickAttributesToCsv(item: Source): string { const values: string[] = [] for (const attribute of this.attributes) { + function fail() { + throw new Error(`Unsupported value ${attributeValue} for ${String(attribute.name)}`) + } const attributeValue = item[attribute.name] if (attributeValue == null) { @@ -45,7 +48,20 @@ export class PickAttributesToCsvTransform continue } - throw new Error(`Unsupported value ${attributeValue} for ${String(attribute.name)}`) + if (typeof attributeValue !== "object") fail() + + if (Array.isArray(attributeValue) && attributeValue.every(isString)) { + values.push(JSON.stringify(attributeValue)) + continue + } + + if (attributeValue instanceof Date) { + const epochSeconds = Math.floor(attributeValue.getTime() / 1000) + values.push(String(epochSeconds)) + continue + } + + fail() } return values.join(";") diff --git a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts new file mode 100644 index 00000000..5702f430 --- /dev/null +++ b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict" +import stream from "node:stream" +import type { UserInfo } from "@/database" +import { isString } from "@/lib/type-guards" + +type AgeGroup = "<18" | "18-21" | "22-25" | "26-29" | "30-39" | "40-59" | "60+" + +export type AgeAugments = { + ageGroup: AgeGroup +} + +type AgeAugmentedUserInfo = { userId: string } & AgeAugments + +export class UserAgeAugmentingTransform extends stream.Transform { + constructor() { + super({ + writableObjectMode: true, + readableObjectMode: true, + }) + } + + resolveAgeGroup(age: number): AgeGroup { + if (age < 18) { + return "<18" + } + if (age <= 21) { + return "18-21" + } + if (age <= 25) { + return "22-25" + } + if (age <= 29) { + return "26-29" + } + if (age <= 39) { + return "30-39" + } + if (age <= 59) { + return "40-59" + } + return "60+" + } + + augmentUserInfo(userInfo: UserInfo): AgeAugmentedUserInfo { + assert(userInfo.applicationSubmittedAt) + assert(userInfo.age) + + const ageGroup: AgeGroup = this.resolveAgeGroup(userInfo.age) + + return { ...userInfo, ageGroup } + } + + async augmentChunk(chunk: UserInfo[]): Promise { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: UserInfo[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies AgeAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +} diff --git a/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts new file mode 100644 index 00000000..a4546be4 --- /dev/null +++ b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts @@ -0,0 +1,46 @@ +import stream from "node:stream" +import { prisma, type UserInfo } from "@/database" + +import { isString } from "@/lib/type-guards" + +export type UserCvAugments = { + is_cv_uploaded: boolean + cv_update_time: Date | null +} + +type CvAugmentedUserInfo = { userId: string } & UserCvAugments + +export class UserCvAugmentingTransform extends stream.Transform { + constructor() { + super({ writableObjectMode: true, readableObjectMode: true }) + } + + async augmentUserInfo(userInfo: UserInfo): Promise { + const foundCV = await prisma.userCV.findUnique({ + where: { + userId: userInfo.userId, + }, + select: { + updatedAt: true, + }, + }) + + return { ...userInfo, is_cv_uploaded: !!foundCV, cv_update_time: foundCV ? foundCV.updatedAt : null } + } + + async augmentChunk(chunk: UserInfo[]): Promise { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: UserInfo[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies CvAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +} diff --git a/server/src/routes/applications/data-export/user-flag-augmenting-transform.ts b/server/src/routes/applications/data-export/user-flag-augmenting-transform.ts new file mode 100644 index 00000000..dfa9d314 --- /dev/null +++ b/server/src/routes/applications/data-export/user-flag-augmenting-transform.ts @@ -0,0 +1,58 @@ +import stream from "node:stream" + +import { prisma } from "@/database" +import { isString } from "@/lib/type-guards" + +export type FlagAugments = { [K in Namespace]: Member[] } + +type FlagAugmentedUserInfo = FlagAugments & { + userId: string +} + +export class UserFlagAugmentingTransform extends stream.Transform { + namespace: Namespace + + constructor(namespace: Namespace) { + super({ + writableObjectMode: true, + readableObjectMode: true, + }) + + this.namespace = namespace + } + + async augmentUserInfo(userInfo: { userId: string }): Promise> { + const flags = await prisma.userFlag.findMany({ + where: { userId: userInfo.userId, flagName: { startsWith: `${this.namespace}:` } }, + }) + + const members: Member[] = [] + + for (const flag of flags) { + const flagName = flag.flagName.substring(this.namespace.length + 1) as Member + members.push(flagName) + } + + const result = { [this.namespace as Namespace]: members } satisfies FlagAugments as FlagAugments< + Namespace, + Member + > + return { ...userInfo, ...result } + } + + async augmentChunk(chunk: { userId: string }[]): Promise[]> { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies FlagAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +}