From 6701e7a7f5585f054136b1a1f468a39202acb94a Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Fri, 11 Jul 2025 14:58:47 +0300 Subject: [PATCH 01/12] Wrote class to augment user age to age groups. Started working on the csv-transformer itself --- .../data-export/anonymous-csv-transform.ts | 22 +++++++ .../user-age-augmenting-transform.ts | 59 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 server/src/routes/applications/data-export/anonymous-csv-transform.ts create mode 100644 server/src/routes/applications/data-export/user-age-augmenting-transform.ts diff --git a/server/src/routes/applications/data-export/anonymous-csv-transform.ts b/server/src/routes/applications/data-export/anonymous-csv-transform.ts new file mode 100644 index 00000000..50cf5581 --- /dev/null +++ b/server/src/routes/applications/data-export/anonymous-csv-transform.ts @@ -0,0 +1,22 @@ +import {PickAttributesToCsvTransform} from "@/routes/applications/data-export/pick-attributes-to-csv-transform"; +import {UserInfo} from "@/database"; +import { AgeAugment } from "./user-age-augmenting-transform"; +import {KeycloakAugments} from "@/lib/keycloak-augmenting-transform"; +import {ConsentAugments} from "./consent-augmenting-transform"; +import {AttendanceAugments} from "./attendance-augmenting-transform"; + +export class AnonymousCsvTransform extends PickAttributesToCsvTransform { + constructor(){ + super({ + attributes: [ + {name: "countryOfResidence", label: "country_of_residence"}, + {name: "university", label: "university"}, + {name: "levelOfStudy", label: "level_of_study"}, + {name: "ageGroup", label: "age_group"}, + {name: "hackathonExperience", label: "hackathon_experience"}, + {name: "tShirtSize", label: "t_shirt_size"}, + {name: "isCheckedIn", label: "has_attended"} + ], + }) + } +} diff --git a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts new file mode 100644 index 00000000..376d0ddf --- /dev/null +++ b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts @@ -0,0 +1,59 @@ +import stream from "node:stream"; +import assert from "node:assert/strict"; +import {UserInfo, prisma} from "@/database"; +import {isString} from "@/lib/type-guards"; + +export type AgeAugment = { + ageGroup: "<18" | "18-21" | "22-25" | "26-29" | "30-39" | "40-59" | "60+" +} + +type AgeAugmentedUserInfo = {userId: string} & AgeAugment; + +export class UserAgeAugmentingTransform extends stream.Transform { + constructor() { + super({ + writableObjectMode: true, + readableObjectMode: true + }); + } + + augmentUserInfo(userInfo: UserInfo): AgeAugmentedUserInfo { + const age = userInfo.age; + let ageGroupAugment: AgeAugment; + // If the user has submitted, the age must have been provided + assert(userInfo.applicationSubmittedAt); + assert(age); + + if (age < 18) { + ageGroupAugment = {ageGroup: "<18"}; + } else if (age < 22) { + ageGroupAugment = {ageGroup: "18-21"}; + } else if (age < 26) { + ageGroupAugment = {ageGroup: "22-25"}; + } else if (age < 30) { + ageGroupAugment = {ageGroup: "26-29"}; + } else if (age < 40) { + ageGroupAugment = {ageGroup: "30-39"}; + } else if (age < 60) { + ageGroupAugment = {ageGroup: "40-59"}; + } else { + ageGroupAugment = {ageGroup: "60+"}; + } + + return {...userInfo, ...ageGroupAugment} + } + + async augmentChunk(chunk: UserInfo[]): Promise { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))); + } + + _transform(chunk: UserInfo[], encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk).then((filteredChunk) => { + callback(null, filteredChunk satisfies AgeAugmentedUserInfo[]); + }).catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }); + } +} From 1e646d81c2d022349be1123f8e516c86a96952c9 Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Fri, 11 Jul 2025 15:26:55 +0300 Subject: [PATCH 02/12] Added DSU Privacy policy and photo/media consent to AnonymousCsvTransform --- .../data-export/anonymous-csv-transform.ts | 33 ++++++----- .../user-age-augmenting-transform.ts | 58 ++++++++++--------- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/server/src/routes/applications/data-export/anonymous-csv-transform.ts b/server/src/routes/applications/data-export/anonymous-csv-transform.ts index 50cf5581..dd544c96 100644 --- a/server/src/routes/applications/data-export/anonymous-csv-transform.ts +++ b/server/src/routes/applications/data-export/anonymous-csv-transform.ts @@ -1,21 +1,24 @@ -import {PickAttributesToCsvTransform} from "@/routes/applications/data-export/pick-attributes-to-csv-transform"; -import {UserInfo} from "@/database"; -import { AgeAugment } from "./user-age-augmenting-transform"; -import {KeycloakAugments} from "@/lib/keycloak-augmenting-transform"; -import {ConsentAugments} from "./consent-augmenting-transform"; -import {AttendanceAugments} from "./attendance-augmenting-transform"; +import type { UserInfo } from "@/database" +import { PickAttributesToCsvTransform } from "@/routes/applications/data-export/pick-attributes-to-csv-transform" +import type { AttendanceAugments } from "./attendance-augmenting-transform" +import type { ConsentAugments } from "./consent-augmenting-transform" +import type { AgeAugment } from "./user-age-augmenting-transform" -export class AnonymousCsvTransform extends PickAttributesToCsvTransform { - constructor(){ +export class AnonymousCsvTransform extends PickAttributesToCsvTransform< + UserInfo & AgeAugment & AttendanceAugments & ConsentAugments<"media" | "dsuPrivacy"> +> { + constructor() { super({ attributes: [ - {name: "countryOfResidence", label: "country_of_residence"}, - {name: "university", label: "university"}, - {name: "levelOfStudy", label: "level_of_study"}, - {name: "ageGroup", label: "age_group"}, - {name: "hackathonExperience", label: "hackathon_experience"}, - {name: "tShirtSize", label: "t_shirt_size"}, - {name: "isCheckedIn", label: "has_attended"} + { name: "countryOfResidence", label: "country_of_residence" }, + { name: "university", label: "university" }, + { name: "levelOfStudy", label: "level_of_study" }, + { name: "tShirtSize", label: "t_shirt_size" }, + { name: "hackathonExperience", label: "hackathon_experience" }, + { name: "ageGroup", label: "age_group" }, + { name: "dsuPrivacy", label: "dsu_privacy" }, + { name: "media", label: "photo_media_consent" }, + { name: "isCheckedIn", label: "has_attended" }, ], }) } diff --git a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts index 376d0ddf..272ddcbc 100644 --- a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts +++ b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts @@ -1,59 +1,61 @@ -import stream from "node:stream"; -import assert from "node:assert/strict"; -import {UserInfo, prisma} from "@/database"; -import {isString} from "@/lib/type-guards"; +import assert from "node:assert/strict" +import stream from "node:stream" +import type { UserInfo } from "@/database" +import { isString } from "@/lib/type-guards" export type AgeAugment = { ageGroup: "<18" | "18-21" | "22-25" | "26-29" | "30-39" | "40-59" | "60+" } -type AgeAugmentedUserInfo = {userId: string} & AgeAugment; +type AgeAugmentedUserInfo = { userId: string } & AgeAugment export class UserAgeAugmentingTransform extends stream.Transform { constructor() { super({ writableObjectMode: true, - readableObjectMode: true - }); + readableObjectMode: true, + }) } augmentUserInfo(userInfo: UserInfo): AgeAugmentedUserInfo { - const age = userInfo.age; - let ageGroupAugment: AgeAugment; + const age = userInfo.age + let ageGroupAugment: AgeAugment // If the user has submitted, the age must have been provided - assert(userInfo.applicationSubmittedAt); - assert(age); + assert(userInfo.applicationSubmittedAt) + assert(age) if (age < 18) { - ageGroupAugment = {ageGroup: "<18"}; + ageGroupAugment = { ageGroup: "<18" } } else if (age < 22) { - ageGroupAugment = {ageGroup: "18-21"}; + ageGroupAugment = { ageGroup: "18-21" } } else if (age < 26) { - ageGroupAugment = {ageGroup: "22-25"}; + ageGroupAugment = { ageGroup: "22-25" } } else if (age < 30) { - ageGroupAugment = {ageGroup: "26-29"}; + ageGroupAugment = { ageGroup: "26-29" } } else if (age < 40) { - ageGroupAugment = {ageGroup: "30-39"}; + ageGroupAugment = { ageGroup: "30-39" } } else if (age < 60) { - ageGroupAugment = {ageGroup: "40-59"}; + ageGroupAugment = { ageGroup: "40-59" } } else { - ageGroupAugment = {ageGroup: "60+"}; + ageGroupAugment = { ageGroup: "60+" } } - return {...userInfo, ...ageGroupAugment} + return { ...userInfo, ...ageGroupAugment } } async augmentChunk(chunk: UserInfo[]): Promise { - return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))); + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) } - _transform(chunk: UserInfo[], encoding: never, callback: stream.TransformCallback): void { - this.augmentChunk(chunk).then((filteredChunk) => { - callback(null, filteredChunk satisfies AgeAugmentedUserInfo[]); - }).catch((error: unknown) => { - if (error instanceof Error) return callback(error) - if (isString(error)) return callback(new Error(error)) - return callback(new Error(`Something really strange happened. Error object: ${error}`)) - }); + _transform(chunk: UserInfo[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies AgeAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) } } From 887097e0c19ad6bc6bae89bfd4c125dc429857ff Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Fri, 11 Jul 2025 16:31:53 +0300 Subject: [PATCH 03/12] Wrote augment to find user CV and last time CV was updated --- .../data-export/anonymous-csv-transform.ts | 5 ++- .../user-cv-augmenting-transform.ts | 43 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 server/src/routes/applications/data-export/user-cv-augmenting-transform.ts diff --git a/server/src/routes/applications/data-export/anonymous-csv-transform.ts b/server/src/routes/applications/data-export/anonymous-csv-transform.ts index dd544c96..080de564 100644 --- a/server/src/routes/applications/data-export/anonymous-csv-transform.ts +++ b/server/src/routes/applications/data-export/anonymous-csv-transform.ts @@ -1,11 +1,12 @@ import type { UserInfo } from "@/database" import { PickAttributesToCsvTransform } from "@/routes/applications/data-export/pick-attributes-to-csv-transform" +import type { UserCvAugment } from "@/routes/applications/data-export/user-cv-augmenting-transform" import type { AttendanceAugments } from "./attendance-augmenting-transform" import type { ConsentAugments } from "./consent-augmenting-transform" import type { AgeAugment } from "./user-age-augmenting-transform" export class AnonymousCsvTransform extends PickAttributesToCsvTransform< - UserInfo & AgeAugment & AttendanceAugments & ConsentAugments<"media" | "dsuPrivacy"> + UserInfo & AgeAugment & AttendanceAugments & ConsentAugments<"media" | "dsuPrivacy"> & UserCvAugment > { constructor() { super({ @@ -18,6 +19,8 @@ export class AnonymousCsvTransform extends PickAttributesToCsvTransform< { name: "ageGroup", label: "age_group" }, { name: "dsuPrivacy", label: "dsu_privacy" }, { name: "media", label: "photo_media_consent" }, + { name: "is_cv_uploaded", label: "has_uploaded_cv" }, + { name: "cv_update_time", label: "cv_update_time" }, { name: "isCheckedIn", label: "has_attended" }, ], }) diff --git a/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts new file mode 100644 index 00000000..af0fea0d --- /dev/null +++ b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts @@ -0,0 +1,43 @@ +import stream from "node:stream" +import { prisma, type UserInfo } from "@/database" + +import { isString } from "@/lib/type-guards" + +export type UserCvAugment = { + is_cv_uploaded: boolean + cv_update_time: Date | null +} + +type CvAugmentedUserInfo = { userId: string } & UserCvAugment + +export class UserCvAugmentingTransform extends stream.Transform { + constructor() { + super({ writableObjectMode: true, readableObjectMode: true }) + } + + async augmentUserInfo(userInfo: UserInfo): Promise { + const foundCV = await prisma.userCV.findUnique({ + where: { + userId: userInfo.userId, + }, + }) + + return { ...userInfo, is_cv_uploaded: !!foundCV, cv_update_time: foundCV ? foundCV.updatedAt : null } + } + + async augmentChunk(chunk: UserInfo[]): Promise { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: UserInfo[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies CvAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +} From ce5d11b77bd8b52b9fb0d0d2f6d2521875e3c29b Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Sat, 12 Jul 2025 23:15:39 +0300 Subject: [PATCH 04/12] Wrote augmentor to augment user flags in DB to json arrays stored as strings that can be exported into a CSV --- .../data-export/user-flag-json-augmentor.ts | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 server/src/routes/applications/data-export/user-flag-json-augmentor.ts diff --git a/server/src/routes/applications/data-export/user-flag-json-augmentor.ts b/server/src/routes/applications/data-export/user-flag-json-augmentor.ts new file mode 100644 index 00000000..c21057d9 --- /dev/null +++ b/server/src/routes/applications/data-export/user-flag-json-augmentor.ts @@ -0,0 +1,57 @@ +import stream from "node:stream" + +import { prisma } from "@/database" +import { isString } from "@/lib/type-guards" + +export type FlagAugments = Record + +type FlagAugmentedUserInfo = FlagAugments & { userId: string } + +export class UserFlagAugmentor extends stream.Transform { + namespaces: Namespace[] + + constructor(flags: Record) { + super({ + writableObjectMode: true, + readableObjectMode: true, + }) + + this.namespaces = Object.keys(flags) as Namespace[] + } + + async augmentUserInfo(userInfo: { userId: string }): Promise> { + const flags = await prisma.userFlag.findMany({ where: { userId: userInfo.userId } }) + + const result = Object.fromEntries(this.namespaces.map((namespace) => [namespace, ""])) as Record< + Namespace, + string | undefined + > + + for (const namespace of this.namespaces) { + const namespace_arr: string[] = [] + for (const flag of flags) { + const [flagNamespace, flagName]: string[] = flag.flagName.split(":") + if (flagNamespace === namespace) namespace_arr.push(flagName) + } + result[namespace] = JSON.stringify(namespace_arr) + } + + return { ...userInfo, ...result } + } + + async augmentChunk(chunk: { userId: string }[]): Promise[]> { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies FlagAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +} From bb2fe8c51c9163b3cf69932759d68eb06c9110bd Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Sat, 12 Jul 2025 23:18:23 +0300 Subject: [PATCH 05/12] Added dietary requirements and discipline of study to the csv transformer --- .../data-export/anonymous-csv-transform.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/routes/applications/data-export/anonymous-csv-transform.ts b/server/src/routes/applications/data-export/anonymous-csv-transform.ts index 080de564..8dea3cb2 100644 --- a/server/src/routes/applications/data-export/anonymous-csv-transform.ts +++ b/server/src/routes/applications/data-export/anonymous-csv-transform.ts @@ -1,12 +1,18 @@ import type { UserInfo } from "@/database" import { PickAttributesToCsvTransform } from "@/routes/applications/data-export/pick-attributes-to-csv-transform" import type { UserCvAugment } from "@/routes/applications/data-export/user-cv-augmenting-transform" +import type { FlagAugments } from "@/routes/applications/data-export/user-flag-json-augmentor" import type { AttendanceAugments } from "./attendance-augmenting-transform" import type { ConsentAugments } from "./consent-augmenting-transform" import type { AgeAugment } from "./user-age-augmenting-transform" export class AnonymousCsvTransform extends PickAttributesToCsvTransform< - UserInfo & AgeAugment & AttendanceAugments & ConsentAugments<"media" | "dsuPrivacy"> & UserCvAugment + UserInfo & + AgeAugment & + AttendanceAugments & + ConsentAugments<"media" | "dsuPrivacy"> & + UserCvAugment & + FlagAugments<"discipline-of-study" | "dietary-requirement"> > { constructor() { super({ @@ -16,6 +22,8 @@ export class AnonymousCsvTransform extends PickAttributesToCsvTransform< { name: "levelOfStudy", label: "level_of_study" }, { name: "tShirtSize", label: "t_shirt_size" }, { name: "hackathonExperience", label: "hackathon_experience" }, + { name: "dietary-requirement", label: "dietary_requirements" }, + { name: "discipline-of-study", label: "discipline_of_study" }, { name: "ageGroup", label: "age_group" }, { name: "dsuPrivacy", label: "dsu_privacy" }, { name: "media", label: "photo_media_consent" }, From 3a92289912baeb7cb27741ade0c672a6830d668e Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Sat, 12 Jul 2025 23:37:07 +0300 Subject: [PATCH 06/12] Added route and export handler --- .../data-export/data-export-handlers.ts | 24 +++++++++++++++++++ .../routes/applications/data-export/index.ts | 7 ++++++ 2 files changed, 31 insertions(+) diff --git a/server/src/routes/applications/data-export/data-export-handlers.ts b/server/src/routes/applications/data-export/data-export-handlers.ts index 8e3d87b2..0918b736 100644 --- a/server/src/routes/applications/data-export/data-export-handlers.ts +++ b/server/src/routes/applications/data-export/data-export-handlers.ts @@ -17,6 +17,8 @@ import { ConsentAugmentingTransform, type ConsentAugments, } from "@/routes/applications/data-export/consent-augmenting-transform" +import { UserAgeAugmentingTransform } from "@/routes/applications/data-export/user-age-augmenting-transform" +import { UserFlagAugmentor } from "@/routes/applications/data-export/user-flag-json-augmentor" import type { Middleware } from "@/types" import { AttendanceAugmentingTransform, type AttendanceAugments } from "./attendance-augmenting-transform" import { CvExportingWritable } from "./cv-exporting-writable" @@ -159,6 +161,28 @@ class DataExportHandlers { } } } + + @onlyGroups([Group.organisers, Group.admins]) + getAnonymousDataExport(): Middleware { + return async (_request, _response) => { + const tempDir = await getTempDir() + try { + const fileName: string = "anonymous-data-export.csv" + const fileDestination: string = pathJoin(tempDir, fileName) + + await pipeline( + Readable.from(generateUserInfo()), + new AttendanceAugmentingTransform(), + new ConsentAugmentingTransform({ media: true, dsuPrivacy: true }), + new UserAgeAugmentingTransform(), + new UserFlagAugmentor({}), // Not sure what to put in here + createWriteStream(fileDestination), + ) + } finally { + await rm(tempDir, { recursive: true, force: true }) + } + } + } } const dataExportHandlers = new DataExportHandlers() diff --git a/server/src/routes/applications/data-export/index.ts b/server/src/routes/applications/data-export/index.ts index 8a601fc0..0e804674 100644 --- a/server/src/routes/applications/data-export/index.ts +++ b/server/src/routes/applications/data-export/index.ts @@ -36,3 +36,10 @@ applicationsDataExportApp .all(authenticate()) .get(dataExportHandlers.getCVArchive()) .all(forbiddenOrUnauthorised()) + +applicationsDataExportApp + .route("/anon-csv-export") + .all(methodNotAllowed(["GET"])) + .all(authenticate()) + .get(dataExportHandlers.getAnonymousDataExport()) + .all(forbiddenOrUnauthorised()) From e40342fcbe7fc5bf31b7b6c3cd90acf57a21ba03 Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Mon, 14 Jul 2025 15:26:32 +0300 Subject: [PATCH 07/12] Refactored age augment transformer --- .../user-age-augmenting-transform.ts | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts index 272ddcbc..73766620 100644 --- a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts +++ b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts @@ -3,8 +3,10 @@ import stream from "node:stream" import type { UserInfo } from "@/database" import { isString } from "@/lib/type-guards" +type AgeGroup = "<18" | "18-21" | "22-25" | "26-29" | "30-39" | "40-59" | "60+" + export type AgeAugment = { - ageGroup: "<18" | "18-21" | "22-25" | "26-29" | "30-39" | "40-59" | "60+" + ageGroup: AgeGroup } type AgeAugmentedUserInfo = { userId: string } & AgeAugment @@ -17,30 +19,35 @@ export class UserAgeAugmentingTransform extends stream.Transform { }) } + resolveAgeGroup(age: number): AgeGroup { + if (age < 18) { + return "<18" + } + if (age <= 21) { + return "18-21" + } + if (age <= 25) { + return "22-25" + } + if (age <= 29) { + return "26-29" + } + if (age <= 39) { + return "30-39" + } + if (age <= 59) { + return "40-59" + } + return "60+" + } + augmentUserInfo(userInfo: UserInfo): AgeAugmentedUserInfo { - const age = userInfo.age - let ageGroupAugment: AgeAugment - // If the user has submitted, the age must have been provided assert(userInfo.applicationSubmittedAt) - assert(age) + assert(userInfo.age) - if (age < 18) { - ageGroupAugment = { ageGroup: "<18" } - } else if (age < 22) { - ageGroupAugment = { ageGroup: "18-21" } - } else if (age < 26) { - ageGroupAugment = { ageGroup: "22-25" } - } else if (age < 30) { - ageGroupAugment = { ageGroup: "26-29" } - } else if (age < 40) { - ageGroupAugment = { ageGroup: "30-39" } - } else if (age < 60) { - ageGroupAugment = { ageGroup: "40-59" } - } else { - ageGroupAugment = { ageGroup: "60+" } - } + const ageGroup: AgeGroup = this.resolveAgeGroup(userInfo.age) - return { ...userInfo, ...ageGroupAugment } + return { ...userInfo, ageGroup } } async augmentChunk(chunk: UserInfo[]): Promise { From dfb88d4d14ac51913361e4e0dcfa560b2b77f101 Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Mon, 14 Jul 2025 15:38:06 +0300 Subject: [PATCH 08/12] Selected just updatedAt date in DB query to get userCV data --- .../applications/data-export/user-cv-augmenting-transform.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts index af0fea0d..b83cdf25 100644 --- a/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts +++ b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts @@ -20,6 +20,9 @@ export class UserCvAugmentingTransform extends stream.Transform { where: { userId: userInfo.userId, }, + select: { + updatedAt: true, + }, }) return { ...userInfo, is_cv_uploaded: !!foundCV, cv_update_time: foundCV ? foundCV.updatedAt : null } From 71f179bdeaaa716df1df91372cd4438148c699b1 Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Mon, 14 Jul 2025 16:25:39 +0300 Subject: [PATCH 09/12] Refactored user flag augmenting transform. Fixed API route. Added a few more data points to CSV transform --- .../data-export/anonymous-csv-transform.ts | 31 ++++++---- .../data-export/data-export-handlers.ts | 13 +++-- .../routes/applications/data-export/index.ts | 4 +- .../pick-attributes-to-csv-transform.ts | 5 ++ .../user-flag-augmenting-transform.ts | 58 +++++++++++++++++++ .../data-export/user-flag-json-augmentor.ts | 57 ------------------ 6 files changed, 92 insertions(+), 76 deletions(-) create mode 100644 server/src/routes/applications/data-export/user-flag-augmenting-transform.ts delete mode 100644 server/src/routes/applications/data-export/user-flag-json-augmentor.ts diff --git a/server/src/routes/applications/data-export/anonymous-csv-transform.ts b/server/src/routes/applications/data-export/anonymous-csv-transform.ts index 8dea3cb2..23715771 100644 --- a/server/src/routes/applications/data-export/anonymous-csv-transform.ts +++ b/server/src/routes/applications/data-export/anonymous-csv-transform.ts @@ -1,19 +1,22 @@ +import type { DisciplineOfStudy } from "@durhack/durhack-common/input/discipline-of-study" +import type { DietaryRequirement } from "@durhack/durhack-common/types/application" import type { UserInfo } from "@/database" import { PickAttributesToCsvTransform } from "@/routes/applications/data-export/pick-attributes-to-csv-transform" import type { UserCvAugment } from "@/routes/applications/data-export/user-cv-augmenting-transform" -import type { FlagAugments } from "@/routes/applications/data-export/user-flag-json-augmentor" +import type { FlagAugments } from "@/routes/applications/data-export/user-flag-augmenting-transform" import type { AttendanceAugments } from "./attendance-augmenting-transform" import type { ConsentAugments } from "./consent-augmenting-transform" import type { AgeAugment } from "./user-age-augmenting-transform" -export class AnonymousCsvTransform extends PickAttributesToCsvTransform< - UserInfo & - AgeAugment & - AttendanceAugments & - ConsentAugments<"media" | "dsuPrivacy"> & - UserCvAugment & - FlagAugments<"discipline-of-study" | "dietary-requirement"> -> { +type Source = UserInfo & + AgeAugment & + AttendanceAugments & + ConsentAugments<"media" | "dsuPrivacy"> & + UserCvAugment & + FlagAugments<"discipline-of-study", DisciplineOfStudy> & + FlagAugments<"dietary-requirement", DietaryRequirement> + +export class AnonymousCsvTransform extends PickAttributesToCsvTransform { constructor() { super({ attributes: [ @@ -21,15 +24,19 @@ export class AnonymousCsvTransform extends PickAttributesToCsvTransform< { name: "university", label: "university" }, { name: "levelOfStudy", label: "level_of_study" }, { name: "tShirtSize", label: "t_shirt_size" }, + { name: "gender", label: "gender" }, + { name: "ethnicity", label: "ethnicity" }, { name: "hackathonExperience", label: "hackathon_experience" }, { name: "dietary-requirement", label: "dietary_requirements" }, { name: "discipline-of-study", label: "discipline_of_study" }, { name: "ageGroup", label: "age_group" }, { name: "dsuPrivacy", label: "dsu_privacy" }, { name: "media", label: "photo_media_consent" }, - { name: "is_cv_uploaded", label: "has_uploaded_cv" }, - { name: "cv_update_time", label: "cv_update_time" }, - { name: "isCheckedIn", label: "has_attended" }, + { name: "is_cv_uploaded", label: "did_upload_cv" }, + { name: "cv_update_time", label: "cv_last_updated_at" }, + { name: "applicationSubmittedAt", label: "application_submitted_at" }, + { name: "applicationStatus", label: "application_status" }, + { name: "isCheckedIn", label: "attendance" }, ], }) } diff --git a/server/src/routes/applications/data-export/data-export-handlers.ts b/server/src/routes/applications/data-export/data-export-handlers.ts index 0918b736..f7205e14 100644 --- a/server/src/routes/applications/data-export/data-export-handlers.ts +++ b/server/src/routes/applications/data-export/data-export-handlers.ts @@ -13,12 +13,13 @@ import { Group, onlyGroups } from "@/decorators/authorise" import { KeycloakAugmentingTransform } from "@/lib/keycloak-augmenting-transform" import { getTempDir } from "@/lib/temp-dir" import { hasCode } from "@/lib/type-guards" +import { AnonymousCsvTransform } from "@/routes/applications/data-export/anonymous-csv-transform" import { ConsentAugmentingTransform, type ConsentAugments, } from "@/routes/applications/data-export/consent-augmenting-transform" import { UserAgeAugmentingTransform } from "@/routes/applications/data-export/user-age-augmenting-transform" -import { UserFlagAugmentor } from "@/routes/applications/data-export/user-flag-json-augmentor" +import { UserFlagAugmentingTransform } from "@/routes/applications/data-export/user-flag-augmenting-transform" import type { Middleware } from "@/types" import { AttendanceAugmentingTransform, type AttendanceAugments } from "./attendance-augmenting-transform" import { CvExportingWritable } from "./cv-exporting-writable" @@ -163,19 +164,21 @@ class DataExportHandlers { } @onlyGroups([Group.organisers, Group.admins]) - getAnonymousDataExport(): Middleware { + getAnonymous(): Middleware { return async (_request, _response) => { const tempDir = await getTempDir() try { - const fileName: string = "anonymous-data-export.csv" - const fileDestination: string = pathJoin(tempDir, fileName) + const fileName = "anonymous-data-export.csv" + const fileDestination = pathJoin(tempDir, fileName) await pipeline( Readable.from(generateUserInfo()), new AttendanceAugmentingTransform(), new ConsentAugmentingTransform({ media: true, dsuPrivacy: true }), new UserAgeAugmentingTransform(), - new UserFlagAugmentor({}), // Not sure what to put in here + new UserFlagAugmentingTransform("dietary-requirements"), + new UserFlagAugmentingTransform("discipline-of-study"), + new AnonymousCsvTransform(), createWriteStream(fileDestination), ) } finally { diff --git a/server/src/routes/applications/data-export/index.ts b/server/src/routes/applications/data-export/index.ts index 0e804674..83456ebe 100644 --- a/server/src/routes/applications/data-export/index.ts +++ b/server/src/routes/applications/data-export/index.ts @@ -38,8 +38,8 @@ applicationsDataExportApp .all(forbiddenOrUnauthorised()) applicationsDataExportApp - .route("/anon-csv-export") + .route("/anonymous") .all(methodNotAllowed(["GET"])) .all(authenticate()) - .get(dataExportHandlers.getAnonymousDataExport()) + .get(dataExportHandlers.getAnonymous()) .all(forbiddenOrUnauthorised()) diff --git a/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts b/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts index 7b35c056..8ffe0acc 100644 --- a/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts +++ b/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts @@ -45,6 +45,11 @@ export class PickAttributesToCsvTransform continue } + if (Array.isArray(attributeValue) && attributeValue.every(isString)) { + values.push(JSON.stringify(attributeValue)) + continue + } + throw new Error(`Unsupported value ${attributeValue} for ${String(attribute.name)}`) } diff --git a/server/src/routes/applications/data-export/user-flag-augmenting-transform.ts b/server/src/routes/applications/data-export/user-flag-augmenting-transform.ts new file mode 100644 index 00000000..dfa9d314 --- /dev/null +++ b/server/src/routes/applications/data-export/user-flag-augmenting-transform.ts @@ -0,0 +1,58 @@ +import stream from "node:stream" + +import { prisma } from "@/database" +import { isString } from "@/lib/type-guards" + +export type FlagAugments = { [K in Namespace]: Member[] } + +type FlagAugmentedUserInfo = FlagAugments & { + userId: string +} + +export class UserFlagAugmentingTransform extends stream.Transform { + namespace: Namespace + + constructor(namespace: Namespace) { + super({ + writableObjectMode: true, + readableObjectMode: true, + }) + + this.namespace = namespace + } + + async augmentUserInfo(userInfo: { userId: string }): Promise> { + const flags = await prisma.userFlag.findMany({ + where: { userId: userInfo.userId, flagName: { startsWith: `${this.namespace}:` } }, + }) + + const members: Member[] = [] + + for (const flag of flags) { + const flagName = flag.flagName.substring(this.namespace.length + 1) as Member + members.push(flagName) + } + + const result = { [this.namespace as Namespace]: members } satisfies FlagAugments as FlagAugments< + Namespace, + Member + > + return { ...userInfo, ...result } + } + + async augmentChunk(chunk: { userId: string }[]): Promise[]> { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies FlagAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +} diff --git a/server/src/routes/applications/data-export/user-flag-json-augmentor.ts b/server/src/routes/applications/data-export/user-flag-json-augmentor.ts deleted file mode 100644 index c21057d9..00000000 --- a/server/src/routes/applications/data-export/user-flag-json-augmentor.ts +++ /dev/null @@ -1,57 +0,0 @@ -import stream from "node:stream" - -import { prisma } from "@/database" -import { isString } from "@/lib/type-guards" - -export type FlagAugments = Record - -type FlagAugmentedUserInfo = FlagAugments & { userId: string } - -export class UserFlagAugmentor extends stream.Transform { - namespaces: Namespace[] - - constructor(flags: Record) { - super({ - writableObjectMode: true, - readableObjectMode: true, - }) - - this.namespaces = Object.keys(flags) as Namespace[] - } - - async augmentUserInfo(userInfo: { userId: string }): Promise> { - const flags = await prisma.userFlag.findMany({ where: { userId: userInfo.userId } }) - - const result = Object.fromEntries(this.namespaces.map((namespace) => [namespace, ""])) as Record< - Namespace, - string | undefined - > - - for (const namespace of this.namespaces) { - const namespace_arr: string[] = [] - for (const flag of flags) { - const [flagNamespace, flagName]: string[] = flag.flagName.split(":") - if (flagNamespace === namespace) namespace_arr.push(flagName) - } - result[namespace] = JSON.stringify(namespace_arr) - } - - return { ...userInfo, ...result } - } - - async augmentChunk(chunk: { userId: string }[]): Promise[]> { - return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) - } - - _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void { - this.augmentChunk(chunk) - .then((filteredChunk) => { - callback(null, filteredChunk satisfies FlagAugmentedUserInfo[]) - }) - .catch((error: unknown) => { - if (error instanceof Error) return callback(error) - if (isString(error)) return callback(new Error(error)) - return callback(new Error(`Something really strange happened. Error object: ${error}`)) - }) - } -} From cb2216c0caf2838ea5bdfebf0c2273a4046b8a3d Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Mon, 14 Jul 2025 17:04:45 +0300 Subject: [PATCH 10/12] Tested and checked whether data is exported correctly. Made changes to do so --- .../data-export/anonymous-csv-transform.ts | 15 ++++--- .../anonymous-id-augmenting-transform.ts | 40 +++++++++++++++++++ .../data-export/data-export-handlers.ts | 8 +++- .../pick-attributes-to-csv-transform.ts | 13 +++++- .../user-age-augmenting-transform.ts | 4 +- .../user-cv-augmenting-transform.ts | 4 +- 6 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts diff --git a/server/src/routes/applications/data-export/anonymous-csv-transform.ts b/server/src/routes/applications/data-export/anonymous-csv-transform.ts index 23715771..937ec6f8 100644 --- a/server/src/routes/applications/data-export/anonymous-csv-transform.ts +++ b/server/src/routes/applications/data-export/anonymous-csv-transform.ts @@ -1,18 +1,20 @@ import type { DisciplineOfStudy } from "@durhack/durhack-common/input/discipline-of-study" import type { DietaryRequirement } from "@durhack/durhack-common/types/application" import type { UserInfo } from "@/database" +import type { IdAugments } from "@/routes/applications/data-export/anonymous-id-augmenting-transform" +import type { AttendanceAugments } from "@/routes/applications/data-export/attendance-augmenting-transform" +import type { ConsentAugments } from "@/routes/applications/data-export/consent-augmenting-transform" import { PickAttributesToCsvTransform } from "@/routes/applications/data-export/pick-attributes-to-csv-transform" -import type { UserCvAugment } from "@/routes/applications/data-export/user-cv-augmenting-transform" +import type { AgeAugments } from "@/routes/applications/data-export/user-age-augmenting-transform" +import type { UserCvAugments } from "@/routes/applications/data-export/user-cv-augmenting-transform" import type { FlagAugments } from "@/routes/applications/data-export/user-flag-augmenting-transform" -import type { AttendanceAugments } from "./attendance-augmenting-transform" -import type { ConsentAugments } from "./consent-augmenting-transform" -import type { AgeAugment } from "./user-age-augmenting-transform" type Source = UserInfo & - AgeAugment & + IdAugments & + AgeAugments & AttendanceAugments & ConsentAugments<"media" | "dsuPrivacy"> & - UserCvAugment & + UserCvAugments & FlagAugments<"discipline-of-study", DisciplineOfStudy> & FlagAugments<"dietary-requirement", DietaryRequirement> @@ -20,6 +22,7 @@ export class AnonymousCsvTransform extends PickAttributesToCsvTransform constructor() { super({ attributes: [ + { name: "id", label: "anon_id" }, { name: "countryOfResidence", label: "country_of_residence" }, { name: "university", label: "university" }, { name: "levelOfStudy", label: "level_of_study" }, diff --git a/server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts b/server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts new file mode 100644 index 00000000..55129616 --- /dev/null +++ b/server/src/routes/applications/data-export/anonymous-id-augmenting-transform.ts @@ -0,0 +1,40 @@ +import stream from "node:stream" +import { isString } from "@/lib/type-guards" + +export type IdAugments = { id: number } + +type IdAugmentedUserInfo = IdAugments & { userId: string } + +export class AnonymousIdAugmentingTransform extends stream.Transform { + counter: number + + constructor() { + super({ + writableObjectMode: true, + readableObjectMode: true, + }) + + this.counter = 0 + } + + augmentUserInfo(userInfo: { userId: string; id?: number }): IdAugmentedUserInfo { + userInfo.id = this.counter++ + return userInfo as IdAugmentedUserInfo + } + + async augmentChunk(chunk: { userId: string }[]): Promise { + return await Promise.all(chunk.map((userInfo) => this.augmentUserInfo(userInfo))) + } + + _transform(chunk: { userId: string }[], _encoding: never, callback: stream.TransformCallback): void { + this.augmentChunk(chunk) + .then((filteredChunk) => { + callback(null, filteredChunk satisfies IdAugmentedUserInfo[]) + }) + .catch((error: unknown) => { + if (error instanceof Error) return callback(error) + if (isString(error)) return callback(new Error(error)) + return callback(new Error(`Something really strange happened. Error object: ${error}`)) + }) + } +} diff --git a/server/src/routes/applications/data-export/data-export-handlers.ts b/server/src/routes/applications/data-export/data-export-handlers.ts index f7205e14..0ae3054c 100644 --- a/server/src/routes/applications/data-export/data-export-handlers.ts +++ b/server/src/routes/applications/data-export/data-export-handlers.ts @@ -14,6 +14,7 @@ import { KeycloakAugmentingTransform } from "@/lib/keycloak-augmenting-transform import { getTempDir } from "@/lib/temp-dir" import { hasCode } from "@/lib/type-guards" import { AnonymousCsvTransform } from "@/routes/applications/data-export/anonymous-csv-transform" +import { AnonymousIdAugmentingTransform } from "@/routes/applications/data-export/anonymous-id-augmenting-transform" import { ConsentAugmentingTransform, type ConsentAugments, @@ -165,7 +166,7 @@ class DataExportHandlers { @onlyGroups([Group.organisers, Group.admins]) getAnonymous(): Middleware { - return async (_request, _response) => { + return async (_request, response) => { const tempDir = await getTempDir() try { const fileName = "anonymous-data-export.csv" @@ -176,11 +177,14 @@ class DataExportHandlers { new AttendanceAugmentingTransform(), new ConsentAugmentingTransform({ media: true, dsuPrivacy: true }), new UserAgeAugmentingTransform(), - new UserFlagAugmentingTransform("dietary-requirements"), + new UserFlagAugmentingTransform("dietary-requirement"), new UserFlagAugmentingTransform("discipline-of-study"), + new AnonymousIdAugmentingTransform(), new AnonymousCsvTransform(), createWriteStream(fileDestination), ) + + await response.download(fileDestination, fileName) } finally { await rm(tempDir, { recursive: true, force: true }) } diff --git a/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts b/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts index 8ffe0acc..efd3d177 100644 --- a/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts +++ b/server/src/routes/applications/data-export/pick-attributes-to-csv-transform.ts @@ -23,6 +23,9 @@ export class PickAttributesToCsvTransform pickAttributesToCsv(item: Source): string { const values: string[] = [] for (const attribute of this.attributes) { + function fail() { + throw new Error(`Unsupported value ${attributeValue} for ${String(attribute.name)}`) + } const attributeValue = item[attribute.name] if (attributeValue == null) { @@ -45,12 +48,20 @@ export class PickAttributesToCsvTransform continue } + if (typeof attributeValue !== "object") fail() + if (Array.isArray(attributeValue) && attributeValue.every(isString)) { values.push(JSON.stringify(attributeValue)) continue } - throw new Error(`Unsupported value ${attributeValue} for ${String(attribute.name)}`) + if (attributeValue instanceof Date) { + const epochSeconds = Math.floor(attributeValue.getTime() / 1000) + values.push(String(epochSeconds)) + continue + } + + fail() } return values.join(";") diff --git a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts index 73766620..5702f430 100644 --- a/server/src/routes/applications/data-export/user-age-augmenting-transform.ts +++ b/server/src/routes/applications/data-export/user-age-augmenting-transform.ts @@ -5,11 +5,11 @@ import { isString } from "@/lib/type-guards" type AgeGroup = "<18" | "18-21" | "22-25" | "26-29" | "30-39" | "40-59" | "60+" -export type AgeAugment = { +export type AgeAugments = { ageGroup: AgeGroup } -type AgeAugmentedUserInfo = { userId: string } & AgeAugment +type AgeAugmentedUserInfo = { userId: string } & AgeAugments export class UserAgeAugmentingTransform extends stream.Transform { constructor() { diff --git a/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts index b83cdf25..a4546be4 100644 --- a/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts +++ b/server/src/routes/applications/data-export/user-cv-augmenting-transform.ts @@ -3,12 +3,12 @@ import { prisma, type UserInfo } from "@/database" import { isString } from "@/lib/type-guards" -export type UserCvAugment = { +export type UserCvAugments = { is_cv_uploaded: boolean cv_update_time: Date | null } -type CvAugmentedUserInfo = { userId: string } & UserCvAugment +type CvAugmentedUserInfo = { userId: string } & UserCvAugments export class UserCvAugmentingTransform extends stream.Transform { constructor() { From 3902edc03d6efb9a11ae8c94d3bc394d05bf435a Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Mon, 14 Jul 2025 17:07:58 +0300 Subject: [PATCH 11/12] Added URL for anonymous application export to the root of the data-export/ route --- .../src/routes/applications/data-export/data-export-handlers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/routes/applications/data-export/data-export-handlers.ts b/server/src/routes/applications/data-export/data-export-handlers.ts index 0ae3054c..d9ca209a 100644 --- a/server/src/routes/applications/data-export/data-export-handlers.ts +++ b/server/src/routes/applications/data-export/data-export-handlers.ts @@ -40,6 +40,7 @@ class DataExportHandlers { major_league_hacking_attendees_url: new URL("/applications/data-export/major-league-hacking?attendees", origin), hackathons_uk_applications_url: new URL("/applications/data-export/hackathons-uk", origin), hackathons_uk_attendees_url: new URL("/applications/data-export/hackathons-uk?attendees", origin), + anonymous_applications_url: new URL("/applications/data-export/anonymous", origin), }) } } From 08213c30e81e8cc285db6ff88ab83c277b134819 Mon Sep 17 00:00:00 2001 From: Emre Ozden Date: Thu, 17 Jul 2025 00:51:34 +0300 Subject: [PATCH 12/12] Fixed small mistake made while rebasing, had to add CV augmenting transform to the route handler --- .../src/routes/applications/data-export/data-export-handlers.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/routes/applications/data-export/data-export-handlers.ts b/server/src/routes/applications/data-export/data-export-handlers.ts index d9ca209a..06ebc19c 100644 --- a/server/src/routes/applications/data-export/data-export-handlers.ts +++ b/server/src/routes/applications/data-export/data-export-handlers.ts @@ -20,6 +20,7 @@ import { type ConsentAugments, } from "@/routes/applications/data-export/consent-augmenting-transform" import { UserAgeAugmentingTransform } from "@/routes/applications/data-export/user-age-augmenting-transform" +import { UserCvAugmentingTransform } from "@/routes/applications/data-export/user-cv-augmenting-transform" import { UserFlagAugmentingTransform } from "@/routes/applications/data-export/user-flag-augmenting-transform" import type { Middleware } from "@/types" import { AttendanceAugmentingTransform, type AttendanceAugments } from "./attendance-augmenting-transform" @@ -180,6 +181,7 @@ class DataExportHandlers { new UserAgeAugmentingTransform(), new UserFlagAugmentingTransform("dietary-requirement"), new UserFlagAugmentingTransform("discipline-of-study"), + new UserCvAugmentingTransform(), new AnonymousIdAugmentingTransform(), new AnonymousCsvTransform(), createWriteStream(fileDestination),