Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor app-simplefin.ts to make it easily testable #440

Open
wants to merge 22 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 20 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docker/edge-alpine.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ RUN curl -L -o /tmp/desktop-client.zip --header "Authorization: Bearer ${GITHUB_
RUN unzip /tmp/desktop-client.zip -d /public

FROM alpine:3.17 as prod
RUN apk add --no-cache nodejs tini
RUN apk add --no-cache nodejs npm tini

ARG USERNAME=actual
ARG USER_UID=1001
Expand All @@ -33,4 +33,4 @@ ADD migrations ./migrations
ENTRYPOINT ["/sbin/tini","-g", "--"]
ENV ACTUAL_WEB_ROOT=/public
EXPOSE 5006
CMD ["node", "app.js"]
CMD ["npx", "tsx", "app.js"]
2 changes: 1 addition & 1 deletion docker/edge-ubuntu.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,4 @@ ADD migrations ./migrations
ENTRYPOINT ["/usr/bin/tini","-g", "--"]
ENV ACTUAL_WEB_ROOT=/public
EXPOSE 5006
CMD ["node", "app.js"]
CMD ["npx", "tsx", "app.js"]
4 changes: 2 additions & 2 deletions docker/stable-alpine.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ RUN yarn workspaces focus --all --production
RUN if [ "$(uname -m)" = "armv7l" ]; then npm install bcrypt better-sqlite3 --build-from-source; fi

FROM alpine:3.17 as prod
RUN apk add --no-cache nodejs tini
RUN apk add --no-cache nodejs npm tini

ARG USERNAME=actual
ARG USER_UID=1001
Expand All @@ -23,4 +23,4 @@ ADD src ./src
ADD migrations ./migrations
ENTRYPOINT ["/sbin/tini","-g", "--"]
EXPOSE 5006
CMD ["node", "app.js"]
CMD ["npx", "tsx", "app.js"]
2 changes: 1 addition & 1 deletion docker/stable-ubuntu.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ ADD src ./src
ADD migrations ./migrations
ENTRYPOINT ["/usr/bin/tini","-g", "--"]
EXPOSE 5006
CMD ["node", "app.js"]
CMD ["npx", "tsx", "app.js"]
6 changes: 4 additions & 2 deletions jest.config.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
"roots": ["<rootDir>"],
"moduleFileExtensions": ["ts", "js", "json"],
"testEnvironment": "node",
"collectCoverage": true,
"collectCoverage": false,
"collectCoverageFrom": ["**/*.{js,ts,tsx}"],
"coveragePathIgnorePatterns": ["dist", "/node_modules/", "/build/", "/coverage/"],
"coverageReporters": ["html", "lcov", "text", "text-summary"],
"extensionsToTreatAsEsm": [".ts"],
"resetMocks": true,
"restoreMocks": true
"restoreMocks": true,
"silent": true
}
21 changes: 15 additions & 6 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,24 @@
"description": "actual syncing server",
"type": "module",
"scripts": {
"start": "node app",
"start": "tsx app",
"docker:artifacts": "chmod +x docker/download-artifacts.sh && ./docker/download-artifacts.sh",
"docker:build-common": "yarn docker:artifacts && docker build --build-arg GITHUB_TOKEN=$(gh auth token) -t actual-server-dev",
"docker:build-edge": "yarn docker:build-common -f docker/edge-ubuntu.Dockerfile .",
"docker:build-edge-alpine": "yarn docker:build-common -f docker/edge-alpine.Dockerfile .",
"docker:build-stable": "yarn docker:build-common -f docker/stable-ubuntu.Dockerfile .",
"docker:build-stable-alpine": "yarn docker:build-common -f docker/stable-alpine.Dockerfile .",
"docker:run": "docker run --rm -p 5006:5006 actual-server-dev",
"lint": "eslint . --max-warnings 0",
Comment on lines +9 to 16
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I built all the docker images and ran them, making sure the server ran and was accessible with curl.

"lint:fix": "eslint . --fix",
"build": "tsc",
"test": "NODE_ENV=test NODE_OPTIONS='--experimental-vm-modules --trace-warnings' jest --coverage",
"test": "NODE_ENV=test NODE_NO_WARNINGS=1 NODE_OPTIONS='--experimental-vm-modules --trace-warnings' jest",
"test:coverage": "NODE_ENV=test NODE_NO_WARNINGS=1 NODE_OPTIONS='--experimental-vm-modules --trace-warnings' jest --coverage",
"db:migrate": "NODE_ENV=development node src/run-migrations.js up",
"db:downgrade": "NODE_ENV=development node src/run-migrations.js down",
"db:test-migrate": "NODE_ENV=test node src/run-migrations.js up",
"db:test-downgrade": "NODE_ENV=test node src/run-migrations.js down",
"types": "tsc --noEmit --incremental",
"types": "tsc --emitDeclarationOnly --incremental",
"verify": "yarn lint && yarn types",
"reset-password": "node src/scripts/reset-password.js",
"health-check": "node src/scripts/health-check.js"
Expand All @@ -36,6 +44,7 @@
"jws": "^4.0.0",
"migrate": "^2.0.1",
"nordigen-node": "^1.4.0",
"tsx": "^4.17.0",
"uuid": "^9.0.0",
"winston": "^3.14.2"
},
Expand All @@ -50,14 +59,14 @@
"@types/node": "^17.0.45",
"@types/supertest": "^2.0.12",
"@types/uuid": "^9.0.0",
"@typescript-eslint/eslint-plugin": "^5.51.0",
"@typescript-eslint/parser": "^5.51.0",
"@typescript-eslint/eslint-plugin": "^8.2",
"@typescript-eslint/parser": "^8.2",
"eslint": "^8.33.0",
"eslint-plugin-prettier": "^4.2.1",
"jest": "^29.3.1",
"prettier": "^2.8.3",
"supertest": "^6.3.1",
"typescript": "^4.9.5"
"typescript": "^5.5.4"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Needed to update typescript to have certain flags in tsconfig.json work.

},
"engines": {
"node": ">=18.0.0"
Expand Down
139 changes: 24 additions & 115 deletions src/app-simplefin/app-simplefin.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
import express from 'express';
import { inspect } from 'util';
import https from 'https';
import { SecretName, secretsService } from '../services/secrets-service.js';
import { handleError } from '../app-gocardless/util/handle-error.js';
import { requestLoggerMiddleware } from '../util/middlewares.js';
import { SimpleFinService } from './services/simplefin-service.ts';
import { SimplefinApi } from './services/simplefin-api.ts';
import { HttpsClient } from './httpClient.ts';

const app = express();
export { app as handlers };
app.use(express.json());
app.use(requestLoggerMiddleware);

const simplefinService = new SimpleFinService(
new SimplefinApi(new HttpsClient()),
);
Comment on lines +15 to +17
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dependency injection, makes it easier to test

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to mock simplefinService to be able to test app-simplefin.js as an integration test, but I couldn't get jest.spyOn to work, so I didn't. I'm relying on someone checking out this branch and manually testing that I haven't broken anything.

Copy link
Contributor Author

@tcrasset tcrasset Aug 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was able to test that, if the SimplefinAPI returns a valid account/transaction, the code in app-simplefin.js works. I just need someone to test with a real SimplefinAPI.


app.post(
'/status',
handleError(async (req, res) => {
Expand All @@ -36,13 +42,14 @@
if (token == null || token === 'Forbidden') {
throw new Error('No token');
} else {
accessKey = await getAccessKey(token);
accessKey = await simplefinService.getAccessKey(token);
secretsService.set(SecretName.simplefin_accessKey, accessKey);
if (accessKey == null || accessKey === 'Forbidden') {
throw new Error('No access key');
}
}
}
//eslint-disable-next-line @typescript-eslint/no-unused-vars
} catch (error) {
invalidToken(res);
return;
Expand All @@ -53,7 +60,11 @@
const endDate = new Date(now.getFullYear(), now.getMonth() + 1, 1);

try {
const accounts = await getAccounts(accessKey, startDate, endDate);
const accounts = await simplefinService.getAccounts(
accessKey,
startDate,
endDate,
);

res.send({
status: 'ok',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've gone over the code in the frontend that uses the accounts from the server, and made sure that the variables used there had the same name as the ones in Account (from models/account.ts)

Expand Down Expand Up @@ -82,15 +93,17 @@

let results;
try {
results = await getTransactions(accessKey, new Date(startDate));
results = await simplefinService.getTransactions(
accessKey,
new Date(startDate),
);
} catch (e) {
serverDown(e, res);
return;
}

try {
const account =
!results?.accounts || results.accounts.find((a) => a.id === accountId);
const account = results.accounts.find((a) => a.id === accountId);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If no error is caught on line 100, results should always be set, as well as results.accounts (albeit it being an empty list) if there are any error. In that case, results.errors will have an error.

if (!account) {
console.log(
`The account "${accountId}" was not found. Here were the accounts returned:`,
Expand Down Expand Up @@ -131,8 +144,7 @@
const response = {};

const balance = parseInt(account.balance.replace('.', ''));
const date = new Date(account['balance-date'] * 1000)
.toISOString()
const date = account.balanceDate.toISOString()

Check failure on line 147 in src/app-simplefin/app-simplefin.js

View workflow job for this annotation

GitHub Actions / lint

Delete `⏎········`
.split('T')[0];

response.balances = [
Expand Down Expand Up @@ -164,20 +176,20 @@

let dateToUse = 0;

if (trans.posted == 0) {
if (trans.isPending()) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

trans.posted is now a Date, so couldn't compare it to 0. Added a method on Transaction to do this.

newTrans.booked = false;
dateToUse = trans.transacted_at;
dateToUse = trans.transacted_at.getTime() / 1000;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't want to change the logic further down in this function, so I just reverted the transacted_at to use seconds instead of Date objects. getTime() returns milliseconds, so needed to divide by 1000.

} else {
newTrans.booked = true;
dateToUse = trans.posted;
dateToUse = trans.posted.getTime() / 1000;
}

newTrans.bookingDate = new Date(dateToUse * 1000)
.toISOString()
.split('T')[0];

newTrans.date = new Date(dateToUse * 1000).toISOString().split('T')[0];
newTrans.payeeName = trans.payee;
// newTrans.payeeName = trans.payee; TODO: Is this used?
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The payee field appears nowhere in the Simplefin documentation, so I think newTrans.payeeName ends up undefined.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, seems this was changed not too long ago, so I guess the Simplefin docs are not quite up to date :/

newTrans.remittanceInformationUnstructured = trans.description;
newTrans.transactionAmount = { amount: trans.amount, currency: 'USD' };
newTrans.transactionId = trans.id;
Expand Down Expand Up @@ -240,106 +252,3 @@
},
});
}

function parseAccessKey(accessKey) {
let scheme = null;
let rest = null;
let auth = null;
let username = null;
let password = null;
let baseUrl = null;
[scheme, rest] = accessKey.split('//');
[auth, rest] = rest.split('@');
[username, password] = auth.split(':');
baseUrl = `${scheme}//${rest}`;
return {
baseUrl: baseUrl,
username: username,
password: password,
};
}

async function getAccessKey(base64Token) {
const token = Buffer.from(base64Token, 'base64').toString();
const options = {
method: 'POST',
port: 443,
headers: { 'Content-Length': 0 },
};
return new Promise((resolve, reject) => {
const req = https.request(new URL(token), options, (res) => {
res.on('data', (d) => {
resolve(d.toString());
});
});
req.on('error', (e) => {
reject(e);
});
req.end();
});
}

async function getTransactions(accessKey, startDate, endDate) {
const now = new Date();
startDate = startDate || new Date(now.getFullYear(), now.getMonth(), 1);
endDate = endDate || new Date(now.getFullYear(), now.getMonth() + 1, 1);
console.log(
`${startDate.toISOString().split('T')[0]} - ${
endDate.toISOString().split('T')[0]
}`,
);
return await getAccounts(accessKey, startDate, endDate);
}

function normalizeDate(date) {
return (date.valueOf() - date.getTimezoneOffset() * 60 * 1000) / 1000;
}

async function getAccounts(accessKey, startDate, endDate) {
const sfin = parseAccessKey(accessKey);
const options = {
headers: {
Authorization: `Basic ${Buffer.from(
`${sfin.username}:${sfin.password}`,
).toString('base64')}`,
},
};
const params = [];
let queryString = '';
if (startDate) {
params.push(`start-date=${normalizeDate(startDate)}`);
}
if (endDate) {
params.push(`end-date=${normalizeDate(endDate)}`);
}

params.push(`pending=1`);

if (params.length > 0) {
queryString += '?' + params.join('&');
}
return new Promise((resolve, reject) => {
const req = https.request(
new URL(`${sfin.baseUrl}/accounts${queryString}`),
options,
(res) => {
let data = '';
res.on('data', (d) => {
data += d;
});
res.on('end', () => {
try {
resolve(JSON.parse(data));
} catch (e) {
console.log(`Error parsing JSON response: ${data}`);
reject(e);
}
});
},
);
req.on('error', (e) => {
reject(e);
});
req.end();
});
}
7 changes: 7 additions & 0 deletions src/app-simplefin/errors.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export class GenericSimplefinError extends Error {
details: object;
constructor(data = {}) {
super('GoCardless returned error');
this.details = data;
}
}
37 changes: 37 additions & 0 deletions src/app-simplefin/httpClient.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
export interface CustomRequestOptions {
headers?: { [key: string]: string | number };
port: number;
method: string;
}

export interface HttpClient {
request(url: string, options: CustomRequestOptions): Promise<string>;
}

import https from 'https';

export class HttpsClient implements HttpClient {
request(url: string, options: CustomRequestOptions): Promise<string> {
return new Promise((resolve, reject) => {
const req = https.request(new URL(url), options, (response) => {
// reject on bad status
if (response.statusCode < 200 || response.statusCode >= 300) {
return reject(
new Error(`${response.statusCode} ${response.statusMessage}`),
);
}
let data = '';
response.on('data', (d: Buffer) => {
data += d.toString();
});
response.on('end', () => {
resolve(data);
});
});
req.on('error', (e: Error) => {
reject(e);
});
req.end();
});
}
}
Loading
Loading