Skip to content

Commit

Permalink
feat: defer reply of the LLM commands (#68)
Browse files Browse the repository at this point in the history 
* feat: defer reply of the LLM commands

These commands have to be resolved within 3 seconds by default, but it's taking way longer
because of how complex LLMs are, so we will need to defer the replies.

* feat: replace array reduce with for await of for brevity
  • Loading branch information
@samhwang
samhwang authored Apr 27, 2024
1 parent 0df3b8d commit f300def
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 43 deletions.
16 changes: 8 additions & 8 deletions src/slash-commands/ask-llm/command.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ describe('ask command', () => {
throw new Error('Invalid');
}
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Invalid model');
});

Expand All @@ -61,11 +61,11 @@ describe('ask command', () => {
}
});
mockChatCompletions.mockRejectedValueOnce(new Error('Synthetic Error.'));
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Error in asking the LLM');
});

Expand Down Expand Up @@ -98,11 +98,11 @@ describe('ask command', () => {
},
] as ChatCompletion.Choice[],
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('No response');
});

Expand Down Expand Up @@ -136,11 +136,11 @@ describe('ask command', () => {
},
] as ChatCompletion.Choice[],
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain(`Q: asdf1234\nA: ${mockAnswer}`);
});
});
Expand Down
17 changes: 10 additions & 7 deletions src/slash-commands/ask-llm/command.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,28 @@ const data = new SlashCommandBuilder()
.addStringOption((option) => option.setName('question').setDescription('Enter your prompt').setRequired(true).setMinLength(10));

export const execute: SlashCommandHandler = async (interaction) => {
await interaction.deferReply();
const model = interaction.options.getString('model', true).trim().toLowerCase();
const question = interaction.options.getString('question', true).trim();
logger.info(`[ask]: Asking ${model} model with prompt: ${question}`);

const findModelOp = Result.safe(() => findModel(model));
if (findModelOp.isErr()) {
logger.info(`[ask]: Invalid model ${model}`);
interaction.reply('Invalid model. Please choose from the available models.');
interaction.editReply('Invalid model. Please choose from the available models.');
return;
}
const supportedModel = findModelOp.unwrap();

logger.info(`[ask]: Asking LLM with prompt: ${question}`);
const answers = await askQuestion(supportedModel, question);
logger.info('[ask]: Got response from LLM', data);
await answers.reduce(async (accum, chunk) => {
await accum;
await interaction.reply(chunk);
return undefined;
}, Promise.resolve(undefined));

logger.info('[ask]: Got response from LLM. Sending to client.', answers);
const [firstChunk, ...chunks] = answers;
await interaction.editReply(firstChunk);
for await (const chunk of chunks) {
interaction.followUp(chunk);
}
};

const command: SlashCommand = {
Expand Down
36 changes: 18 additions & 18 deletions src/slash-commands/review-resume/command.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ describe('review resume command', () => {
throw new Error('Invalid');
}
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Invalid model');
});

Expand All @@ -63,11 +63,11 @@ describe('review resume command', () => {
throw new Error('Invalid');
}
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Invalid URL');
});

Expand All @@ -84,11 +84,11 @@ describe('review resume command', () => {
throw new Error('Invalid');
}
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Error downloading resume');
});

Expand All @@ -105,11 +105,11 @@ describe('review resume command', () => {
throw new Error('Invalid');
}
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Error downloading resume');
});

Expand All @@ -127,11 +127,11 @@ describe('review resume command', () => {
}
});
readerSpy.mockRejectedValueOnce(new Error('Synthetic Error.'));
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Error reading resume');
});

Expand All @@ -149,11 +149,11 @@ describe('review resume command', () => {
}
});
mockChatCompletions.mockRejectedValueOnce(new Error('Synthetic Error.'));
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('Error in asking the LLM');
});

Expand Down Expand Up @@ -186,11 +186,11 @@ describe('review resume command', () => {
},
] as ChatCompletion.Choice[],
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain('No response');
});

Expand Down Expand Up @@ -224,11 +224,11 @@ describe('review resume command', () => {
},
] as ChatCompletion.Choice[],
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain(`${mockAnswer}`);
});

Expand Down Expand Up @@ -262,11 +262,11 @@ describe('review resume command', () => {
},
] as ChatCompletion.Choice[],
});
const respondInput = captor<Parameters<ChatInputCommandInteraction['reply']>['0']>();
const respondInput = captor<Parameters<ChatInputCommandInteraction['editReply']>['0']>();

await execute(mockChatInputInteraction);

expect(mockChatInputInteraction.reply).toBeCalledWith(respondInput);
expect(mockChatInputInteraction.editReply).toBeCalledWith(respondInput);
expect(respondInput.value).toContain(`${mockAnswer}`);
});
});
Expand Down
23 changes: 13 additions & 10 deletions src/slash-commands/review-resume/command.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,23 @@ const data = new SlashCommandBuilder()
.addStringOption((option) => option.setName('url').setDescription('PDF or Google Drive URL').setRequired(true));

export const execute: SlashCommandHandler = async (interaction) => {
await interaction.deferReply();
const model = interaction.options.getString('model', true).trim().toLowerCase();
const url = interaction.options.getString('url', true);
logger.info(`[review-resume]: Reviewing resume from URL: ${url}`);

const findModelOp = Result.safe(() => findModel(model));
if (findModelOp.isErr()) {
logger.info(`[ask]: Invalid model ${model}`);
interaction.reply('Invalid model. Please choose from the available models.');
interaction.editReply('Invalid model. Please choose from the available models.');
return;
}
const supportedModel = findModelOp.unwrap();

const validURL = PDFURL.safeParse(url);
if (!validURL.success) {
logger.info(`[review-resume]: Invalid URL ${url}`);
await interaction.reply('Invalid URL. The URL must end with `.pdf` or must be valid Google Drive URL.');
await interaction.editReply('Invalid URL. The URL must end with `.pdf` or must be valid Google Drive URL.');
return;
}

Expand All @@ -39,15 +40,15 @@ export const execute: SlashCommandHandler = async (interaction) => {
if (downloadOp.isErr()) {
logger.error(`[review-resume]: Error downloading file: ${downloadOp.unwrapErr().message}`);
cleanup(filename);
await interaction.reply('Error downloading resume. Please try again later.');
await interaction.editReply('Error downloading resume. Please try again later.');
return;
}

const readOp = await Result.safe(readPDF(filename));
if (readOp.isErr()) {
logger.error(`[review-resume]: Error reading file: ${readOp.unwrapErr().message}`);
cleanup(filename);
await interaction.reply('Error reading resume. Please try again later.');
await interaction.editReply('Error reading resume. Please try again later.');
return;
}

Expand All @@ -60,13 +61,15 @@ export const execute: SlashCommandHandler = async (interaction) => {
${doc}
`;
logger.info('[review-resume]: Sending review resume request to LLM.');
const answers = await askQuestion(supportedModel, question, false);
logger.info('[review-resume]: Got response from LLM', data);
await answers.reduce(async (accum, chunk) => {
await accum;
await interaction.reply(chunk);
return undefined;
}, Promise.resolve(undefined));

logger.info('[review-resume]: Got response from LLM. Sending to client', answers);
const [firstChunk, ...chunks] = answers;
await interaction.editReply(firstChunk);
for await (const chunk of chunks) {
interaction.followUp(chunk);
}
};

const command: SlashCommand = {
Expand Down

0 comments on commit f300def

Please sign in to comment.