Add Billing Page - CostByMonth - summary of topic per invoice month.

.github/workflows/deploy.yaml

@@ -43,7 +43,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - uses: actions/setup-java@v3
         with:

.github/workflows/lint.yaml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - uses: actions/setup-java@v2
         with:

.github/workflows/test.yaml

@@ -20,7 +20,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - uses: actions/setup-java@v4
         with:

docs/installation.md

@@ -63,7 +63,7 @@ choco install mariadb --version=10.8.3
 - Additional dev requirements are listed in `requirements-dev.txt`.
 - Packages for the sever-side code are listed in `requirements.txt`.
 
-We *STRONGLY* encourage the use of `pyenv` for managing Python versions. Debugging and the server will run on a minimum python version of 3.10. Refer to the [team-docs](https://github.com/populationgenomics/team-docs/blob/main/python.md) for more instructions on how to set this up.
+We *STRONGLY* encourage the use of `pyenv` for managing Python versions. Debugging and the server will run on a minimum python version of 3.11. Refer to the [team-docs](https://github.com/populationgenomics/team-docs/blob/main/python.md) for more instructions on how to set this up.
 
 Use of a virtual environment to contain all requirements is highly recommended:
 

models/models/billing.py

@@ -243,7 +243,11 @@ def to_filter(self) -> BillingFilter:
             # add filters as attributes
             for fk, fv in self.filters.items():
                 # fk is BillColumn, fv is value
-                setattr(billing_filter, fk.value, GenericBQFilter(eq=fv))
+                # if fv is a list, then use IN filter
+                if isinstance(fv, list):
+                    setattr(billing_filter, fk.value, GenericBQFilter(in_=fv))
+                else:
+                    setattr(billing_filter, fk.value, GenericBQFilter(eq=fv))
 
         return billing_filter
 

mypy.ini

@@ -1,7 +1,7 @@
 # Global options:
 
 [mypy]
-python_version = 3.10
+python_version = 3.11
 ; warn_return_any = True
 ; warn_unused_configs = True
 

requirements-dev.txt

@@ -7,7 +7,7 @@ flake8-bugbear
 nest-asyncio
 pre-commit
 pylint
-testcontainers[mariadb]
+testcontainers[mariadb]==3.7.1
 types-PyMySQL
 # some strawberry dependency
 strawberry-graphql[debug-server]==0.206.0

scripts/create_test_subset.py

@@ -201,7 +201,9 @@ def main(
     logger.info(f'Found {len(all_sids)} sample ids in {project}')
 
     # 3. Randomly select from the remaining sgs
-    additional_samples.update(random.sample(all_sids - additional_samples, samples_n))
+    additional_samples.update(
+        random.sample(list(all_sids - additional_samples), samples_n)
+    )
 
     # 4. Query all the samples from the selected sgs
     logger.info(f'Transfering {len(additional_samples)} samples. Querying metadata.')

web/src/Routes.tsx

@@ -9,6 +9,7 @@ import {
     BillingCostByAnalysis,
     BillingInvoiceMonthCost,
     BillingCostByCategory,
+    BillingCostByMonth,
 } from './pages/billing'
 import DocumentationArticle from './pages/docs/Documentation'
 import SampleView from './pages/sample/SampleView'
@@ -49,6 +50,14 @@ const Routes: React.FunctionComponent = () => (
         />
 
         <Route path="/billing/" element={<BillingHome />} />
+        <Route
+            path="/billing/costByMonth"
+            element={
+                <ErrorBoundary>
+                    <BillingCostByMonth />
+                </ErrorBoundary>
+            }
+        />
         <Route path="/billing/invoiceMonthCost" element={<BillingInvoiceMonthCost />} />
         <Route
             path="/billing/costByTime"

web/src/pages/billing/BillingCostByMonth.tsx

@@ -0,0 +1,258 @@
+import * as React from 'react'
+import { useLocation, useNavigate, useSearchParams } from 'react-router-dom'
+import { Button, Card, Grid, Input, Message, Table as SUITable } from 'semantic-ui-react'
+import {
+    BillingApi,
+    BillingColumn,
+    BillingSource,
+    BillingTotalCostQueryModel,
+    BillingTotalCostRecord,
+} from '../../sm-api'
+
+import {
+    getAdjustedDay,
+    generateInvoiceMonths,
+    getCurrentInvoiceMonth,
+    getCurrentInvoiceYearStart,
+} from '../../shared/utilities/formatDates'
+import { IStackedAreaByDateChartData } from '../../shared/components/Graphs/StackedAreaByDateChart'
+import BillingCostByMonthTable from './components/BillingCostByMonthTable'
+import LoadingDucks from '../../shared/components/LoadingDucks/LoadingDucks'
+import generateUrl from '../../shared/utilities/generateUrl'
+import FieldSelector from './components/FieldSelector'
+
+const BillingCostByTime: React.FunctionComponent = () => {
+    const [searchParams] = useSearchParams()
+
+    const [start, setStart] = React.useState<string>(
+        searchParams.get('start') ?? getCurrentInvoiceYearStart()
+    )
+    const [end, setEnd] = React.useState<string>(
+        searchParams.get('end') ?? getCurrentInvoiceMonth()
+    )
+
+    // Data loading
+    const [isLoading, setIsLoading] = React.useState<boolean>(true)
+    const [error, setError] = React.useState<string | undefined>()
+    const [message, setMessage] = React.useState<string | undefined>()
+    const [months, setMonths] = React.useState<string[]>([])
+    const [data, setData] = React.useState<IStackedAreaByDateChartData[]>([])
+
+    // use navigate and update url params
+    const location = useLocation()
+    const navigate = useNavigate()
+
+    const updateNav = (st: string, ed: string) => {
+        const url = generateUrl(location, {
+            start: st,
+            end: ed,
+        })
+        navigate(url)
+    }
+
+    const changeDate = (name: string, value: string) => {
+        let start_update = start
+        let end_update = end
+        if (name === 'start') start_update = value
+        if (name === 'end') end_update = value
+        setStart(start_update)
+        setEnd(end_update)
+        updateNav(start_update, end_update)
+    }
+
+    const convertInvoiceMonth = (invoiceMonth: string, start: Boolean) => {
+        const year = invoiceMonth.substring(0, 4)
+        const month = invoiceMonth.substring(4, 6)
+        if (start) return `${year}-${month}-01`
+        // get last day of month
+        const lastDay = new Date(parseInt(year), parseInt(month), 0).getDate()
+        return `${year}-${month}-${lastDay}`
+    }
+
+    const convertCostCategory = (costCategory: string) => {
+        if (costCategory === 'Cloud Storage') {
+            return 'Storage Cost'
+        }
+        return 'Compute Cost'
+    }
+
+    const getData = (query: BillingTotalCostQueryModel) => {
+        setIsLoading(true)
+        setError(undefined)
+        setMessage(undefined)
+        new BillingApi()
+            .getTotalCost(query)
+            .then((response) => {
+                setIsLoading(false)
+
+                // calc totals per topic, month and category
+                const recTotals: { [key: string]: { [key: string]: number } } = {}
+                const recMonths: string[] = []
+
+                response.data.forEach((item: BillingTotalCostRecord) => {
+                    const { day, cost_category, topic, cost } = item
+                    const ccat = convertCostCategory(cost_category)
+                    if (recMonths.indexOf(day) === -1) {
+                        recMonths.push(day)
+                    }
+                    if (!recTotals[topic]) {
+                        recTotals[topic] = {}
+                    }
+                    if (!recTotals[topic][day]) {
+                        recTotals[topic][day] = {}
+                    }
+                    if (!recTotals[topic][day][ccat]) {
+                        recTotals[topic][day][ccat] = 0
+                    }
+                    recTotals[topic][day][ccat] += cost
+                })
+
+                setMonths(recMonths)
+                setData(recTotals)
+            })
+            .catch((er) => setError(er.message))
+    }
+
+    const messageComponent = () => {
+        if (message) {
+            return (
+                <Message negative onDismiss={() => setError(undefined)}>
+                    {message}
+                </Message>
+            )
+        }
+        if (error) {
+            return (
+                <Message negative onDismiss={() => setError(undefined)}>
+                    {error}
+                    <br />
+                    <Button negative onClick={() => setStart(start)}>
+                        Retry
+                    </Button>
+                </Message>
+            )
+        }
+        if (isLoading) {
+            return (
+                <div>
+                    <LoadingDucks />
+                    <p style={{ textAlign: 'center', marginTop: '5px' }}>
+                        <em>This query takes a while...</em>
+                    </p>
+                </div>
+            )
+        }
+        return null
+    }
+
+    const dataComponent = () => {
+        if (message || error || isLoading) {
+            return null
+        }
+
+        if (!message && !error && !isLoading && (!data || data.length === 0)) {
+            return (
+                <Card
+                    fluid
+                    style={{ padding: '20px', overflowX: 'scroll' }}
+                    id="billing-container-charts"
+                >
+                    No Data
+                </Card>
+            )
+        }
+
+        return (
+            <>
+                <Card
+                    fluid
+                    style={{ padding: '20px', overflowX: 'scroll' }}
+                    id="billing-container-data"
+                >
+                    <BillingCostByMonthTable
+                        start={start}
+                        end={end}
+                        isLoading={isLoading}
+                        data={data}
+                        months={months}
+                    />
+                </Card>
+            </>
+        )
+    }
+
+    const onMonthStart = (event: any, data: any) => {
+        changeDate('start', data.value)
+    }
+
+    const onMonthEnd = (event: any, data: any) => {
+        changeDate('end', data.value)
+    }
+
+    React.useEffect(() => {
+        if (Boolean(start) && Boolean(end)) {
+            // valid selection, retrieve data
+            getData({
+                fields: [BillingColumn.Topic, BillingColumn.CostCategory],
+                start_date: getAdjustedDay(convertInvoiceMonth(start, true), -2),
+                end_date: getAdjustedDay(convertInvoiceMonth(end, false), 3),
+                order_by: { day: false },
+                source: BillingSource.Aggregate,
+                time_periods: 'invoice_month',
+                filters: {
+                    invoice_month: generateInvoiceMonths(start, end),
+                },
+            })
+        } else {
+            // invalid selection,
+            setIsLoading(false)
+            setError(undefined)
+
+            if (start === undefined || start === null || start === '') {
+                setMessage('Please select Start date')
+            } else if (end === undefined || end === null || end === '') {
+                setMessage('Please select End date')
+            }
+        }
+    }, [start, end])
+
+    return (
+        <>
+            <Card fluid style={{ padding: '20px' }} id="billing-container">
+                <h1
+                    style={{
+                        fontSize: 40,
+                    }}
+                >
+                    Cost Across Invoice Months (Topic only)
+                </h1>
+
+                <Grid columns="equal" stackable doubling>
+                    <Grid.Column className="field-selector-label">
+                        <FieldSelector
+                            label="Start"
+                            fieldName={BillingColumn.InvoiceMonth}
+                            onClickFunction={onMonthStart}
+                            selected={start}
+                        />
+                    </Grid.Column>
+
+                    <Grid.Column className="field-selector-label">
+                        <FieldSelector
+                            label="Finish"
+                            fieldName={BillingColumn.InvoiceMonth}
+                            onClickFunction={onMonthEnd}
+                            selected={end}
+                        />
+                    </Grid.Column>
+                </Grid>
+            </Card>
+
+            {messageComponent()}
+
+            {dataComponent()}
+        </>
+    )
+}
+
+export default BillingCostByTime

web/src/pages/billing/BillingInvoiceMonthCost.tsx

@@ -183,7 +183,7 @@ const BillingCurrentCost = () => {
 
     return (
         <>
-            <h1>Billing By Invoice Month</h1>
+            <h1>Cost By Invoice Month</h1>
 
             <Grid columns="equal" stackable doubling>
                 <Grid.Column>