Merge pull request #2 from CortisolAI/feat/log-cost

Feat/log cost

.github/workflows/ci.yml

@@ -6,9 +6,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
-        poetry-version: [1.3.2, 1.4.2, 1.5.1]
-        os: [ubuntu-18.04, macos-latest, windows-latest]
+        python-version: ['3.11']
+        poetry-version: [1.5.1]
+        os: [ubuntu-18.04]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -157,4 +157,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+.DS_Store
+
+cortisol/cortisollib/templates/locustfile.py

CONTRIBUTING.md

@@ -5,9 +5,9 @@
 
 ## Branching Model
 - Standard Fork & Pull Request Workflow is used in this project
-- Every new functionality should be created in a branch (from develop branch) with name format `feature/new-functionality-name`
-- Every bug should be fixed in a branch (from develop branch) with name format `fix/bug-name`
-- All branches will be merged to develop branch
+- Every new functionality should be created in a branch (from main branch) with name format `feature/new-functionality-name`
+- Every bug should be fixed in a branch (from main branch) with name format `fix/bug-name`
+- All branches will be merged to main branch
 
 ## Setup Environment
 - [Poetry](https://python-poetry.org/) is used as a dependency management tool

README.md

@@ -30,7 +30,33 @@ At the command line:
 
 ## Getting started
 
-TODO
+First things first! We need a RESTful service and so you'll need to do the following steps:
+
+1. Clone this example repo https://github.com/CortisolAI/getting-started-example
+2. `cd getting-started-example`
+3. `mkvirtualenv getting-started-cortisol`
+4. `python -m app.main` which will make the service available at `http://127.0.0.1:8080/`
+
+And, now, it's time to create your first cortisol file. Copy and paste the following in a file named `cortisolfile.py`:
+
+```python
+from locust import task
+
+from cortisol.cortisollib.users import CortisolHttpUser
+
+
+class WebsiteUser(CortisolHttpUser):
+    @task
+    def my_task(self):
+        self.client.get("/")
+
+```
+
+Go to the virtualenv where the cortisol library is installed and run the following command in the terminal. Make sure to change the base path for the `--log-file` argument:
+
+```terminal
+cortisol logs cost-estimate --host http://127.0.0.1:8080 --users 10 --spawn-rate 5 --run-time 10s --cortisol-file cortisolfile.py --log-file /some/path/getting-started-example/cortisol_app.log
+```
 
 ## Commands
 
@@ -50,7 +76,7 @@ Forecast log costs pre-production with Cortisol for Datadog, New Relic, and Graf
 
 ### Example
 
-    cortisol logs cost-estimate --host http://10.20.31.32:8000 --users 100 --spawn-rate 30 --run-time 20m -cortisol-file some_cortisol_file.py
+    cortisol logs cost-estimate --host http://10.20.31.32:8000 --users 10 --spawn-rate 5 --run-time 10s --cortisol-file ./examples/cortisolfile.py --log-file /app/playground_app.log
 
 #### Required Flags - Option 1
 
@@ -72,6 +98,10 @@ All the latter options plus the following in case your application run in a Dock
 
 `-c, --container-id TEXT`      Optional docker container id where your application runs
 
+##### Example
+    cortisol logs cost-estimate --host http://127.0.0.1:8080 --users 100 --spawn-rate 5 --run-time 10s --cortisol-file ./examples/cortisolfile.py --log-file /app/playground_app.log --container-id 1212aa67e530af75b3310e1e5b30261b36844a6748df1d321088c4d48a20ebd0
+
+
 #### Required Flags - Option 3
 
 `--config PATH`                 Path to config file (YAML or JSON) containing the long version of flags from option 1

cortisol/commands/logs.py

@@ -4,25 +4,42 @@
 import typer
 import yaml
 
+from cortisol.cortisollib.log_cost_estimator import get_cost_estimate
+
 app = typer.Typer()
 
 
-def _check_keys_in_file(file_path: Path):
+def _config_reader(file_path: Path):
     try:
         file_content = file_path.read_text()
 
         try:
             data = yaml.safe_load(file_content)
+            return data
         except yaml.YAMLError:
             # If parsing as YAML fails, try parsing as JSON
-            import pdb;pdb.set_trace()
+            import pdb
+
+            pdb.set_trace()
             data = json.loads(file_content)
+            return data
         except json.JSONDecodeError:
             raise ValueError("Invalid YAML or JSON format in the input file.")
     except FileNotFoundError:
         raise FileNotFoundError(f"File not found at path: {file_path}")
 
-    keys_to_check = ['cortisol-file', 'host', 'log-file', 'run-time', 'spawn-rate', 'users']     
+
+def _check_keys_in_file(file_path: Path):
+    data = _config_reader(file_path)
+
+    keys_to_check = [
+        "cortisol-file",
+        "host",
+        "log-file",
+        "run-time",
+        "spawn-rate",
+        "users",
+    ]
     missing_keys = [key for key in keys_to_check if key not in data]
 
     if missing_keys:
@@ -78,11 +95,31 @@ def cost_estimate(
     if config:
         try:
             _check_keys_in_file(config)
+            data = _config_reader(config)
+            cortisol_file = data["cortisol-file"]
+            host = data["host"]
+            log_file = data["log-file"]
+            num_users = data["users"]
+            spawn_rate = data["spawn-rate"]
+            run_time = data["run-time"]
+            container_id = data.get("container-id", "")
         except (FileNotFoundError, ValueError, KeyError) as e:
             typer.echo(str(e))
             raise typer.Abort()
 
     typer.echo("Cost estimate command in the making")
+    if not container_id:
+        container_id = ""
+
+    get_cost_estimate(
+        cortisol_file=cortisol_file,
+        host=host,
+        log_file=log_file,
+        num_users=num_users,
+        spawn_rate=spawn_rate,
+        run_time=run_time,
+        container_id=container_id,
+    )
 
 
 if __name__ == "__main__":

cortisol/cortisollib/calculators.py

@@ -0,0 +1,97 @@
+def datadog_log_cost_calculator(logs_in_gb, log_events_in_million):
+    """
+    Calculate the estimated annual cost of logging using Datadog, including any free tier.
+
+    This function calculates the estimated annual cost of logging using the Datadog platform,
+    considering the amount of data logs in gigabytes and the number of log events in millions.
+
+    Args:
+        logs_in_gb (float): Amount of data logs in gigabytes.
+        log_events_in_million (int): Number of log events in millions.
+
+    Returns:
+        float: Estimated annual cost of logging in USD for log ingestion plus log retention/storage.
+
+    Note:
+        The estimated cost is returned based on the assumption that the Datadog Pro Plan is chosen, billed annually and a 30 day retention is chosen.
+    """
+    cost_per_gb = logs_in_gb * 0.1
+    cost_per_event_million = log_events_in_million * 2.5
+    return cost_per_gb + cost_per_event_million
+
+
+def grafana_log_cost_calculator(logs_in_gb):
+    """
+    Calculate the estimated cost of logs in USD currency based on the given amount of logs in gigabytes,
+    assuming the Grafana Cloud Pro Plan is chosen and including the free tier.
+
+    Args:
+        logs_in_gb (float): The amount of logs in gigabytes.
+
+    Returns:
+        float: The estimated cost of the logs in USD currency for the Grafana Cloud Pro Plan.
+
+    Example:
+        >>> grafana_log_cost_calculator(100)
+        50.0
+
+    Note:
+        The return value is calculated based on the pricing of the Grafana Cloud Pro Plan and does consider the free tier.
+    """
+    if logs_in_gb <= 100:
+        return 0.0
+
+    return (logs_in_gb - 100.0) * 0.5
+
+
+def new_relic_log_cost_calculator(logs_in_gb):
+    """
+    Calculate the estimated cost of logs in USD currency based on the given amount of logs in gigabytes,
+    assuming the New Relic Pro Plan is chosen and including the free tier.
+    Args:
+        logs_in_gb (float): The amount of logs in gigabytes.
+    Returns:
+        float: The estimated cost of the logs in USD currency for the New Relic Pro Plan.
+    Example:
+        >>> new_relic_log_cost_calculator(120)
+        36.0
+    Note:
+        The return value is calculated based on the pricing of the New Relic Pro Plan and does consider the free tier.
+    """
+    if logs_in_gb <= 100:
+        return 0.0
+    return (logs_in_gb - 100.0) * 0.3
+
+
+def format_bytes(file_size):
+    """
+    Converts file size from Bytes to GB
+
+    Args:
+        file_size (float): file size in bytes
+    Returns:
+        float: File size in GB
+    """
+    k = 1024
+    file_size_in_gb = file_size / k**3  # Directly convert to GB (1024^3)
+    return file_size_in_gb
+
+
+def gcp_cloud_logging_log_cost_calculator(logs_in_gb):
+    """
+    Calculate the estimated annual cost of logging using GCP Cloud Logging, including the free tier.
+    This function calculates the estimated annual cost of logging using the Cloud Logging platform,
+    considering the amount of data logs in gigabytes.
+    Args:
+        logs_in_gb (float): Amount of data logs in gigabytes.
+    Returns:
+        float: Estimated annual cost of logging in USD for log ingestion plus log retention/storage.
+    Note:
+        The estimated cost is returned based on the assumption that logs are going to be retainted more than 30 days.
+        No cost is incurred with default retention period https://cloud.google.com/logging/quotas#logs_retention_periods.
+        Google Commitment Agreement is not taken into account.
+    """
+    if logs_in_gb <= 50:
+        return logs_in_gb * 0.01
+
+    return (logs_in_gb - 50.0) * 0.5 + logs_in_gb * 0.01

cortisol/cortisollib/estimators.py

@@ -0,0 +1,22 @@
+def linear_extrapolator(value: float, run_time: float):
+    """
+    Linearly extrapolate a value over a specific run time.
+
+    This function extrapolates a given value over a specified run time of one month (30 days).
+    The extrapolated value is calculated by dividing the given value by the run time in seconds,
+    and then multiplying it by the total number of seconds in 30 days (2592000 seconds).
+
+    Args:
+        value (float): The value to be extrapolated.
+        run_time (float): The duration of the run time in seconds.
+
+    Returns:
+        float: The extrapolated value over the specified run time.
+
+    Example:
+        value_to_extrapolate = 1000
+        test_run_time = 3600  # 1 hour in seconds
+        extrapolated_value = linear_extrapolator(value_to_extrapolate, test_run_time)
+    """
+    extrapolated_value = (2592000.0 * value) / run_time
+    return extrapolated_value