Setup Funnel Unordered persons and Testing

ee/clickhouse/queries/funnels/base.py

@@ -272,6 +272,36 @@ def _build_filters(self, entity: Entity, index: int) -> str:
             return prop_filters
         return ""
 
+    def _get_funnel_person_step_condition(self):
+        step_num = self._filter.funnel_step
+        max_steps = len(self._filter.entities)
+
+        if step_num is None:
+            raise ValueError("funnel_step should not be none")
+
+        if step_num >= 0:
+            self.params.update({"step_num": [i for i in range(step_num, max_steps + 1)]})
+            return "steps IN %(step_num)s"
+        else:
+            self.params.update({"step_num": abs(step_num) - 1})
+            return "steps = %(step_num)s"
+
+    def _get_count_columns(self, max_steps: int):
+        cols: List[str] = []
+
+        for i in range(max_steps):
+            cols.append(f"countIf(steps = {i + 1}) step_{i + 1}")
+
+        return ", ".join(cols)
+
+    def _get_step_time_avgs(self, max_steps: int):
+        conditions: List[str] = []
+        for i in range(1, max_steps):
+            conditions.append(f"avg(step_{i}_average_conversion_time) step_{i}_average_conversion_time")
+
+        formatted = ", ".join(conditions)
+        return f", {formatted}" if formatted else ""
+
     @abstractmethod
     def get_query(self, format_properties):
         pass

ee/clickhouse/queries/funnels/funnel.py

@@ -11,28 +11,24 @@ def get_query(self, format_properties):
 
 class ClickhouseFunnelNew(ClickhouseFunnelBase):
     def get_query(self, format_properties):
-        return self.get_step_counts_query()
 
-    def get_step_counts_query(self):
-
-        steps_per_person_query = self._get_steps_per_person_query()
+        steps_per_person_query = self.get_step_counts_query()
         max_steps = len(self._filter.entities)
 
         return f"""
         SELECT {self._get_count_columns(max_steps)} {self._get_step_time_avgs(max_steps)} FROM (
-            SELECT person_id, max(steps) AS furthest {self._get_step_time_avgs(max_steps)} FROM (
                 {steps_per_person_query}
-            ) GROUP BY person_id
         ) SETTINGS allow_experimental_window_functions = 1
         """
 
-    def _get_count_columns(self, max_steps: int):
-        cols: List[str] = []
-
-        for i in range(max_steps):
-            cols.append(f"countIf(furthest = {i + 1}) step_{i + 1}")
+    def get_step_counts_query(self):
+        steps_per_person_query = self._get_steps_per_person_query()
+        max_steps = len(self._filter.entities)
 
-        return ", ".join(cols)
+        return f"""SELECT person_id, max(steps) AS steps {self._get_step_time_avgs(max_steps)} FROM (
+            {steps_per_person_query}
+        ) GROUP BY person_id
+        """
 
     def _format_results(self, results):
         # Format of this is [step order, person count (that reached that step), array of person uuids]
@@ -61,12 +57,3 @@ def _get_breakdown_prop(self) -> str:
             return ", prop"
         else:
             return ""
-
-    # TODO: include in the inner query to handle time to convert
-    def _get_step_time_avgs(self, max_steps: int):
-        conditions: List[str] = []
-        for i in range(1, max_steps):
-            conditions.append(f"avg(step_{i}_average_conversion_time) step_{i}_average_conversion_time")
-
-        formatted = ", ".join(conditions)
-        return f", {formatted}" if formatted else ""

ee/clickhouse/queries/funnels/funnel_persons.py

@@ -1,31 +1,16 @@
-from ee.clickhouse.queries.funnels.base import ClickhouseFunnelBase
+from ee.clickhouse.queries.funnels.funnel import ClickhouseFunnelNew
 from ee.clickhouse.sql.funnels.funnel import FUNNEL_PERSONS_BY_STEP_SQL
 from posthog.models import Person
 
 
-class ClickhouseFunnelPersons(ClickhouseFunnelBase):
+class ClickhouseFunnelPersons(ClickhouseFunnelNew):
     def get_query(self, format_properties):
-        steps_per_person_query = self._get_steps_per_person_query()
         return FUNNEL_PERSONS_BY_STEP_SQL.format(
             **format_properties,
-            steps_per_person_query=steps_per_person_query,
+            steps_per_person_query=self.get_step_counts_query(),
             persons_steps=self._get_funnel_person_step_condition()
         )
 
-    def _get_funnel_person_step_condition(self):
-        step_num = self._filter.funnel_step
-        max_steps = len(self._filter.entities)
-
-        if step_num is None:
-            raise ValueError("funnel_step should not be none")
-
-        if step_num >= 0:
-            self.params.update({"step_num": [i for i in range(step_num, max_steps + 1)]})
-            return "steps IN %(step_num)s"
-        else:
-            self.params.update({"step_num": abs(step_num) - 1})
-            return "steps = %(step_num)s"
-
     def _format_results(self, results):
         people = Person.objects.filter(team_id=self._team.pk, uuid__in=[val[0] for val in results])
 

ee/clickhouse/queries/funnels/funnel_trends.py

@@ -1,7 +1,7 @@
 from datetime import date, datetime, timedelta
 from typing import Union
 
-from ee.clickhouse.queries.funnels.funnel import ClickhouseFunnelNew
+from ee.clickhouse.queries.funnels.base import ClickhouseFunnelBase
 from ee.clickhouse.queries.util import get_time_diff, get_trunc_func_ch
 from posthog.constants import BREAKDOWN
 from posthog.models.filters.filter import Filter
@@ -15,7 +15,7 @@
 HUMAN_READABLE_TIMESTAMP_FORMAT = "%a. %-d %b"
 
 
-class ClickhouseFunnelTrends(ClickhouseFunnelNew):
+class ClickhouseFunnelTrends(ClickhouseFunnelBase):
     """
     ## Funnel trends assumptions
 

ee/clickhouse/queries/funnels/funnel_unordered.py

@@ -24,7 +24,14 @@ class ClickhouseFunnelUnordered(ClickhouseFunnelBase):
     """
 
     def get_query(self, format_properties):
-        return self.get_step_counts_query()
+
+        max_steps = len(self._filter.entities)
+
+        return f"""
+        SELECT {self._get_count_columns(max_steps)} {self._get_step_time_avgs(max_steps)} FROM (
+            {self.get_step_counts_query()}
+        ) SETTINGS allow_experimental_window_functions = 1
+        """
 
     def get_step_counts_query(self):
 
@@ -45,7 +52,7 @@ def get_step_counts_query(self):
             """
 
             formatted_query = f"""
-                SELECT *, {sorting_condition} AS steps FROM (
+                SELECT *, {sorting_condition} AS steps {self._get_step_times(max_steps)} FROM (
                         {inner_query}
                     ) WHERE step_0 = 1"""
 
@@ -56,11 +63,9 @@ def get_step_counts_query(self):
         union_formatted_query = " UNION ALL ".join(union_queries)
 
         return f"""
-        SELECT furthest, count(1), groupArray(100)(person_id) FROM (
-            SELECT person_id, max(steps) AS furthest FROM (
+        SELECT person_id, max(steps) AS steps {self._get_step_time_avgs(max_steps)} FROM (
                 {union_formatted_query}
-            ) GROUP BY person_id
-        ) GROUP BY furthest SETTINGS allow_experimental_window_functions = 1
+        ) GROUP BY person_id
         """
 
     def get_sorting_condition(self, max_steps: int):
@@ -72,3 +77,25 @@ def get_sorting_condition(self, max_steps: int):
             )
 
         return f"arraySum([{','.join(basic_conditions)}, 1])"
+
+    # TODO: copied from funnel.py. Once the new funnel query replaces old one, the base format_results function can use this
+    def _format_results(self, results):
+        # Format of this is [step order, person count (that reached that step), array of person uuids]
+        steps = []
+        total_people = 0
+
+        for step in reversed(self._filter.entities):
+
+            if results[0] and len(results[0]) > 0:
+                total_people += results[0][step.order]
+
+            serialized_result = self._serialize_step(step, total_people, [])
+            if step.order > 0:
+                serialized_result.update(
+                    {"average_conversion_time": results[0][step.order + len(self._filter.entities) - 1]}
+                )
+            else:
+                serialized_result.update({"average_conversion_time": None})
+            steps.append(serialized_result)
+
+        return steps[::-1]  #  reverse

ee/clickhouse/queries/funnels/funnel_unordered_persons.py

@@ -0,0 +1,19 @@
+from ee.clickhouse.queries.funnels.funnel_unordered import ClickhouseFunnelUnordered
+from ee.clickhouse.sql.funnels.funnel import FUNNEL_PERSONS_BY_STEP_SQL
+from posthog.models import Person
+
+
+class ClickhouseFunnelUnorderedPersons(ClickhouseFunnelUnordered):
+    def get_query(self, format_properties):
+        return FUNNEL_PERSONS_BY_STEP_SQL.format(
+            **format_properties,
+            steps_per_person_query=self.get_step_counts_query(),
+            persons_steps=self._get_funnel_person_step_condition()
+        )
+
+    def _format_results(self, results):
+        people = Person.objects.filter(team_id=self._team.pk, uuid__in=[val[0] for val in results])
+
+        from posthog.api.person import PersonSerializer
+
+        return PersonSerializer(people, many=True).data