Comments outside expression parentheses (#7873)

<!--
Thank you for contributing to Ruff! To help us out with reviewing,
please consider the following:

- Does this pull request include a summary of the change? (See below.)
- Does this pull request include a descriptive title?
- Does this pull request include references to any relevant issues?
-->

## Summary

Fixes #7448
Fixes #7892

I've removed automatic dangling comment formatting, we're doing manual
dangling comment formatting everywhere anyway (the
assert-all-comments-formatted ensures this) and dangling comments would
break the formatting there.

## Test Plan

New test file.

---------

Co-authored-by: Micha Reiser <micha@reiser.io>

crates/ruff_formatter/src/lib.rs

@@ -575,6 +575,10 @@ where
             context: PhantomData,
         }
     }
+
+    pub fn rule(&self) -> &R {
+        &self.rule
+    }
 }
 
 impl<T, R, O, C> FormatRefWithRule<'_, T, R, C>

crates/ruff_python_ast/src/parenthesize.rs

@@ -4,35 +4,44 @@ use ruff_text_size::{Ranged, TextLen, TextRange};
 use crate::AnyNodeRef;
 use crate::ExpressionRef;
 
-/// Returns the [`TextRange`] of a given expression including parentheses, if the expression is
-/// parenthesized; or `None`, if the expression is not parenthesized.
-pub fn parenthesized_range(
-    expr: ExpressionRef,
-    parent: AnyNodeRef,
-    comment_ranges: &CommentRanges,
-    source: &str,
-) -> Option<TextRange> {
-    // If the parent is a node that brings its own parentheses, exclude the closing parenthesis
-    // from our search range. Otherwise, we risk matching on calls, like `func(x)`, for which
-    // the open and close parentheses are part of the `Arguments` node.
-    //
-    // There are a few other nodes that may have their own parentheses, but are fine to exclude:
-    // - `Parameters`: The parameters to a function definition. Any expressions would represent
-    //   default arguments, and so must be preceded by _at least_ the parameter name. As such,
-    //   we won't mistake any parentheses for the opening and closing parentheses on the
-    //   `Parameters` node itself.
-    // - `Tuple`: The elements of a tuple. The only risk is a single-element tuple (e.g., `(x,)`),
-    //    which must have a trailing comma anyway.
-    let exclusive_parent_end = if parent.is_arguments() {
-        parent.end() - ")".text_len()
+/// Returns an iterator over the ranges of the optional parentheses surrounding an expression.
+///
+/// E.g. for `((f()))` with `f()` as expression, the iterator returns the ranges (1, 6) and (0, 7).
+///
+/// Note that without a parent the range can be inaccurate, e.g. `f(a)` we falsely return a set of
+/// parentheses around `a` even if the parentheses actually belong to `f`. That is why you should
+/// generally prefer [`parenthesized_range`].
+pub fn parentheses_iterator<'a>(
+    expr: ExpressionRef<'a>,
+    parent: Option<AnyNodeRef>,
+    comment_ranges: &'a CommentRanges,
+    source: &'a str,
+) -> impl Iterator<Item = TextRange> + 'a {
+    let right_tokenizer = if let Some(parent) = parent {
+        // If the parent is a node that brings its own parentheses, exclude the closing parenthesis
+        // from our search range. Otherwise, we risk matching on calls, like `func(x)`, for which
+        // the open and close parentheses are part of the `Arguments` node.
+        //
+        // There are a few other nodes that may have their own parentheses, but are fine to exclude:
+        // - `Parameters`: The parameters to a function definition. Any expressions would represent
+        //   default arguments, and so must be preceded by _at least_ the parameter name. As such,
+        //   we won't mistake any parentheses for the opening and closing parentheses on the
+        //   `Parameters` node itself.
+        // - `Tuple`: The elements of a tuple. The only risk is a single-element tuple (e.g., `(x,)`),
+        //    which must have a trailing comma anyway.
+        let exclusive_parent_end = if parent.is_arguments() {
+            parent.end() - ")".text_len()
+        } else {
+            parent.end()
+        };
+        SimpleTokenizer::new(source, TextRange::new(expr.end(), exclusive_parent_end))
     } else {
-        parent.end()
+        SimpleTokenizer::starts_at(expr.end(), source)
     };
 
-    let right_tokenizer =
-        SimpleTokenizer::new(source, TextRange::new(expr.end(), exclusive_parent_end))
-            .skip_trivia()
-            .take_while(|token| token.kind == SimpleTokenKind::RParen);
+    let right_tokenizer = right_tokenizer
+        .skip_trivia()
+        .take_while(|token| token.kind == SimpleTokenKind::RParen);
 
     let left_tokenizer = BackwardsTokenizer::up_to(expr.start(), source, comment_ranges)
         .skip_trivia()
@@ -43,6 +52,16 @@ pub fn parenthesized_range(
     // the `right_tokenizer` is exhausted.
     right_tokenizer
         .zip(left_tokenizer)
-        .last()
         .map(|(right, left)| TextRange::new(left.start(), right.end()))
 }
+
+/// Returns the [`TextRange`] of a given expression including parentheses, if the expression is
+/// parenthesized; or `None`, if the expression is not parenthesized.
+pub fn parenthesized_range(
+    expr: ExpressionRef,
+    parent: AnyNodeRef,
+    comment_ranges: &CommentRanges,
+    source: &str,
+) -> Option<TextRange> {
+    parentheses_iterator(expr, Some(parent), comment_ranges, source).last()
+}

crates/ruff_python_formatter/resources/test/fixtures/ruff/expression/unary.py

@@ -161,3 +161,14 @@
                 + "WARNING: Removing listed files. Do you really want to continue. yes/n)? "
             ):
                 pass
+
+# https://github.com/astral-sh/ruff/issues/7448
+x = (
+    # a
+    not # b
+    # c
+    ( # d
+        # e
+        True
+    )
+)

crates/ruff_python_formatter/resources/test/fixtures/ruff/parentheses/expression_parentheses_comments.py

@@ -0,0 +1,113 @@
+list_with_parenthesized_elements1 = [
+    # comment leading outer
+    (
+        # comment leading inner
+            1 + 2 # comment trailing inner
+    ) # comment trailing outer
+]
+
+list_with_parenthesized_elements2 = [
+    # leading outer
+    (1 + 2)
+]
+list_with_parenthesized_elements3 = [
+    # leading outer
+    (1 + 2) # trailing outer
+]
+list_with_parenthesized_elements4 = [
+    # leading outer
+    (1 + 2), # trailing outer
+]
+list_with_parenthesized_elements5 = [
+    (1), # trailing outer
+    (2), # trailing outer
+]
+
+nested_parentheses1 = (
+    (
+        (
+            1
+        ) # i
+    ) # j
+) # k
+nested_parentheses2 = [
+    (
+        (
+            (
+                1
+            ) # i
+            # i2
+        ) # j
+        # j2
+    ) # k
+    # k2
+]
+nested_parentheses3 = (
+    ( # a
+        ( # b
+            1
+        ) # i
+    ) # j
+) # k
+nested_parentheses4 = [
+    # a
+    ( # b
+        # c
+        ( # d
+            # e
+            ( #f
+                1
+            ) # i
+            # i2
+        ) # j
+        # j2
+    ) # k
+    # k2
+]
+
+
+x = (
+    # unary comment
+    not
+    # in-between comment
+    (
+        # leading inner
+        "a"
+    ),
+    not # in-between comment
+    (
+        # leading inner
+        "b"
+    ),
+    not
+    (  # in-between comment
+        # leading inner
+        "c"
+    ),
+    # 1
+    not # 2
+    ( # 3
+        # 4
+        "d"
+    )
+)
+
+if (
+    # unary comment
+    not
+    # in-between comment
+    (
+            # leading inner
+            1
+    )
+):
+    pass
+
+# Make sure we keep a inside the parentheses
+# https://github.com/astral-sh/ruff/issues/7892
+x = (
+    # a
+    ( # b
+        1
+    )
+)

crates/ruff_python_formatter/resources/test/fixtures/ruff/statement/with.py

@@ -86,10 +86,6 @@
     )
 ): pass
 
-with (a # trailing same line comment
-    # trailing own line comment
-    ) as b: pass
-
 with (
     a # trailing same line comment
     # trailing own line comment

crates/ruff_python_formatter/src/comments/placement.rs

@@ -1878,8 +1878,7 @@ fn handle_lambda_comment<'a>(
     CommentPlacement::Default(comment)
 }
 
-/// Attach trailing end-of-line comments on the operator as dangling comments on the enclosing
-/// node.
+/// Move comment between a unary op and its operand before the unary op by marking them as trailing.
 ///
 /// For example, given:
 /// ```python
@@ -1896,26 +1895,27 @@ fn handle_unary_op_comment<'a>(
     unary_op: &'a ast::ExprUnaryOp,
     locator: &Locator,
 ) -> CommentPlacement<'a> {
-    if comment.line_position().is_own_line() {
-        return CommentPlacement::Default(comment);
-    }
-
-    if comment.start() > unary_op.operand.start() {
-        return CommentPlacement::Default(comment);
-    }
-
-    let tokenizer = SimpleTokenizer::new(
+    let mut tokenizer = SimpleTokenizer::new(
         locator.contents(),
-        TextRange::new(comment.start(), unary_op.operand.start()),
-    );
-    if tokenizer
-        .skip_trivia()
-        .any(|token| token.kind == SimpleTokenKind::LParen)
-    {
-        return CommentPlacement::Default(comment);
+        TextRange::new(unary_op.start(), unary_op.operand.start()),
+    )
+    .skip_trivia();
+    let op_token = tokenizer.next();
+    debug_assert!(op_token.is_some_and(|token| matches!(
+        token.kind,
+        SimpleTokenKind::Tilde
+            | SimpleTokenKind::Not
+            | SimpleTokenKind::Plus
+            | SimpleTokenKind::Minus
+    )));
+    let up_to = tokenizer
+        .find(|token| token.kind == SimpleTokenKind::LParen)
+        .map_or(unary_op.operand.start(), |lparen| lparen.start());
+    if comment.end() < up_to {
+        CommentPlacement::leading(unary_op, comment)
+    } else {
+        CommentPlacement::Default(comment)
     }
-
-    CommentPlacement::dangling(comment.enclosing_node(), comment)
 }
 
 /// Attach an end-of-line comment immediately following an open bracket as a dangling comment on