forked from apache/lucene-solr
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
LUCENE-9384: Backport for field sort optimization (apache#1610)
Backport for: LUCENE-9280: Collectors to skip noncompetitive documents (apache#1351) Similar how scorers can update their iterators to skip non-competitive documents, collectors and comparators should also provide and update iterators that allow them to skip non-competive documents. To enable sort optimization for numeric sort fields, the following needs to be done: 1) the field should be indexed with both doc_values and points, that must have the same field name and same data 2) SortField#setCanUsePoints must be set 3) totalHitsThreshold should not be set to max value.
- Loading branch information
1 parent
648fd90
commit d06fb96
Showing
16 changed files
with
1,032 additions
and
64 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
lucene/core/src/java/org/apache/lucene/search/FilteringFieldComparator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.lucene.search; | ||
|
|
||
| import org.apache.lucene.index.LeafReaderContext; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * A wrapper over {@code FieldComparator} that provides a leaf comparator that can filter non-competitive docs. | ||
| */ | ||
| abstract class FilteringFieldComparator<T> extends FieldComparator<T> { | ||
| protected final FieldComparator<T> in; | ||
| protected final boolean reverse; | ||
| // singleSort is true, if sort is based on a single sort field. As there are no other sorts configured | ||
| // as tie breakers, we can filter out docs with equal values. | ||
| protected final boolean singleSort; | ||
| protected boolean hasTopValue = false; | ||
|
|
||
| public FilteringFieldComparator(FieldComparator<T> in, boolean reverse, boolean singleSort) { | ||
| this.in = in; | ||
| this.reverse = reverse; | ||
| this.singleSort = singleSort; | ||
| } | ||
|
|
||
| @Override | ||
| public abstract FilteringLeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException; | ||
|
|
||
| @Override | ||
| public int compare(int slot1, int slot2) { | ||
| return in.compare(slot1, slot2); | ||
| } | ||
|
|
||
| @Override | ||
| public T value(int slot) { | ||
| return in.value(slot); | ||
| } | ||
|
|
||
| @Override | ||
| public void setTopValue(T value) { | ||
| in.setTopValue(value); | ||
| hasTopValue = true; | ||
| } | ||
|
|
||
| @Override | ||
| public int compareValues(T first, T second) { | ||
| return in.compareValues(first, second); | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * Try to wrap a given field comparator to add to it a functionality to skip over non-competitive docs. | ||
| * If for the given comparator the skip functionality is not implemented, return the comparator itself. | ||
| * @param comparator – comparator to wrap | ||
| * @param reverse – if this sort is reverse | ||
| * @param singleSort – true if this sort is based on a single field and there are no other sort fields for tie breaking | ||
| * @return comparator wrapped as a filtering comparator or the original comparator if the filtering functionality | ||
| * is not implemented for it | ||
| */ | ||
| public static FieldComparator<?> wrapToFilteringComparator(FieldComparator<?> comparator, boolean reverse, boolean singleSort) { | ||
| Class<?> comparatorClass = comparator.getClass(); | ||
| if (comparatorClass == FieldComparator.LongComparator.class){ | ||
| return new FilteringNumericComparator<>((FieldComparator.LongComparator) comparator, reverse, singleSort); | ||
| } | ||
| if (comparatorClass == FieldComparator.IntComparator.class){ | ||
| return new FilteringNumericComparator<>((FieldComparator.IntComparator) comparator, reverse, singleSort); | ||
| } | ||
| if (comparatorClass == FieldComparator.DoubleComparator.class){ | ||
| return new FilteringNumericComparator<>((FieldComparator.DoubleComparator) comparator, reverse, singleSort); | ||
| } | ||
| if (comparatorClass == FieldComparator.FloatComparator.class){ | ||
| return new FilteringNumericComparator<>((FieldComparator.FloatComparator) comparator, reverse, singleSort); | ||
| } | ||
| return comparator; | ||
| } | ||
|
|
||
| } | ||
|
|
||
|
|
39 changes: 39 additions & 0 deletions
39
lucene/core/src/java/org/apache/lucene/search/FilteringLeafFieldComparator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.lucene.search; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * Decorates a wrapped LeafFieldComparator to add a functionality to skip over non-competitive docs. | ||
| * FilteringLeafFieldComparator provides two additional functions to a LeafFieldComparator: | ||
| * {@code competitiveIterator()} and {@code setCanUpdateIterator()}. | ||
| */ | ||
| public interface FilteringLeafFieldComparator extends LeafFieldComparator { | ||
| /** | ||
| * Returns a competitive iterator | ||
| * @return an iterator over competitive docs that are stronger than already collected docs | ||
| * or {@code null} if such an iterator is not available for the current segment. | ||
| */ | ||
| DocIdSetIterator competitiveIterator() throws IOException; | ||
|
|
||
| /** | ||
| * Informs this leaf comparator that it is allowed to start updating its competitive iterator. | ||
| * This method is called from a collector when queue becomes full and threshold is reached. | ||
| */ | ||
| void setCanUpdateIterator() throws IOException; | ||
| } |
52 changes: 52 additions & 0 deletions
52
lucene/core/src/java/org/apache/lucene/search/FilteringNumericComparator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.lucene.search; | ||
|
|
||
| import org.apache.lucene.index.LeafReaderContext; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * A wrapper over {@code NumericComparator} that provides a leaf comparator that can filter non-competitive docs. | ||
| */ | ||
| class FilteringNumericComparator<T extends Number> extends FilteringFieldComparator<T> { | ||
| public FilteringNumericComparator(NumericComparator<T> in, boolean reverse, boolean singleSort) { | ||
| super(in, reverse, singleSort); | ||
| } | ||
|
|
||
| @Override | ||
| public final FilteringLeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { | ||
| LeafFieldComparator inLeafComparator = in.getLeafComparator(context); | ||
| Class<?> comparatorClass = inLeafComparator.getClass(); | ||
| if (comparatorClass == FieldComparator.LongComparator.class) { | ||
| return new FilteringNumericLeafComparator.FilteringLongLeafComparator((FieldComparator.LongComparator) inLeafComparator, context, | ||
| ((LongComparator) inLeafComparator).field, reverse, singleSort, hasTopValue); | ||
| } if (comparatorClass == FieldComparator.IntComparator.class) { | ||
| return new FilteringNumericLeafComparator.FilteringIntLeafComparator((FieldComparator.IntComparator) inLeafComparator, context, | ||
| ((IntComparator) inLeafComparator).field, reverse, singleSort, hasTopValue); | ||
| } else if (comparatorClass == FieldComparator.DoubleComparator.class) { | ||
| return new FilteringNumericLeafComparator.FilteringDoubleLeafComparator((FieldComparator.DoubleComparator) inLeafComparator, context, | ||
| ((DoubleComparator) inLeafComparator).field, reverse, singleSort, hasTopValue); | ||
| } else if (comparatorClass == FieldComparator.FloatComparator.class) { | ||
| return new FilteringNumericLeafComparator.FilteringFloatLeafComparator((FieldComparator.FloatComparator) inLeafComparator, context, | ||
| ((FloatComparator) inLeafComparator).field, reverse, singleSort, hasTopValue); | ||
| } else { | ||
| throw new IllegalStateException("Unexpected numeric class of ["+ comparatorClass + "] for [FieldComparator]!"); | ||
| } | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.