8000 [BugFix] Fix mv refresh bugs when contains null partition values (backport #59939) by mergify[bot] · Pull Request #60031 · StarRocks/starrocks · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

[BugFix] Fix mv refresh bugs when contains null partition values (backport #59939) #60031

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 20, 2025
Merged

[BugFix] Fix mv refresh bugs when contains null partition values (backport #59939) #60031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d5fb9da
[BugFix] Fix mv refresh bugs when contains null partition values (#59…
LiShuMing Jun 18, 2025
986578a
fix bugs
LiShuMing Jun 20, 2025
ce4c7f8
fix bugs
LiShuMing Jun 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 39 additions & 24 deletions fe/fe-core/src/main/java/com/starrocks/sql/common/SyncPartitionUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,46 +213,61 @@ public static Range<PartitionKey> transferRange(Range<PartitionKey> baseRange,
if (!functionCallExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.DATE_TRUNC)) {
throw new SemanticException("Do not support function: %s", functionCallExpr.getFnName().getFunction());
}
Preconditions.checkState(baseRange.lowerEndpoint().getTypes().size() == 1);

String granularity = ((StringLiteral) functionCallExpr.getChild(0)).getValue().toLowerCase();
// assume expr partition must be DateLiteral and only one partition
LiteralExpr lowerExpr = baseRange.lowerEndpoint().getKeys().get(0);
LiteralExpr upperExpr = baseRange.upperEndpoint().getKeys().get(0);
Preconditions.checkArgument(lowerExpr instanceof DateLiteral);
DateLiteral lowerDate = (DateLiteral) lowerExpr;
LocalDateTime lowerDateTime = lowerDate.toLocalDateTime();
LocalDateTime truncLowerDateTime = getLowerDateTime(lowerDateTime, granularity);

DateLiteral upperDate;
LocalDateTime truncUpperDateTime;
if (upperExpr instanceof MaxLiteral) {
upperDate = new DateLiteral(Type.DATE, true);
truncUpperDateTime = upperDate.toLocalDateTime();
< 10000 span class='blob-code-inner blob-code-marker js-skip-tagsearch' data-code-marker="-"> } else {
upperDate = (DateLiteral) upperExpr;
truncUpperDateTime = getUpperDateTime(upperDate.toLocalDateTime(), granularity);
}

Preconditions.checkState(baseRange.lowerEndpoint().getTypes().size() == 1);
PrimitiveType partitionType = baseRange.lowerEndpoint().getTypes().get(0);

PartitionKey lowerPartitionKey = new PartitionKey();
PartitionKey upperPartitionKey = new PartitionKey();
try {
if (partitionType == PrimitiveType.DATE) {
lowerPartitionKey.pushColumn(new DateLiteral(truncLowerDateTime, Type.DATE), partitionType);
upperPartitionKey.pushColumn(new DateLiteral(truncUpperDateTime, Type.DATE), partitionType);
} else {
lowerPartitionKey.pushColumn(new DateLiteral(truncLowerDateTime, Type.DATETIME), partitionType);
upperPartitionKey.pushColumn(new DateLiteral(truncUpperDateTime, Type.DATETIME), partitionType);
}
DateLiteral lowerDate = transferDateLiteral(lowerExpr, granularity, true);
DateLiteral upperDate = transferDateLiteral(upperExpr, granularity, false);
lowerPartitionKey.pushColumn(lowerDate, partitionType);
upperPartitionKey.pushColumn(upperDate, partitionType);
} catch (AnalysisException e) {
throw new SemanticException("Convert partition with date_trunc expression to date failed, lower:%s, upper:%s",
truncLowerDateTime, truncUpperDateTime);
lowerExpr, upperExpr);
}
return Range.closedOpen(lowerPartitionKey, upperPartitionKey);
}

/**
* Transfer date literal to the lower or upper key of the partition range.
* @param literalExpr: the date literal to be transferred
* @param granularity: the granularity of the partition, such as "day", "month", etc.
* @param isLowerKey: if true, transfer to the lower key of the partition range,
* @return the transferred date literal
* @throws AnalysisException: if the literalExpr is not a date or datetime type,
*/
private static DateLiteral transferDateLiteral(LiteralExpr literalExpr,
String granularity,
boolean isLowerKey) throws AnalysisException {
if (literalExpr == null) {
return null;
}
if (literalExpr.getType() != Type.DATE && literalExpr.getType() != Type.DATETIME) {
throw new SemanticException("Do not support date_trunc for type: %s", literalExpr.getType());
}
DateLiteral dateLiteral = (DateLiteral) literalExpr;
if (dateLiteral.isMinValue()) {
return dateLiteral;
} else if (literalExpr instanceof MaxLiteral) {
return dateLiteral;
}
LocalDateTime dateTime = dateLiteral.toLocalDateTime();
LocalDateTime localDateTime;
if (isLowerKey) {
localDateTime = getLowerDateTime(dateTime, granularity);
} else {
localDateTime = getUpperDateTime(dateTime, granularity);
}
return new DateLiteral(localDateTime, literalExpr.getType());
}

/**
* return all src partition name to intersected dst partition names which the src partition
* is intersected with dst partitions.
Expand Down
224 changes: 217 additions & 7 deletions fe/fe-core/src/test/java/com/starrocks/sql/common/SyncPartitionUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,23 @@
import com.starrocks.analysis.StringLiteral;
import com.starrocks.analysis.TableName;
import com.starrocks.catalog.Column;
import com.starrocks.catalog.FunctionSet;
import com.starrocks.catalog.PartitionKey;
import com.starrocks.catalog.PrimitiveType;
import com.starrocks.catalog.ScalarType;
import com.starrocks.catalog.Type;
import com.starrocks.common.AnalysisException;
import com.starrocks.common.Pair;
import com.starrocks.common.util.DateUtils;
import com.starrocks.sql.analyzer.SemanticException;
import com.starrocks.sql.ast.PartitionValue;
import com.starrocks.sql.common.mv.MVEagerRangePartitionMapper;
import com.starrocks.sql.common.mv.MVLazyRangePartitionMapper;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -70,6 +74,10 @@ private static Range<PartitionKey> createRangeImpl(PartitionValue lowerValue, Pa
return Range.closedOpen(lowerBoundPartitionKey, upperBoundPartitionKey);
}

private static Range<PartitionKey> createRange(DateLiteral lower, DateLiteral upper) throws AnalysisException {
return createRange(lower.getStringValue(), upper.getStringValue());
}

private static Range<PartitionKey> createRange(String lowerBound, String upperBound) throws AnalysisException {
PartitionValue lowerValue = new PartitionValue(lowerBound);
PartitionValue upperValue = new PartitionValue(upperBound);
Expand All @@ -91,7 +99,7 @@ private static Range<PartitionKey> createLessThanRange(String upperBound) throws
return Range.closedOpen(lowerBoundPartitionKey, upperBoundPartitionKey);
}

private static FunctionCallExpr createFuncExpr(String granularity, PrimitiveType type) {
private static FunctionCallExpr createDateTruncFunc(String granularity, PrimitiveType type) {
List<Expr> children = new ArrayList<>();
children.add(new StringLiteral(granularity));
children.add(slotRef);
Expand Down Expand Up @@ -554,7 +562,7 @@ public void testCalcSyncRollupSpecial() throws AnalysisException {

Map<String, Range<PartitionKey>> mvRange = Maps.newHashMap();
RangePartitionDiff diff = SyncPartitionUtils.getRangePartitionDiffOfExpr(baseRange, mvRange,
createFuncExpr("month", PrimitiveType.DATETIME), null);
createDateTruncFunc("month", PrimitiveType.DATETIME), null);
System.out.println(diff);

Map<String, Range<PartitionKey>> adds = diff.getAdds();
Expand All @@ -580,7 +588,7 @@ public void testCalcSyncRollupSpecial() throws AnalysisException {
mvRange = Maps.newHashMap();
mvRange.put("p20200101_20200102", createRange("2020-01-01", "2020-01-02"));
diff = SyncPartitionUtils.getRangePartitionDiffOfExpr(baseRange, mvRange,
createFuncExpr("day", PrimitiveType.DATETIME), null);
createDateTruncFunc("day", PrimitiveType.DATETIME), null);
adds = diff.getAdds();
deletes = diff.getDeletes();

Expand Down Expand Up @@ -650,7 +658,7 @@ public void testCalcSyncRollupPartition() throws AnalysisException {
baseRange.put("p2", createRange("2020-10-12", "2020-11-12"));

RangePartitionDiff diff = SyncPartitionUtils.getRangePartitionDiffOfExpr(baseRange, mvRange,
createFuncExpr(granularity, PrimitiveType.DATETIME), null);
createDateTruncFunc(granularity, PrimitiveType.DATETIME), null);

Map<String, Range<PartitionKey>> adds = diff.getAdds();
Map<String, Range<PartitionKey>> deletes = diff.getDeletes();
Expand All @@ -671,7 +679,7 @@ public void testCalcSyncRollupPartition() throws AnalysisException {
mvRange.put("p202001_202002", createRange("2020-01-01", "2020-02-01"));

diff = SyncPartitionUtils.getRangePartitionDiffOfExpr(baseRange, mvRange,
createFuncExpr(granularity, PrimitiveType.DATETIME), null);
createDateTruncFunc(granularity, PrimitiveType.DATETIME), null);
adds = diff.getAdds();
deletes = diff.getDeletes();
Assert.assertEquals(11, adds.size());
Expand All @@ -682,7 +690,7 @@ public void testCalcSyncRollupPartition() throws AnalysisException {
mvRange = Maps.newHashMap();
mvRange.put("p202005_202006", createRange("2020-05-01", "2020-06-01"));
diff = SyncPartitionUtils.getRangePartitionDiffOfExpr(baseRange, mvRange,
createFuncExpr("month", PrimitiveType.DATETIME), null);
createDateTruncFunc("month", PrimitiveType.DATETIME), null);
adds = diff.getAdds();
deletes = diff.getDeletes();
Assert.assertEquals(1, adds.size());
Expand All @@ -698,7 +706,7 @@ public void testCalcSyncRollupPartition() throws AnalysisException {
mvRange = Maps.newHashMap();
mvRange.put("p202005_202006", createRange("2020-05-01", "2020-06-01"));
diff = SyncPartitionUtils.getRangePartitionDiffOfExpr(baseRange, mvRange,
createFuncExpr("month", PrimitiveType.DATETIME), null);
createDateTruncFunc("month", PrimitiveType.DATETIME), null);
adds = diff.getAdds();
deletes = diff.getDeletes();
Assert.assertEquals(2, adds.size());
Expand Down Expand Up @@ -740,4 +748,206 @@ public void test_getIntersectedPartitions() throws AnalysisException {
res);
}

@Test
public void transferRangeHandlesNonFunctionExpression() throws Exception {
Range<PartitionKey> baseRange = createRange("2020-01-01", "2020-02-01");
Expr nonFunctionExpr = new SlotRef(TABLE_NAME, "column");

Range<PartitionKey> result = SyncPartitionUtils.transferRange(baseRange, nonFunctionExpr);

Assert.assertEquals(baseRange, result);
}

@Test
public void transferRangeHandlesUnsupportedFunction() throws Exception {
Range<PartitionKey> baseRange = createRange("2020-01-01", "2020-02-01");
FunctionCallExpr unsupportedFunction = new FunctionCallExpr("unsupported_function", Lists.newArrayList());

try {
SyncPartitionUtils.transferRange(baseRange, unsupportedFunction);
Assert.fail("Expected SemanticException to be thrown");
} catch (SemanticException e) {
Assert.assertTrue(e.getMessage().contains("Do not support function"));
}
}

private Pair<String, String> getDateTruncFuncTransform(Range<PartitionKey> range, String granularity) {
FunctionCallExpr dateTruncFunction = createDateTruncFunc(granularity, PrimitiveType.DATE);
Range<PartitionKey> result = SyncPartitionUtils.transferRange(range, dateTruncFunction);
String lower = result.lowerEndpoint().getKeys().get(0).getStringValue();
String upper = result.upperEndpoint().getKeys().get(0).getStringValue();
return Pair.create(lower, upper);
}
@Test
public void transferRangeHandlesDateTruncFunction() throws AnalysisException {
Range<PartitionKey> baseRange = createRange("2020-01-01 12:34:56", "2020-01-02 12:34:56");

{
// MINUTE
Pair<String, String> result = getDateTruncFuncTransform(baseRange, "MINUTE");
Assert.assertEquals("2020-01-01 12:34:00", result.first);
Assert.assertEquals("2020-01-02 12:35:00", result.second);
}

{
// HOUR
Pair<String, String> result = getDateTruncFuncTransform(baseRange, "HOUR");
Assert.assertEquals("2020-01-01 12:00:00", result.first);
Assert.assertEquals("2020-01-02 13:00:00", result.second);
}

{
// day
Pair<String, String> result = getDateTruncFuncTransform(baseRange, "day");
Assert.assertEquals("2020-01-01 00:00:00", result.first);
Assert.assertEquals("2020-01-03 00:00:00", result.second);
}

{
// WEEK
Pair<String, String> result = getDateTruncFuncTransform(baseRange, "WEEK");
Assert.assertEquals("2019-12-30 00:00:00", result.first);
Assert.assertEquals("2020-01-09 00:00:00", result.second);
}

{
// MONTH
Pair<String, String> result = getDateTruncFuncTransform(baseRange, "MONTH");
Assert.assertEquals("2020-01-01 12:34:56", result.first);
Assert.assertEquals("2020-02-01 12:34:56", result.second);
}
{
// QUARTER
Pair<String, String> result = getDateTruncFuncTransform(baseRange, "QUARTER");
Assert.assertEquals("2020-01-01 12:34:56", result.first);
Assert.assertEquals("2020-04-01 12:34:56", result.second);
}
{
// YEAR
Pair<String, String> result = getDateTruncFuncTransform(baseRange, "YEAR");
Assert.assertEquals("2020-01-01 12:34:56", result.first);
Assert.assertEquals("2021-01-01 12:34:56", result.second);
}
}

@Test
public void transferRangeHandlesStr2DateFunction() throws AnalysisException {
Range<PartitionKey> baseRange = createRange("2020-01-01", "2020-02-01");
FunctionCallExpr str2DateFunction = new FunctionCallExpr(FunctionSet.STR2DATE, Lists.newArrayList());

Range<PartitionKey> result = SyncPartitionUtils.transferRange(baseRange, str2DateFunction);

Assert.assertEquals(baseRange, result);
}

@Test
public void transferRangeHandlesInvalidGranularity() throws Exception {
Range<PartitionKey> baseRange = createRange("2020-01-01", "2020-02-01");
FunctionCallExpr invalidGranularityFunction = createDateTruncFunc("invalid_granularity", PrimitiveType.DATE);

try {
SyncPartitionUtils.transferRange(baseRange, invalidGranularityFunction);
Assert.fail("Expected SemanticException to be thrown");
} catch (SemanticException e) {
Assert.assertTrue(e.getMessage().contains("Do not support in date_trunc format string"));
}
}

private DateLiteral plusDay(DateLiteral dateLiteral, int diff) {
try {
LocalDateTime date = dateLiteral.toLocalDateTime().plusDays(diff);
return new DateLiteral(date, Type.DATE);
} catch (Exception e) {
Assert.fail();
return null;
}
}

@Test
public void transferRangeHandlesMinValue() throws AnalysisException {
final DateLiteral minValue = DateLiteral.createMinValue(Type.DATE);
Range<PartitionKey> range = createRange(minValue, plusDay(minValue, 1));
{
Expr partitionExpr = new SlotRef(TABLE_NAME, "column");
Range<PartitionKey> result = SyncPartitionUtils.transferRange(range, partitionExpr);
Assert.assertEquals(result, range);
}
{
FunctionCallExpr partitionExpr = createDateTruncFunc("day", PrimitiveType.DATE);
Range<PartitionKey> result = SyncPartitionUtils.transferRange(range, partitionExpr);
Assert.assertEquals(result, range);
}
{
FunctionCallExpr partitionExpr = createDateTruncFunc("month", PrimitiveType.DATE);
Range<PartitionKey> result = SyncPartitionUtils.transferRange(range, partitionExpr);
Assert.assertEquals("0000-01-01 00:00:00", result.lowerEndpoint().getKeys().get(0).getStringValue());
Assert.assertEquals("0000-02-01 00:00:00", result.upperEndpoint().getKeys().get(0).getStringValue());
}
}

@Test
public void transferRangeHandlesMaxValue() throws AnalysisException {
final DateLiteral maxValue = DateLiteral.createMaxValue(Type.DATE);
Range<PartitionKey> range = createRange(plusDay(maxValue, -1), maxValue);
{
Expr partitionExpr = new SlotRef(TABLE_NAME, "column");
Range<PartitionKey> result = SyncPartitionUtils.transferRange(range, partitionExpr);
Assert.assertEquals(result, range);
}
{
FunctionCallExpr partitionExpr = createDateTruncFunc("day", PrimitiveType.DATE);
Range<PartitionKey> result = SyncPartitionUtils.transferRange(range, partitionExpr);
Assert.assertEquals(result, range);
}
{
FunctionCallExpr partitionExpr = createDateTruncFunc("month", PrimitiveType.DATE);
Range<PartitionKey> result = SyncPartitionUtils.transferRange(range, partitionExpr);
Assert.assertEquals("9999-12-01 00:00:00", result.lowerEndpoint().getKeys().get(0).getStringValue());
Assert.assertEquals("9999-12-31 00:00:00", result.upperEndpoint().getKeys().get(0).getStringValue());
}
}

@Test
public void transferRangeHandlesNullExpression() throws AnalysisException {
Range<PartitionKey> baseRange = createRange("2020-01-01", "2020-02-01");
Range<PartitionKey> result = SyncPartitionUtils.transferRange(baseRange, null);
Assert.assertEquals(baseRange, result);
}

@Test
public void getIntersectedPartitionsHandlesNonOverlappingRanges() throws AnalysisException {
Map<String, Range<PartitionKey>> srcRangeMap = Maps.newHashMap();
srcRangeMap.put("p202001", createRange("2020-01-01", "2020-02-01"));
Map<String, Range<PartitionKey>> dstRangeMap = Maps.newHashMap();
dstRangeMap.put("p202002", createRange("2020-02-01", "2020-03-01"));
Map<String, Set<String>> partitionRefMap = SyncPartitionUtils.getIntersectedPartitions(srcRangeMap, dstRangeMap);
Assert.assertTrue(partitionRefMap.get("p202001").isEmpty());
}

@Test
public void getIntersectedPartitionsHandlesIdenticalRanges() throws AnalysisException {
Map<String, Range<PartitionKey>> srcRangeMap = Maps.newHashMap();
srcRangeMap.put("p202001", createRange("2020-01-01", "2020-02-01"));

Map<String, Range<PartitionKey>> dstRangeMap = Maps.newHashMap();
dstRangeMap.put("p202001", createRange("2020-01-01", "2020-02-01"));

Map<String, Set<String>> partitionRefMap = SyncPartitionUtils.getIntersectedPartitions(srcRangeMap, dstRangeMap);

Assert.assertEquals(1, partitionRefMap.size());
Assert.assertTrue(partitionRefMap.get("p202001").contains("p202001"));
}

@Test
public void transferRangeHandlesMaxValueRange() throws AnalysisException {
final DateLiteral maxValue = DateLiteral.createMaxValue(Type.DATE);
final Range<PartitionKey> maxValueRange = createRange(plusDay(maxValue, -1), maxValue);
FunctionCallExpr dateTruncFunction = createDateTruncFunc("year", PrimitiveType.DATE);
Range<PartitionKey> result = SyncPartitionUtils.transferRange(maxValueRange, dateTruncFunction);

Assert.assertEquals("9999-01-01 00:00:00",
result.lowerEndpoint().getKeys().get(0).getStringValue());
Assert.assertEquals("9999-12-31 00:00:00",
result.upperEndpoint().getKeys().get(0).getStringValue());
}
}
Loading
0