Skip to content

Commit e032908

Browse files
awildturtokawildturtok
authored
Removes support for postgres (#3913)
PG implementation diverges heavily from other databases. Additionally, we found that PG has pretty mediocre performance and found difficulty adjusting the knobs appropriately. A pretty simple query ran 40s on PG (manual optimizations took it down to 20s), where Clickhouse responded within 2s without any optimizations.
1 parent 2847ebc commit e032908

117 files changed

Lines changed: 459 additions & 2321 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import com.bakdata.conquery.sql.conversion.dialect.DialectBundle;
44
import com.bakdata.conquery.sql.conversion.dialect.clickhouse.ClickhouseDialectBundle;
55
import com.bakdata.conquery.sql.conversion.dialect.hana.HanaDialectBundle;
6-
import com.bakdata.conquery.sql.conversion.dialect.pg.PostgreDialectBundle;
76
import lombok.Getter;
87
import lombok.RequiredArgsConstructor;
98

@@ -16,7 +15,6 @@
1615
@Getter
1716
public enum Dialect {
1817

19-
POSTGRESQL(new PostgreDialectBundle()),
2018
CLICKHOUSE(new ClickhouseDialectBundle()),
2119
HANA(new HanaDialectBundle());
2220

‎backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/DaterangeSelectOrFilter.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
public interface DaterangeSelectOrFilter {
1010

11+
//TODO consider dropping this and require single column to just be same in start and end?
1112
ColumnId getColumn();
1213

1314
ColumnId getStartColumn();

‎backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import com.bakdata.conquery.models.events.MajorTypeId;
1515
import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId;
1616
import com.bakdata.conquery.models.query.filter.RangeFilterNode;
17-
import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDateRangeAggregator;
1817
import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator;
1918
import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
2019
import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator;
@@ -51,9 +50,7 @@ public List<ColumnId> getRequiredColumns() {
5150
@Override
5251
public FilterNode createFilterNode(Range.LongRange value) {
5352
final Column column = getColumn().resolve();
54-
if (column.getType() == MajorTypeId.DATE_RANGE) {
55-
return new RangeFilterNode(value, new CountQuartersOfDateRangeAggregator(column));
56-
}
53+
//TODO missing impl for start/end
5754
return new RangeFilterNode(value, new CountQuartersOfDatesAggregator(column));
5855
}
5956

‎backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,5 @@
11
package com.bakdata.conquery.models.datasets.concepts.filters.specific;
22

3-
import java.util.ArrayList;
4-
import java.util.List;
5-
import javax.annotation.Nullable;
6-
7-
import com.fasterxml.jackson.annotation.JsonAlias;
8-
import jakarta.validation.Valid;
9-
103
import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration;
114
import com.bakdata.conquery.apiv1.frontend.FrontendFilterType;
125
import com.bakdata.conquery.io.cps.CPSType;
@@ -28,9 +21,14 @@
2821
import com.bakdata.conquery.sql.conversion.model.filter.FilterConverter;
2922
import com.fasterxml.jackson.annotation.JsonIgnore;
3023
import io.dropwizard.validation.ValidationMethod;
24+
import jakarta.validation.Valid;
3125
import lombok.Data;
3226
import lombok.extern.slf4j.Slf4j;
3327

28+
import javax.annotation.Nullable;
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
3432
@Data
3533
@Slf4j
3634
@CPSType(id = "DURATION_SUM", base = Filter.class)
@@ -39,7 +37,6 @@ public class DurationSumFilter extends Filter<Range.LongRange> implements Datera
3937
@Valid
4038
@Nullable
4139
private List<ColumnId> distinctBy;
42-
@JsonAlias("dateRangeColumn")
4340
@Nullable
4441
private ColumnId column;
4542
@Nullable
@@ -57,8 +54,7 @@ public List<ColumnId> getRequiredColumns() {
5754
}
5855
if (column != null) {
5956
required.add(column);
60-
}
61-
else {
57+
} else {
6258
required.add(startColumn);
6359
required.add(endColumn);
6460
}
@@ -79,8 +75,8 @@ private boolean hasDistinct() {
7975

8076
@Override
8177
public FilterNode createFilterNode(Range.LongRange value) {
82-
ColumnAggregator<?> aggregator = getColumn() != null ? new DurationSumAggregator(getColumn().resolve())
83-
: new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve());
78+
ColumnAggregator<?> aggregator = isSingleColumnDaterange() ? new DurationSumAggregator(getColumn().resolve())
79+
: new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve());
8480

8581
if (hasDistinct()) {
8682
aggregator = new DistinctValuesWrapperAggregator<>(aggregator, distinctBy.stream().map(ColumnId::resolve).toList());

‎backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,11 @@
11
package com.bakdata.conquery.models.datasets.concepts.select.connector.specific;
22

3-
import java.sql.ResultSet;
4-
import java.sql.SQLException;
5-
import java.util.List;
6-
import javax.annotation.Nullable;
7-
83
import com.bakdata.conquery.io.cps.CPSType;
94
import com.bakdata.conquery.models.datasets.Column;
105
import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter;
116
import com.bakdata.conquery.models.datasets.concepts.select.Select;
12-
import com.bakdata.conquery.models.events.MajorTypeId;
137
import com.bakdata.conquery.models.identifiable.ids.specific.ColumnId;
148
import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
15-
import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDateRangeAggregator;
169
import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator;
1710
import com.bakdata.conquery.models.types.ResultType;
1811
import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator;
@@ -23,6 +16,9 @@
2316
import lombok.NoArgsConstructor;
2417
import lombok.Setter;
2518

19+
import javax.annotation.Nullable;
20+
import java.util.List;
21+
2622
/**
2723
* Entity is included when the number of distinct quarters for all events is within a given range.
2824
* Implementation is specific for DateRanges
@@ -51,12 +47,8 @@ public List<ColumnId> getRequiredColumns() {
5147
@Override
5248
public Aggregator<?> createAggregator() {
5349
final Column column = getColumn().resolve();
54-
final MajorTypeId typeId = column.getType();
55-
return switch (typeId) {
56-
case DATE_RANGE -> new CountQuartersOfDateRangeAggregator(column);
57-
case DATE -> new CountQuartersOfDatesAggregator(column);
58-
default -> throw new IllegalArgumentException(String.format("Column '%s' is not of Date (-Range) Type but '%s'", getColumn(), typeId));
59-
};
50+
//TODO missing case for start/end
51+
return new CountQuartersOfDatesAggregator(column);
6052
}
6153

6254
@Override

‎backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public List<ColumnId> getRequiredColumns() {
4545

4646
@Override
4747
public Aggregator<?> createAggregator() {
48-
// TODO fix this for 2 columns
48+
// TODO Missing case for two columns
4949
return new DateUnionAggregator(getColumn().resolve());
5050
}
5151

‎backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java‎

Lines changed: 58 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -29,62 +29,62 @@
2929
@JsonIgnoreProperties("categorical")
3030
public class DurationSumSelect extends Select implements DaterangeSelectOrFilter {
3131

32-
@Nullable
33-
private ColumnId column;
34-
35-
@Nullable
36-
private ColumnId startColumn, endColumn;
37-
38-
private List<ColumnId> distinctBy;
39-
40-
@Override
41-
public List<ColumnId> getRequiredColumns() {
42-
List<ColumnId> out = new ArrayList<>();
43-
44-
if (column != null) {
45-
out.add(column);
46-
}
47-
else {
48-
out.add(startColumn);
49-
out.add(endColumn);
50-
}
51-
52-
if (hasDistinct()) {
53-
out.addAll(distinctBy);
54-
}
55-
return out;
56-
}
57-
58-
@Override
59-
public ResultSetProcessor.Reader<Integer> createResultSetReader(ResultSetProcessor processor) {
60-
return processor::getInteger;
61-
}
62-
63-
@JsonIgnore
64-
private boolean hasDistinct() {
65-
return distinctBy != null && !distinctBy.isEmpty();
66-
}
67-
68-
@Override
69-
public Aggregator<?> createAggregator() {
70-
ColumnAggregator<?> aggregator = getColumn() != null ? new DurationSumAggregator(getColumn().resolve())
71-
: new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve());
72-
73-
if (!hasDistinct()) {
74-
return aggregator;
75-
}
76-
77-
return new DistinctValuesWrapperAggregator<>(aggregator, distinctBy.stream().map(ColumnId::resolve).toList());
78-
}
79-
80-
@Override
81-
public ResultType getResultType() {
82-
return ResultType.Primitive.INTEGER;
83-
}
84-
85-
@Override
86-
public SelectConverter<DurationSumSelect> createConverter() {
87-
//TODO apply distinctBy (though needs to be done once other branches are merged)
88-
return new DurationSumSqlAggregator();
89-
}
32+
@Nullable
33+
private ColumnId column;
34+
35+
@Nullable
36+
private ColumnId startColumn, endColumn;
37+
38+
private List<ColumnId> distinctBy;
39+
40+
@Override
41+
public List<ColumnId> getRequiredColumns() {
42+
List<ColumnId> out = new ArrayList<>();
43+
44+
if (column != null) {
45+
out.add(column);
46+
} else {
47+
out.add(startColumn);
48+
out.add(endColumn);
49+
}
50+
51+
if (hasDistinct()) {
52+
out.addAll(distinctBy);
53+
}
54+
return out;
55+
}
56+
57+
@Override
58+
public ResultSetProcessor.Reader<Integer> createResultSetReader(ResultSetProcessor processor) {
59+
return processor::getInteger;
60+
}
61+
62+
@JsonIgnore
63+
private boolean hasDistinct() {
64+
return distinctBy != null && !distinctBy.isEmpty();
65+
}
66+
67+
@Override
68+
public Aggregator<?> createAggregator() {
69+
ColumnAggregator<?> aggregator = isSingleColumnDaterange()
70+
? new DurationSumAggregator(getColumn().resolve())
71+
: new TwoColumnDurationSumAggregator(startColumn.resolve(), endColumn.resolve());
72+
73+
if (!hasDistinct()) {
74+
return aggregator;
75+
}
76+
77+
return new DistinctValuesWrapperAggregator<>(aggregator, distinctBy.stream().map(ColumnId::resolve).toList());
78+
}
79+
80+
@Override
81+
public ResultType getResultType() {
82+
return ResultType.Primitive.INTEGER;
83+
}
84+
85+
@Override
86+
public SelectConverter<DurationSumSelect> createConverter() {
87+
//TODO apply distinctBy (though needs to be done once other branches are merged)
88+
return new DurationSumSqlAggregator();
89+
}
9090
}

‎backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java‎

Lines changed: 0 additions & 89 deletions
This file was deleted.

‎backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -149,22 +149,14 @@ private Field<Date>[] collectValidityDateFields(Connector connector) {
149149
List<Field<Date>> validityDates = new ArrayList<>();
150150

151151
for (ValidityDate validityDate : connector.getValidityDates()) {
152-
if (!validityDate.isSingleColumnDaterange()) {
152+
if (validityDate.isSingleColumnDaterange()) {
153+
Column column = validityDate.getColumn().get();
154+
validityDates.add(field(name(column.getName()), Date.class));
155+
} else {
153156
validityDates.add(field(name(validityDate.getStartColumn().getColumn()), Date.class));
154157
validityDates.add(field(name(validityDate.getEndColumn().getColumn()), Date.class));
155-
continue;
156158
}
157159

158-
Column column = validityDate.getColumn().get();
159-
160-
if (column.getType() == MajorTypeId.DATE) {
161-
validityDates.add(field(name(column.getName()), Date.class));
162-
} else if (column.getType() == MajorTypeId.DATE_RANGE) {
163-
Field<Object> rangeField = field(name(column.getName()));
164-
165-
validityDates.add(functionProvider.lower(rangeField));
166-
validityDates.add(functionProvider.upper(rangeField));
167-
}
168160
}
169161
return (Field<Date>[]) validityDates.toArray(Field[]::new);
170162
}

0 commit comments

Comments
 (0)