Skip to content

Commit 89730b1

Browse files
CaideyipiCaideyipi
authored
Optimized the overall performance of IoTDB & Fixed the NPE in LimitOperatorTest (#17664)
* Opt * Update UnclosedFileScanHandleImpl.java * Update StorageEngine.java * Update ClosedFileScanHandleImpl.java * column index * spt * Address performance review comments * fix
1 parent 44d4f6d commit 89730b1

17 files changed

Lines changed: 422 additions & 356 deletions

File tree

‎iotdb-client/client-cpp/src/main/SessionDataSet.cpp‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -246,38 +246,39 @@ const std::vector<std::string>& SessionDataSet::DataIterator::getColumnTypeList(
246246

247247
shared_ptr<RowRecord> SessionDataSet::constructRowRecordFromValueArray() {
248248
std::vector<Field> outFields;
249-
for (int i = iotdbRpcDataSet_->getValueColumnStartIndex(); i < iotdbRpcDataSet_->getColumnSize();
250-
i++) {
249+
const int32_t valueColumnStartIndex = iotdbRpcDataSet_->getValueColumnStartIndex();
250+
const int32_t columnSize = iotdbRpcDataSet_->getColumnSize();
251+
outFields.reserve(columnSize - valueColumnStartIndex);
252+
for (int32_t columnIndex = valueColumnStartIndex + 1; columnIndex <= columnSize; ++columnIndex) {
251253
Field field;
252-
std::string columnName = iotdbRpcDataSet_->getColumnNameList().at(i);
253-
if (!iotdbRpcDataSet_->isNullByColumnName(columnName)) {
254-
TSDataType::TSDataType dataType = iotdbRpcDataSet_->getDataType(columnName);
254+
if (!iotdbRpcDataSet_->isNullByIndex(columnIndex)) {
255+
TSDataType::TSDataType dataType = iotdbRpcDataSet_->getDataTypeByIndex(columnIndex);
255256
field.dataType = dataType;
256257
switch (dataType) {
257258
case TSDataType::BOOLEAN:
258-
field.boolV = iotdbRpcDataSet_->getBoolean(columnName);
259+
field.boolV = iotdbRpcDataSet_->getBooleanByIndex(columnIndex).value();
259260
break;
260261
case TSDataType::INT32:
261-
field.intV = iotdbRpcDataSet_->getInt(columnName);
262+
field.intV = iotdbRpcDataSet_->getIntByIndex(columnIndex).value();
262263
break;
263264
case TSDataType::DATE:
264-
field.dateV = iotdbRpcDataSet_->getDate(columnName);
265+
field.dateV = iotdbRpcDataSet_->getDateByIndex(columnIndex).value();
265266
break;
266267
case TSDataType::INT64:
267268
case TSDataType::TIMESTAMP:
268-
field.longV = iotdbRpcDataSet_->getLong(columnName);
269+
field.longV = iotdbRpcDataSet_->getLongByIndex(columnIndex).value();
269270
break;
270271
case TSDataType::FLOAT:
271-
field.floatV = iotdbRpcDataSet_->getFloat(columnName);
272+
field.floatV = iotdbRpcDataSet_->getFloatByIndex(columnIndex).value();
272273
break;
273274
case TSDataType::DOUBLE:
274-
field.doubleV = iotdbRpcDataSet_->getDouble(columnName);
275+
field.doubleV = iotdbRpcDataSet_->getDoubleByIndex(columnIndex).value();
275276
break;
276277
case TSDataType::TEXT:
277278
case TSDataType::BLOB:
278279
case TSDataType::STRING:
279280
case TSDataType::OBJECT:
280-
field.stringV = iotdbRpcDataSet_->getBinary(columnName)->getStringValue();
281+
field.stringV = iotdbRpcDataSet_->getBinaryByIndex(columnIndex)->getStringValue();
281282
break;
282283
default:
283284
throw UnSupportedDataTypeException("Data type %s is not supported." + dataType);

‎iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -153,45 +153,42 @@ public boolean hasNext() throws StatementExecutionException, IoTDBConnectionExce
153153
}
154154

155155
private RowRecord constructRowRecordFromValueArray() throws StatementExecutionException {
156-
List<Field> outFields = new ArrayList<>();
157-
for (int i = ioTDBRpcDataSet.getValueColumnStartIndex();
158-
i < ioTDBRpcDataSet.getColumnSize();
159-
i++) {
156+
int valueColumnStartIndex = ioTDBRpcDataSet.getValueColumnStartIndex();
157+
int columnSize = ioTDBRpcDataSet.getColumnSize();
158+
List<Field> outFields = new ArrayList<>(columnSize - valueColumnStartIndex);
159+
for (int columnIndex = valueColumnStartIndex + 1; columnIndex <= columnSize; columnIndex++) {
160160
Field field;
161-
162-
String columnName = ioTDBRpcDataSet.getColumnNameList().get(i);
163-
164-
if (!ioTDBRpcDataSet.isNull(columnName)) {
165-
TSDataType dataType = ioTDBRpcDataSet.getDataType(columnName);
161+
if (!ioTDBRpcDataSet.isNull(columnIndex)) {
162+
TSDataType dataType = ioTDBRpcDataSet.getDataType(columnIndex);
166163
field = new Field(dataType);
167164
switch (dataType) {
168165
case BOOLEAN:
169-
boolean booleanValue = ioTDBRpcDataSet.getBoolean(columnName);
166+
boolean booleanValue = ioTDBRpcDataSet.getBoolean(columnIndex);
170167
field.setBoolV(booleanValue);
171168
break;
172169
case INT32:
173170
case DATE:
174-
int intValue = ioTDBRpcDataSet.getInt(columnName);
171+
int intValue = ioTDBRpcDataSet.getInt(columnIndex);
175172
field.setIntV(intValue);
176173
break;
177174
case INT64:
178175
case TIMESTAMP:
179-
long longValue = ioTDBRpcDataSet.getLong(columnName);
176+
long longValue = ioTDBRpcDataSet.getLong(columnIndex);
180177
field.setLongV(longValue);
181178
break;
182179
case FLOAT:
183-
float floatValue = ioTDBRpcDataSet.getFloat(columnName);
180+
float floatValue = ioTDBRpcDataSet.getFloat(columnIndex);
184181
field.setFloatV(floatValue);
185182
break;
186183
case DOUBLE:
187-
double doubleValue = ioTDBRpcDataSet.getDouble(columnName);
184+
double doubleValue = ioTDBRpcDataSet.getDouble(columnIndex);
188185
field.setDoubleV(doubleValue);
189186
break;
190187
case TEXT:
191188
case BLOB:
192189
case STRING:
193190
case OBJECT:
194-
field.setBinaryV(ioTDBRpcDataSet.getBinary(columnName));
191+
field.setBinaryV(ioTDBRpcDataSet.getBinary(columnIndex));
195192
break;
196193
default:
197194
throw new UnSupportedDataTypeException(

‎iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -128,20 +128,15 @@ public IoTDBJDBCDataSet(
128128

129129
// deduplicate and map
130130
if (columnNameIndex != null) {
131-
int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count();
132-
this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize);
133-
for (int i = 0; i < deduplicatedColumnSize; i++) {
134-
columnTypeDeduplicatedList.add(null);
135-
}
131+
this.columnTypeDeduplicatedList =
132+
initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex));
136133
for (int i = 0; i < columnNameList.size(); i++) {
137134
String name = columnNameList.get(i);
138135
this.columnNameList.add(name);
139136
this.columnTypeList.add(columnTypeList.get(i));
140137
if (!columnOrdinalMap.containsKey(name)) {
141138
int index = columnNameIndex.get(name);
142-
if (!columnOrdinalMap.containsValue(index + START_INDEX)) {
143-
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i)));
144-
}
139+
setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i));
145140
columnOrdinalMap.put(name, index + START_INDEX);
146141
}
147142
}
@@ -243,11 +238,8 @@ public IoTDBJDBCDataSet(
243238

244239
// deduplicate and map
245240
if (columnNameIndex != null) {
246-
int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count();
247-
this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize);
248-
for (int i = 0; i < deduplicatedColumnSize; i++) {
249-
columnTypeDeduplicatedList.add(null);
250-
}
241+
this.columnTypeDeduplicatedList =
242+
initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex));
251243
for (int i = 0; i < columnNameList.size(); i++) {
252244
String name = "";
253245
if (sgList != null
@@ -263,9 +255,7 @@ public IoTDBJDBCDataSet(
263255
// "Time".equals(name) -> to allow the Time column appear in value columns
264256
if (!columnOrdinalMap.containsKey(name) || "Time".equals(name)) {
265257
int index = columnNameIndex.get(name);
266-
if (!columnOrdinalMap.containsValue(index + START_INDEX)) {
267-
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i)));
268-
}
258+
setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i));
269259
columnOrdinalMap.put(name, index + START_INDEX);
270260
}
271261
}
@@ -321,6 +311,31 @@ public IoTDBJDBCDataSet(
321311
this.emptyResultSet = (queryDataSet == null || !queryDataSet.time.hasRemaining());
322312
}
323313

314+
private static int getDeduplicatedColumnSize(Map<String, Integer> columnNameIndex) {
315+
int deduplicatedColumnSize = 0;
316+
for (Integer index : columnNameIndex.values()) {
317+
if (index != null && index + 1 > deduplicatedColumnSize) {
318+
deduplicatedColumnSize = index + 1;
319+
}
320+
}
321+
return deduplicatedColumnSize;
322+
}
323+
324+
private static List<TSDataType> initDeduplicatedColumnTypes(int deduplicatedColumnSize) {
325+
List<TSDataType> columnTypes = new ArrayList<>(deduplicatedColumnSize);
326+
for (int i = 0; i < deduplicatedColumnSize; i++) {
327+
columnTypes.add(null);
328+
}
329+
return columnTypes;
330+
}
331+
332+
private static void setColumnTypeIfAbsent(
333+
List<TSDataType> columnTypeDeduplicatedList, int index, String columnType) {
334+
if (columnTypeDeduplicatedList.get(index) == null) {
335+
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnType));
336+
}
337+
}
338+
324339
public void close() throws StatementExecutionException, TException {
325340
if (isClosed) {
326341
return;

‎iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerde.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
/** Serializer for PreparedStatement parameters. */
3737
public class PreparedParameterSerde {
3838

39+
private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray();
40+
3941
public static class DeserializedParam {
4042
public final TSDataType type;
4143
public final Object value;
@@ -168,10 +170,13 @@ private static Object deserializeValue(ByteBuffer buffer, TSDataType type) {
168170

169171
/** Convert byte array to hexadecimal string representation. */
170172
public static String bytesToHex(byte[] bytes) {
171-
StringBuilder sb = new StringBuilder(bytes.length * 2);
172-
for (byte b : bytes) {
173-
sb.append(String.format("%02X", b));
173+
char[] chars = new char[bytes.length * 2];
174+
for (int i = 0; i < bytes.length; i++) {
175+
int value = bytes[i] & 0xFF;
176+
int index = i * 2;
177+
chars[index] = HEX_DIGITS[value >>> 4];
178+
chars[index + 1] = HEX_DIGITS[value & 0x0F];
174179
}
175-
return sb.toString();
180+
return new String(chars);
176181
}
177182
}

0 commit comments

Comments
 (0)