improvements in subcolumns reading

CurtizJ · CurtizJ · commit 6de8b05b7c06 · 2020-12-17T20:10:33.000+03:00
diff --git a/src/DataTypes/DataTypeArray.cpp b/src/DataTypes/DataTypeArray.cpp
@@ -149,21 +149,42 @@ namespace
         offset_values.resize(i);
     }
 
-    MutableColumnPtr getArraySizesPositionIndependent(const ColumnArray & column_array)
+    ColumnPtr arrayOffsetsToSizes(const IColumn & column)
     {
-        const auto & offset_values = column_array.getOffsets();
-        MutableColumnPtr new_offsets = column_array.getOffsetsColumn().cloneEmpty();
+        const auto & column_offsets = assert_cast<const ColumnArray::ColumnOffsets &>(column);
+        MutableColumnPtr column_sizes = column_offsets.cloneEmpty();
 
-        if (offset_values.empty())
-            return new_offsets;
+        if (column_offsets.empty())
+            return column_sizes;
 
-        auto & new_offsets_values = assert_cast<ColumnVector<ColumnArray::Offset> &>(*new_offsets).getData();
-        new_offsets_values.reserve(offset_values.size());
-        new_offsets_values.push_back(offset_values[0]);
-        for (size_t i = 1; i < offset_values.size(); ++i)
-            new_offsets_values.push_back(offset_values[i] - offset_values[i - 1]);
+        const auto & offsets_data = column_offsets.getData();
+        auto & sizes_data = assert_cast<ColumnArray::ColumnOffsets &>(*column_sizes).getData();
 
-        return new_offsets;
+        sizes_data.resize(offsets_data.size());
+        sizes_data[0] = offsets_data[0];
+        for (size_t i = 1; i < offsets_data.size(); ++i)
+            sizes_data[i] = offsets_data[i] - offsets_data[i - 1];
+
+        return column_sizes;
+    }
+
+    ColumnPtr arraySizesToOffsets(const IColumn & column)
+    {
+        const auto & column_sizes = assert_cast<const ColumnArray::ColumnOffsets &>(column);
+        MutableColumnPtr column_offsets = column_sizes.cloneEmpty();
+
+        if (column_sizes.empty())
+            return column_offsets;
+
+        const auto & sizes_data = column_sizes.getData();
+        auto & offsets_data = assert_cast<ColumnArray::ColumnOffsets &>(*column_offsets).getData();
+
+        offsets_data.resize(sizes_data.size());
+        offsets_data[0] = sizes_data[0];
+        for (size_t i = 0; i < sizes_data.size(); ++i)
+            offsets_data[i] = offsets_data[i - 1] + sizes_data[i];
+
+        return column_offsets;
     }
 }
 
@@ -263,12 +284,11 @@ void DataTypeArray::deserializeBinaryBulkWithMultipleStreamsImpl(
     SubstreamsCache * cache) const
 {
     ColumnArray & column_array = typeid_cast<ColumnArray &>(column);
-
     settings.path.push_back(Substream::ArraySizes);
 
     if (auto cached_column = getFromSubstreamsCache(cache, settings.path))
     {
-        column_array.getOffsetsPtr() = cached_column;
+        column_array.getOffsetsPtr() = arraySizesToOffsets(*cached_column);
     }
     else if (auto * stream = settings.getter(settings.path))
     {
@@ -277,7 +297,7 @@ void DataTypeArray::deserializeBinaryBulkWithMultipleStreamsImpl(
         else
             DataTypeNumber<ColumnArray::Offset>().deserializeBinaryBulk(column_array.getOffsetsColumn(), *stream, limit, 0);
 
-        addToSubstreamsCache(cache, settings.path, column_array.getOffsetsPtr());
+        addToSubstreamsCache(cache, settings.path, arrayOffsetsToSizes(column_array.getOffsetsColumn()));
     }
 
     settings.path.back() = Substream::ArrayElements;
@@ -547,24 +567,24 @@ DataTypePtr DataTypeArray::tryGetSubcolumnTypeImpl(const String & subcolumn_name
     return (subcolumn ? std::make_shared<DataTypeArray>(std::move(subcolumn)) : subcolumn);
 }
 
-MutableColumnPtr DataTypeArray::getSubcolumn(const String & subcolumn_name, IColumn & column) const
+ColumnPtr DataTypeArray::getSubcolumn(const String & subcolumn_name, const IColumn & column) const
 {
     return getSubcolumnImpl(subcolumn_name, column, 0);
 }
 
-MutableColumnPtr DataTypeArray::getSubcolumnImpl(const String & subcolumn_name, IColumn & column, size_t level) const
+ColumnPtr DataTypeArray::getSubcolumnImpl(const String & subcolumn_name, const IColumn & column, size_t level) const
 {
-    auto & column_array = assert_cast<ColumnArray &>(column);
+    const auto & column_array = assert_cast<const ColumnArray &>(column);
     if (subcolumn_name == "size" + std::to_string(level))
-        return getArraySizesPositionIndependent(column_array);
+        return arrayOffsetsToSizes(column_array.getOffsetsColumn());
 
-    MutableColumnPtr subcolumn;
+    ColumnPtr subcolumn;
     if (const auto * nested_array = typeid_cast<const DataTypeArray *>(nested.get()))
         subcolumn = nested_array->getSubcolumnImpl(subcolumn_name, column_array.getData(), level + 1);
     else
         subcolumn = nested->getSubcolumn(subcolumn_name, column_array.getData());
 
-    return ColumnArray::create(std::move(subcolumn), column_array.getOffsetsPtr()->assumeMutable());
+    return ColumnArray::create(subcolumn, column_array.getOffsetsPtr());
 }
 
 size_t DataTypeArray::getNumberOfDimensions() const
diff --git a/src/DataTypes/DataTypeArray.h b/src/DataTypes/DataTypeArray.h
@@ -113,15 +113,15 @@ class DataTypeArray final : public DataTypeWithSimpleSerialization
     }
 
     DataTypePtr tryGetSubcolumnType(const String & subcolumn_name) const override;
-    MutableColumnPtr getSubcolumn(const String & subcolumn_name, IColumn & column) const override;
+    ColumnPtr getSubcolumn(const String & subcolumn_name, const IColumn & column) const override;
 
     const DataTypePtr & getNestedType() const { return nested; }
 
     //github.com/ 1 for plain array, 2 for array of arrays and so on.
     size_t getNumberOfDimensions() const;
 
 private:
-    MutableColumnPtr getSubcolumnImpl(const String & subcolumn_name, IColumn & column, size_t level) const;
+    ColumnPtr getSubcolumnImpl(const String & subcolumn_name, const IColumn & column, size_t level) const;
     DataTypePtr tryGetSubcolumnTypeImpl(const String & subcolumn_name, size_t level) const;
 };
 
diff --git a/src/DataTypes/DataTypeNullable.cpp b/src/DataTypes/DataTypeNullable.cpp
@@ -542,9 +542,9 @@ DataTypePtr DataTypeNullable::tryGetSubcolumnType(const String & subcolumn_name)
     return nested_data_type->tryGetSubcolumnType(subcolumn_name);
 }
 
-MutableColumnPtr DataTypeNullable::getSubcolumn(const String & subcolumn_name, IColumn & column) const
+ColumnPtr DataTypeNullable::getSubcolumn(const String & subcolumn_name, const IColumn & column) const
 {
-    auto & column_nullable = assert_cast<ColumnNullable &>(column);
+    const auto & column_nullable = assert_cast<const ColumnNullable &>(column);
     if (subcolumn_name == "null")
         return column_nullable.getNullMapColumnPtr()->assumeMutable();
 
diff --git a/src/DataTypes/DataTypeNullable.h b/src/DataTypes/DataTypeNullable.h
@@ -99,7 +99,7 @@ class DataTypeNullable final : public IDataType
     bool onlyNull() const override;
     bool canBeInsideLowCardinality() const override { return nested_data_type->canBeInsideLowCardinality(); }
     DataTypePtr tryGetSubcolumnType(const String & subcolumn_name) const override;
-    MutableColumnPtr getSubcolumn(const String & subcolumn_name, IColumn & column) const override;
+    ColumnPtr getSubcolumn(const String & subcolumn_name, const IColumn & column) const override;
 
     const DataTypePtr & getNestedType() const { return nested_data_type; }
 
diff --git a/src/DataTypes/DataTypeOneElementTuple.cpp b/src/DataTypes/DataTypeOneElementTuple.cpp
@@ -22,7 +22,7 @@ class DataTypeOneElementTupleStreams : public IDataTypeCustomStreams
     bool escape_delimiter;
 
 public:
-    DataTypeOneElementTupleStreams(const DataTypePtr & nested_, const String & name_, bool escape_delimiter_ = true)
+    DataTypeOneElementTupleStreams(const DataTypePtr & nested_, const String & name_, bool escape_delimiter_)
         : nested(nested_), name(name_), escape_delimiter(escape_delimiter_) {}
 
     void enumerateStreams(
@@ -99,7 +99,7 @@ class DataTypeOneElementTupleStreams : public IDataTypeCustomStreams
 DataTypePtr createOneElementTuple(const DataTypePtr & type, const String & name, bool escape_delimiter)
 {
     auto custom_desc = std::make_unique<DataTypeCustomDesc>(
-        std::make_unique<DataTypeCustomFixedName>(type->getName()), nullptr,
+        std::make_unique<DataTypeCustomFixedName>(type->getName()),nullptr,
         std::make_unique<DataTypeOneElementTupleStreams>(type, name, escape_delimiter));
 
     return DataTypeFactory::instance().getCustom(std::move(custom_desc));
diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp
@@ -635,14 +635,14 @@ DataTypePtr DataTypeTuple::tryGetSubcolumnType(const String & subcolumn_name) co
     return nullptr;
 }
 
-MutableColumnPtr DataTypeTuple::getSubcolumn(const String & subcolumn_name, IColumn & column) const
+ColumnPtr DataTypeTuple::getSubcolumn(const String & subcolumn_name, const IColumn & column) const
 {
     for (size_t i = 0; i < names.size(); ++i)
     {
         if (startsWith(subcolumn_name, names[i]))
         {
             size_t name_length = names[i].size();
-            auto & subcolumn = extractElementColumn(column, i);
+            const auto & subcolumn = extractElementColumn(column, i);
 
             if (subcolumn_name.size() == name_length)
                 return subcolumn.assumeMutable();
diff --git a/src/DataTypes/DataTypeTuple.h b/src/DataTypes/DataTypeTuple.h
@@ -100,7 +100,7 @@ class DataTypeTuple final : public DataTypeWithSimpleSerialization
     size_t getSizeOfValueInMemory() const override;
 
     DataTypePtr tryGetSubcolumnType(const String & subcolumn_name) const override;
-    MutableColumnPtr getSubcolumn(const String & subcolumn_name, IColumn & column) const override;
+    ColumnPtr getSubcolumn(const String & subcolumn_name, const IColumn & column) const override;
 
     const DataTypes & getElements() const { return elems; }
     const Strings & getElementNames() const { return names; }
diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp
@@ -156,7 +156,7 @@ DataTypePtr IDataType::getSubcolumnType(const String & subcolumn_name) const
     throw Exception(ErrorCodes::ILLEGAL_COLUMN, "There is no subcolumn {} in type {}", subcolumn_name, getName());
 }
 
-MutableColumnPtr IDataType::getSubcolumn(const String & subcolumn_name, IColumn &) const
+ColumnPtr IDataType::getSubcolumn(const String & subcolumn_name, const IColumn &) const
 {
     throw Exception(ErrorCodes::ILLEGAL_COLUMN, "There is no subcolumn {} in type {}", subcolumn_name, getName());
 }
@@ -173,11 +173,7 @@ Names IDataType::getSubcolumnNames() const
             new_path.push_back(elem);
             auto subcolumn_name = getSubcolumnNameForStream(new_path);
             if (!subcolumn_name.empty() && tryGetSubcolumnType(subcolumn_name))
-            {
-                //github.com/ Not all of substreams have its subcolumn.
-                if (tryGetSubcolumnType(subcolumn_name))
-                    res.insert(subcolumn_name);
-            }
+                res.insert(subcolumn_name);
         }
     });
 
@@ -329,7 +325,7 @@ void IDataType::deserializeBinaryBulkWithMultipleStreams(
     }
 
     //github.com/ Do not cache complex type, because they can be constructed
-    //github.com/ their subcolumns, which are in cache.
+    //github.com/ from their subcolumns, which are in cache.
     if (!haveSubtypes())
     {
         auto cached_column = getFromSubstreamsCache(cache, settings.path);
@@ -340,7 +336,7 @@ void IDataType::deserializeBinaryBulkWithMultipleStreams(
         }
     }
 
-    auto mutable_column = IColumn::mutate(std::move(column));
+    auto mutable_column = column->assumeMutable();
     deserializeBinaryBulkWithMultipleStreamsImpl(*mutable_column, limit, settings, state, cache);
     column = std::move(mutable_column);
 
diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h
@@ -126,7 +126,7 @@ class IDataType : private boost::noncopyable
 
     virtual DataTypePtr tryGetSubcolumnType(const String & /* subcolumn_name */) const { return nullptr; }
     DataTypePtr getSubcolumnType(const String & subcolumn_name) const;
-    virtual MutableColumnPtr getSubcolumn(const String & subcolumn_name, IColumn & column) const;
+    virtual ColumnPtr getSubcolumn(const String & subcolumn_name, const IColumn & column) const;
     Names getSubcolumnNames() const;
 
     using OutputStreamGetter = std::function<WriteBuffer*(const SubstreamPath &)>;
diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
@@ -220,7 +220,7 @@ void MergeTreeReaderCompact::readData(
 
         storage_type->deserializeBinaryBulkStatePrefix(deserialize_settings, state);
         storage_type->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, state);
-        column = storage_type->getSubcolumn(name_and_type.getSubcolumnName(), *temp_column->assumeMutable());
+        column = storage_type->getSubcolumn(name_and_type.getSubcolumnName(), *temp_column);
     }
     else
     {
diff --git a/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp b/src/Storages/MergeTree/MergeTreeReaderInMemory.cpp
@@ -47,7 +47,7 @@ static ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair &
 
     const auto & column = block.getByName(storage_name).column;
     if (name_and_type.isSubcolumn())
-        return name_and_type.getStorageType()->getSubcolumn(name_and_type.getSubcolumnName(), *column->assumeMutable());
+        return name_and_type.getStorageType()->getSubcolumn(name_and_type.getSubcolumnName(), *column);
 
     return column;
 }
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
@@ -121,7 +121,7 @@ class BufferSource : public SourceWithProgress
         {
             const auto & current_column = buffer.data.getByName(elem.getStorageName()).column;
             if (elem.isSubcolumn())
-                columns.emplace_back(elem.getStorageType()->getSubcolumn(elem.getSubcolumnName(), *current_column->assumeMutable()));
+                columns.emplace_back(elem.getStorageType()->getSubcolumn(elem.getSubcolumnName(), *current_column));
             else
                 columns.emplace_back(std::move(current_column));
         }
diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp
@@ -69,7 +69,7 @@ class MemorySource : public SourceWithProgress
         {
             auto current_column = src.getByName(elem.getStorageName()).column;
             if (elem.isSubcolumn())
-                columns.emplace_back(elem.getStorageType()->getSubcolumn(elem.getSubcolumnName(), *current_column->assumeMutable()));
+                columns.emplace_back(elem.getStorageType()->getSubcolumn(elem.getSubcolumnName(), *current_column));
             else
                 columns.emplace_back(std::move(current_column));
         }

Original file line number	Diff line number	Diff line change
`@@ -542,9 +542,9 @@ DataTypePtr DataTypeNullable::tryGetSubcolumnType(const String & subcolumn_name)`
`542`	`542`	`return nested_data_type->tryGetSubcolumnType(subcolumn_name);`
`543`	`543`	`}`
`544`	`544`
`545`		`-MutableColumnPtr DataTypeNullable::getSubcolumn(const String & subcolumn_name, IColumn & column) const`
	`545`	`+ColumnPtr DataTypeNullable::getSubcolumn(const String & subcolumn_name, const IColumn & column) const`
`546`	`546`	`{`
`547`		`- auto & column_nullable = assert_cast<ColumnNullable &>(column);`
	`547`	`+ const auto & column_nullable = assert_cast<const ColumnNullable &>(column);`
`548`	`548`	`if (subcolumn_name == "null")`
`549`	`549`	`return column_nullable.getNullMapColumnPtr()->assumeMutable();`
`550`	`550`
Original file line number	Diff line number	Diff line change
`@@ -635,14 +635,14 @@ DataTypePtr DataTypeTuple::tryGetSubcolumnType(const String & subcolumn_name) co`
`635`	`635`	`return nullptr;`
`636`	`636`	`}`
`637`	`637`
`638`		`-MutableColumnPtr DataTypeTuple::getSubcolumn(const String & subcolumn_name, IColumn & column) const`
	`638`	`+ColumnPtr DataTypeTuple::getSubcolumn(const String & subcolumn_name, const IColumn & column) const`
`639`	`639`	`{`
`640`	`640`	`for (size_t i = 0; i < names.size(); ++i)`
`641`	`641`	`{`
`642`	`642`	`if (startsWith(subcolumn_name, names[i]))`
`643`	`643`	`{`
`644`	`644`	`size_t name_length = names[i].size();`
`645`		`- auto & subcolumn = extractElementColumn(column, i);`
	`645`	`+ const auto & subcolumn = extractElementColumn(column, i);`
`646`	`646`
`647`	`647`	`if (subcolumn_name.size() == name_length)`
`648`	`648`	`return subcolumn.assumeMutable();`
Original file line number	Diff line number	Diff line change
`@@ -156,7 +156,7 @@ DataTypePtr IDataType::getSubcolumnType(const String & subcolumn_name) const`
`156`	`156`	`throw Exception(ErrorCodes::ILLEGAL_COLUMN, "There is no subcolumn {} in type {}", subcolumn_name, getName());`
`157`	`157`	`}`
`158`	`158`
`159`		`-MutableColumnPtr IDataType::getSubcolumn(const String & subcolumn_name, IColumn &) const`
	`159`	`+ColumnPtr IDataType::getSubcolumn(const String & subcolumn_name, const IColumn &) const`
`160`	`160`	`{`
`161`	`161`	`throw Exception(ErrorCodes::ILLEGAL_COLUMN, "There is no subcolumn {} in type {}", subcolumn_name, getName());`
`162`	`162`	`}`
`@@ -173,11 +173,7 @@ Names IDataType::getSubcolumnNames() const`
`173`	`173`	`new_path.push_back(elem);`
`174`	`174`	`auto subcolumn_name = getSubcolumnNameForStream(new_path);`
`175`	`175`	`if (!subcolumn_name.empty() && tryGetSubcolumnType(subcolumn_name))`
`176`		`- {`
`177`		`- /// Not all of substreams have its subcolumn.`
`178`		`- if (tryGetSubcolumnType(subcolumn_name))`
`179`		`- res.insert(subcolumn_name);`
`180`		`- }`
	`176`	`+ res.insert(subcolumn_name);`
`181`	`177`	`}`
`182`	`178`	`});`
`183`	`179`
`@@ -329,7 +325,7 @@ void IDataType::deserializeBinaryBulkWithMultipleStreams(`
`329`	`325`	`}`
`330`	`326`
`331`	`327`	`/// Do not cache complex type, because they can be constructed`
`332`		`- /// their subcolumns, which are in cache.`
	`328`	`+ /// from their subcolumns, which are in cache.`
`333`	`329`	`if (!haveSubtypes())`
`334`	`330`	`{`
`335`	`331`	`auto cached_column = getFromSubstreamsCache(cache, settings.path);`
`@@ -340,7 +336,7 @@ void IDataType::deserializeBinaryBulkWithMultipleStreams(`
`340`	`336`	`}`
`341`	`337`	`}`
`342`	`338`
`343`		`- auto mutable_column = IColumn::mutate(std::move(column));`
	`339`	`+ auto mutable_column = column->assumeMutable();`
`344`	`340`	`deserializeBinaryBulkWithMultipleStreamsImpl(*mutable_column, limit, settings, state, cache);`
`345`	`341`	`column = std::move(mutable_column);`
`346`	`342`
Original file line number	Diff line number	Diff line change
`@@ -220,7 +220,7 @@ void MergeTreeReaderCompact::readData(`
`220`	`220`
`221`	`221`	`storage_type->deserializeBinaryBulkStatePrefix(deserialize_settings, state);`
`222`	`222`	`storage_type->deserializeBinaryBulkWithMultipleStreams(temp_column, rows_to_read, deserialize_settings, state);`
`223`		`- column = storage_type->getSubcolumn(name_and_type.getSubcolumnName(), *temp_column->assumeMutable());`
	`223`	`+ column = storage_type->getSubcolumn(name_and_type.getSubcolumnName(), *temp_column);`
`224`	`224`	`}`
`225`	`225`	`else`
`226`	`226`	`{`