{

if (const auto * type_array = typeid_cast<const DataTypeArray *>(&type))

return type_array->getNumberOfDimensions();

return 0;

{

if (const auto * column_array = checkAndGetColumn<ColumnArray>(&column))

return column_array->getNumberOfDimensions();

return 0;

{

/// Get raw pointers to avoid extra copying of type pointers.

const DataTypeArray * last_array = nullptr;

const auto * current_type = type.get();

while (const auto * type_array = typeid_cast<const DataTypeArray *>(current_type))

{

current_type = type_array->getNestedType().get();

last_array = type_array;

}

return last_array ? last_array->getNestedType() : type;

{

auto it = tuple_elements.begin();

while (true)

{

if (const auto * type_array = typeid_cast<const DataTypeArray *>(type.get()))

{

type = type_array->getNestedType();

}

else if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get()))

{

if (it == tuple_elements.end())

break;

auto pos = type_tuple->tryGetPositionByName(*it);

if (!pos)

break;

++it;

type = type_tuple->getElement(*pos);

}

else

{

break;

}

}

return type;

{

/// Get raw pointers to avoid extra copying of column pointers.

const ColumnArray * last_array = nullptr;

const auto * current_column = column.get();

while (const auto * column_array = checkAndGetColumn<ColumnArray>(current_column))

{

current_column = &column_array->getData();

last_array = column_array;

}

return last_array ? last_array->getDataPtr() : column;

{

for (size_t i = 0; i < num_dimensions; ++i)

type = std::make_shared<DataTypeArray>(std::move(type));

return type;

{

for (size_t i = 0; i < num_dimensions; ++i)

column = ColumnArray::create(column);

return column;

{

if (num_dimensions == 0)

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Cannot create array field with 0 dimensions");

Array array;

Array * current_array = &array;

for (size_t i = 1; i < num_dimensions; ++i)

{

current_array->push_back(Array());

current_array = &current_array->back().safeGet<Array>();

}

return array;

{

auto idx = column.getDataType();

WhichDataType which(idx);

if (which.isSimple())

return DataTypeFactory::instance().get(String(magic_enum::enum_name(idx)));

if (which.isNothing())

return std::make_shared<DataTypeNothing>();

if (const auto * column_array = checkAndGetColumn<ColumnArray>(&column))

return std::make_shared<DataTypeArray>(getDataTypeByColumn(column_array->getData()));

if (const auto * column_nullable = checkAndGetColumn<ColumnNullable>(&column))

return makeNullable(getDataTypeByColumn(column_nullable->getNestedColumn()));

/// TODO: add more types.

throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get data type of column {}", column.getFamilyName());

{

static_assert(I < std::tuple_size_v<Tuple>);

std::vector<std::tuple_element_t<I, Tuple>> res;

res.reserve(vec.size());

for (const auto & elem : vec)

res.emplace_back(std::get<I>(elem));

return res;

{

return type_tuple.hasExplicitNames()

? std::make_shared<DataTypeTuple>(elements, type_tuple.getElementNames())

: std::make_shared<DataTypeTuple>(elements);

const ColumnObjectDeprecated & column_object, const DataTypeObjectDeprecated & type_object)

{

if (!column_object.isFinalized())

{

auto finalized = column_object.cloneFinalized();

const auto & finalized_object = assert_cast<const ColumnObjectDeprecated &>(*finalized);

return convertObjectColumnToTuple(finalized_object, type_object);

}

const auto & subcolumns = column_object.getSubcolumns();

PathsInData tuple_paths;

DataTypes tuple_types;

Columns tuple_columns;

for (const auto & entry : subcolumns)

{

tuple_paths.emplace_back(entry->path);

tuple_types.emplace_back(entry->data.getLeastCommonType());

tuple_columns.emplace_back(entry->data.getFinalizedColumnPtr());

}

return unflattenTuple(tuple_paths, tuple_types, tuple_columns);

const ColumnPtr & column, const DataTypePtr & type)

{

if (!type->hasDynamicSubcolumnsDeprecated())

return {column, type};

if (const auto * type_object = typeid_cast<const DataTypeObjectDeprecated *>(type.get()))

{

const auto & column_object = assert_cast<const ColumnObjectDeprecated &>(*column);

return convertObjectColumnToTuple(column_object, *type_object);

}

if (const auto * type_array = typeid_cast<const DataTypeArray *>(type.get()))

{

const auto & column_array = assert_cast<const ColumnArray &>(*column);

auto [new_column, new_type] = recursivlyConvertDynamicColumnToTuple(

column_array.getDataPtr(), type_array->getNestedType());

return

{

ColumnArray::create(new_column, column_array.getOffsetsPtr()),

std::make_shared<DataTypeArray>(std::move(new_type)),

};

}

if (const auto * type_map = typeid_cast<const DataTypeMap *>(type.get()))

{

const auto & column_map = assert_cast<const ColumnMap &>(*column);

auto [new_column, new_type] = recursivlyConvertDynamicColumnToTuple(

column_map.getNestedColumnPtr(), type_map->getNestedType());

return

{

ColumnMap::create(new_column),

std::make_shared<DataTypeMap>(std::move(new_type)),

};

}

if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get()))

{

const auto & tuple_columns = assert_cast<const ColumnTuple &>(*column).getColumns();

const auto & tuple_types = type_tuple->getElements();

assert(tuple_columns.size() == tuple_types.size());

const size_t tuple_size = tuple_types.size();

Columns new_tuple_columns(tuple_size);

DataTypes new_tuple_types(tuple_size);

for (size_t i = 0; i < tuple_size; ++i)

{

std::tie(new_tuple_columns[i], new_tuple_types[i])

= recursivlyConvertDynamicColumnToTuple(tuple_columns[i], tuple_types[i]);

}

auto new_column = tuple_size == 0 ? column : ColumnPtr(ColumnTuple::create(new_tuple_columns));

return

{

new_column,

recreateTupleWithElements(*type_tuple, new_tuple_types)

};

}

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Type {} unexpectedly has dynamic columns", type->getName());

{

for (auto & column : block)

{

if (!column.type->hasDynamicSubcolumnsDeprecated())

continue;

std::tie(column.column, column.type)

= recursivlyConvertDynamicColumnToTuple(column.column, column.type);

GetColumnsOptions options(GetColumnsOptions::AllPhysical);

auto storage_column = storage_snapshot->tryGetColumn(options, column.name);

if (!storage_column)

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Column '{}' not found in storage", column.name);

auto storage_column_concrete = storage_snapshot->getColumn(options.withExtendedObjects(), column.name);

/// Check that constructed Tuple type and type in storage are compatible.

getLeastCommonTypeForDynamicColumns(

storage_column->type, {column.type, storage_column_concrete.type}, true);

}

{

if (prefix.size() > parts.size())

return false;

for (size_t i = 0; i < prefix.size(); ++i)

if (prefix[i].key != parts[i].key)

return false;

return true;

{

for (size_t i = 0; i < std::min(lhs.size(), rhs.size()); ++i)

{

if (lhs[i].key != rhs[i].key)

return false;

if (lhs[i] != rhs[i])

return true;

}

return false;

{

size_t size = paths.size();

for (size_t i = 0; i < size; ++i)

{

for (size_t j = 0; j < i; ++j)

{

if (isPrefix(paths[i].getParts(), paths[j].getParts())

|| isPrefix(paths[j].getParts(), paths[i].getParts()))

throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS,

"Data in Object has ambiguous paths: '{}' and '{}'",

paths[i].getPath(), paths[j].getPath());

if (hasDifferentStructureInPrefix(paths[i].getParts(), paths[j].getParts()))

throw Exception(ErrorCodes::INCOMPATIBLE_COLUMNS,

"Data in Object has ambiguous paths: '{}' and '{}'. "

"Paths have prefixes matched by names, but different in structure",

paths[i].getPath(), paths[j].getPath());

}

}

{

/// Types of subcolumns by path from all tuples.

std::unordered_map<PathInData, DataTypes, PathInData::Hash> subcolumns_types;

/// First we flatten tuples, then get common type for paths

/// and finally unflatten paths and create new tuple type.

for (const auto & type : types)

{

const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get());

if (!type_tuple)

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR,

"Least common type for object can be deduced only from tuples, but {} given", type->getName());

auto [tuple_paths, tuple_types] = flattenTuple(type);

assert(tuple_paths.size() == tuple_types.size());

for (size_t i = 0; i < tuple_paths.size(); ++i)

subcolumns_types[tuple_paths[i]].push_back(tuple_types[i]);

}

PathsInData tuple_paths;

DataTypes tuple_types;

/// Get the least common type for all paths.

for (const auto & [key, subtypes] : subcolumns_types)

{

assert(!subtypes.empty());

if (key.getPath() == ColumnObjectDeprecated::COLUMN_NAME_DUMMY)

continue;

size_t first_dim = getNumberOfDimensions(*subtypes[0]);

for (size_t i = 1; i < subtypes.size(); ++i)

if (first_dim != getNumberOfDimensions(*subtypes[i]))

throw Exception(ErrorCodes::TYPE_MISMATCH,

"Incompatible types of subcolumn '{}': {} and {}",

key.getPath(), subtypes[0]->getName(), subtypes[i]->getName());

tuple_paths.emplace_back(key);

tuple_types.emplace_back(getLeastSupertypeOrString(subtypes));

}

if (tuple_paths.empty())

{

tuple_paths.emplace_back(ColumnObjectDeprecated::COLUMN_NAME_DUMMY);

tuple_types.emplace_back(std::make_shared<DataTypeUInt8>());

}

if (check_ambiguos_paths)

checkObjectHasNoAmbiguosPaths(tuple_paths);

return unflattenTuple(tuple_paths, tuple_types);

const Type & type, const DataTypes & concrete_types, bool check_ambiguos_paths)

{

DataTypes nested_types;

nested_types.reserve(concrete_types.size());

for (const auto & concrete_type : concrete_types)

{

const auto * type_with_nested_conctete = typeid_cast<const Type *>(concrete_type.get());

if (!type_with_nested_conctete)

throw Exception(ErrorCodes::TYPE_MISMATCH, "Expected {} type, got {}", demangle(typeid(Type).name()), concrete_type->getName());

nested_types.push_back(type_with_nested_conctete->getNestedType());

}

return std::make_shared<Type>(

getLeastCommonTypeForDynamicColumnsImpl(

type.getNestedType(), nested_types, check_ambiguos_paths));

const DataTypeTuple & type, const DataTypes & concrete_types, bool check_ambiguos_paths)

{

const auto & element_types = type.getElements();

DataTypes new_element_types(element_types.size());

for (size_t i = 0; i < element_types.size(); ++i)

{

DataTypes concrete_element_types;

concrete_element_types.reserve(concrete_types.size());

for (const auto & type_concrete : concrete_types)

{

const auto * type_tuple_conctete = typeid_cast<const DataTypeTuple *>(type_concrete.get());

if (!type_tuple_conctete)

throw Exception(ErrorCodes::TYPE_MISMATCH, "Expected Tuple type, got {}", type_concrete->getName());

concrete_element_types.push_back(type_tuple_conctete->getElement(i));

}

new_element_types[i] = getLeastCommonTypeForDynamicColumnsImpl(

element_types[i], concrete_element_types, check_ambiguos_paths);

}

return recreateTupleWithElements(type, new_element_types);

const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths)

{

if (!type_in_storage->hasDynamicSubcolumnsDeprecated())

return type_in_storage;

if (isObjectDeprecated(type_in_storage))

return getLeastCommonTypeForObject(concrete_types, check_ambiguos_paths);

if (const auto * type_array = typeid_cast<const DataTypeArray *>(type_in_storage.get()))

return getLeastCommonTypeForColumnWithNestedType(*type_array, concrete_types, check_ambiguos_paths);

if (const auto * type_map = typeid_cast<const DataTypeMap *>(type_in_storage.get()))

return getLeastCommonTypeForColumnWithNestedType(*type_map, concrete_types, check_ambiguos_paths);

if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_in_storage.get()))

return getLeastCommonTypeForTuple(*type_tuple, concrete_types, check_ambiguos_paths);

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Type {} unexpectedly has dynamic columns", type_in_storage->getName());

const DataTypePtr & type_in_storage, const DataTypes & concrete_types, bool check_ambiguos_paths)

{

if (concrete_types.empty())

return nullptr;

bool all_equal = true;

for (size_t i = 1; i < concrete_types.size(); ++i)

{

if (!concrete_types[i]->equals(*concrete_types[0]))

{

all_equal = false;

break;

}

}

if (all_equal)

return concrete_types[0];

return getLeastCommonTypeForDynamicColumnsImpl(type_in_storage, concrete_types, check_ambiguos_paths);

{

if (!type_in_storage->hasDynamicSubcolumnsDeprecated())

return type_in_storage;

if (isObjectDeprecated(type_in_storage))

return std::make_shared<DataTypeTuple>(

DataTypes{std::make_shared<DataTypeUInt8>()}, Names{ColumnObjectDeprecated::COLUMN_NAME_DUMMY});

if (const auto * type_array = typeid_cast<const DataTypeArray *>(type_in_storage.get()))

return std::make_shared<DataTypeArray>(

createConcreteEmptyDynamicColumn(type_array->getNestedType()));

if (const auto * type_map = typeid_cast<const DataTypeMap *>(type_in_storage.get()))

return std::make_shared<DataTypeMap>(

createConcreteEmptyDynamicColumn(type_map->getNestedType()));

if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type_in_storage.get()))

{

const auto & elements = type_tuple->getElements();

DataTypes new_elements;

new_elements.reserve(elements.size());

for (const auto & element : elements)

new_elements.push_back(createConcreteEmptyDynamicColumn(element));

return recreateTupleWithElements(*type_tuple, new_elements);

}

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Type {} unexpectedly has dynamic columns", type_in_storage->getName());

{

return std::any_of(columns.begin(), columns.end(),

[](const auto & column)

{

return column.type->hasDynamicSubcolumnsDeprecated();

});

{

NamesAndTypesList subcolumns_list;

for (auto & column : columns_list)

{

auto object_column = object_columns.tryGetColumn(GetColumnsOptions::All, column.name);

if (object_column)

{

column.type = object_column->type;

if (with_subcolumns)

subcolumns_list.splice(subcolumns_list.end(), object_columns.getSubcolumns(column.name));

}

}

columns_list.splice(columns_list.end(), std::move(subcolumns_list));

ColumnsDescription & object_columns,

const ColumnsDescription & storage_columns,

const NamesAndTypesList & new_columns)

{

for (const auto & new_column : new_columns)

{

auto object_column = object_columns.tryGetColumn(GetColumnsOptions::All, new_column.name);

if (object_column && !object_column->type->equals(*new_column.type))

{

auto storage_column = storage_columns.getColumn(GetColumnsOptions::All, new_column.name);

object_columns.modify(new_column.name, [&](auto & column)

{

column.type = getLeastCommonTypeForDynamicColumns(storage_column.type, {object_column->type, new_column.type});

});

}

}

PathInDataBuilder & builder,

DataTypePtr type,

std::vector<PathInData::Parts> & new_paths,

DataTypes & new_types)

{

if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get()))

{

const auto & tuple_names = type_tuple->getElementNames();

const auto & tuple_types = type_tuple->getElements();

for (size_t i = 0; i < tuple_names.size(); ++i)

{

builder.append(tuple_names[i], false);

flattenTupleImpl(builder, tuple_types[i], new_paths, new_types);

builder.popBack();

}

}

else if (const auto * type_array = typeid_cast<const DataTypeArray *>(type.get()))

{

PathInDataBuilder element_builder;

std::vector<PathInData::Parts> element_paths;

DataTypes element_types;

flattenTupleImpl(element_builder, type_array->getNestedType(), element_paths, element_types);

assert(element_paths.size() == element_types.size());

for (size_t i = 0; i < element_paths.size(); ++i)

{

builder.append(element_paths[i], true);

new_paths.emplace_back(builder.getParts());

new_types.emplace_back(std::make_shared<DataTypeArray>(element_types[i]));

builder.popBack(element_paths[i].size());

}

}

else

{

new_paths.emplace_back(builder.getParts());

new_types.emplace_back(type);

}

{

if (const auto * column_tuple = checkAndGetColumn<ColumnTuple>(column.get()))

{

const auto & subcolumns = column_tuple->getColumns();

for (const auto & subcolumn : subcolumns)

flattenTupleImpl(subcolumn, new_columns, offsets_columns);

}

else if (const auto * column_array = checkAndGetColumn<ColumnArray>(column.get()))

{

offsets_columns.push_back(column_array->getOffsetsPtr());

flattenTupleImpl(column_array->getDataPtr(), new_columns, offsets_columns);

offsets_columns.pop_back();

}

else

{

if (!offsets_columns.empty())

{

auto new_column = ColumnArray::create(column, offsets_columns.back());

for (auto it = offsets_columns.rbegin() + 1; it != offsets_columns.rend(); ++it)

new_column = ColumnArray::create(new_column, *it);

new_columns.push_back(std::move(new_column));

}

else

{

new_columns.push_back(column);

}

}

{

while (dimensions_to_reduce--)

{

const auto * type_array = typeid_cast<const DataTypeArray *>(type.get());

if (!type_array)

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Not enough dimensions to reduce");

type = type_array->getNestedType();

}

return type;

{

while (dimensions_to_reduce--)

{

const auto * column_array = typeid_cast<const ColumnArray *>(column.get());

if (!column_array)

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Not enough dimensions to reduce");

column = column_array->getDataPtr();

}

return column;

{

ColumnPtr column;

DataTypePtr type;

size_t array_dimensions;

{

auto collect_tuple_elemets = [](const auto & children)

{

if (children.empty())

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Cannot create type from empty Tuple or Nested node");

std::vector<std::tuple<String, ColumnWithTypeAndDimensions>> tuple_elements;

tuple_elements.reserve(children.size());

for (const auto & [name, child] : children)

{

assert(child);

auto column = createTypeFromNode(*child);

tuple_elements.emplace_back(name, std::move(column));

}

/// Sort to always create the same type for the same set of subcolumns.

::sort(tuple_elements.begin(), tuple_elements.end(),

[](const auto & lhs, const auto & rhs) { return std::get<0>(lhs) < std::get<0>(rhs); });

auto tuple_names = extractVector<0>(tuple_elements);

auto tuple_columns = extractVector<1>(tuple_elements);

return std::make_tuple(std::move(tuple_names), std::move(tuple_columns));

};

if (node.kind == Node::SCALAR)

{

return node.data;

}

if (node.kind == Node::NESTED)

{

auto [tuple_names, tuple_columns] = collect_tuple_elemets(node.children);

Columns offsets_columns;

offsets_columns.reserve(tuple_columns[0].array_dimensions + 1);

/// If we have a Nested node and child node with anonymous array levels

/// we need to push a Nested type through all array levels.

/// Example: { "k1": [[{"k2": 1, "k3": 2}] } should be parsed as

/// `k1 Array(Nested(k2 Int, k3 Int))` and k1 is marked as Nested

/// and `k2` and `k3` has anonymous_array_level = 1 in that case.

const auto & current_array = assert_cast<const ColumnArray &>(*node.data.column);

offsets_columns.push_back(current_array.getOffsetsPtr());

auto first_column = tuple_columns[0].column;

for (size_t i = 0; i < tuple_columns[0].array_dimensions; ++i)

{

const auto & column_array = assert_cast<const ColumnArray &>(*first_column);

offsets_columns.push_back(column_array.getOffsetsPtr());

first_column = column_array.getDataPtr();

}

size_t num_elements = tuple_columns.size();

Columns tuple_elements_columns(num_elements);

DataTypes tuple_elements_types(num_elements);

size_t last_offset = assert_cast<const ColumnArray::ColumnOffsets &>(*offsets_columns.back()).getData().back();

/// Reduce extra array dimensions to get columns and types of Nested elements.

for (size_t i = 0; i < num_elements; ++i)

{

assert(tuple_columns[i].array_dimensions == tuple_columns[0].array_dimensions);

tuple_elements_columns[i] = reduceNumberOfDimensions(tuple_columns[i].column, tuple_columns[i].array_dimensions);

tuple_elements_types[i] = reduceNumberOfDimensions(tuple_columns[i].type, tuple_columns[i].array_dimensions);

if (tuple_elements_columns[i]->size() != last_offset)

throw Exception(

ErrorCodes::EXPERIMENTAL_FEATURE_ERROR,

"Cannot create a type for subcolumn {} in Object data type: offsets_column has data inconsistent with nested_column. "

"Data size: {}, last offset: {}",

node.path.getPath(),

tuple_elements_columns[i]->size(),

last_offset);

}

auto result_column = ColumnArray::create(ColumnTuple::create(tuple_elements_columns), offsets_columns.back());

auto result_type = createNested(tuple_elements_types, tuple_names);

/// Recreate result Array type and Array column.

for (auto it = offsets_columns.rbegin() + 1; it != offsets_columns.rend(); ++it)

{

last_offset = assert_cast<const ColumnArray::ColumnOffsets &>((**it)).getData().back();

if (result_column->size() != last_offset)

throw Exception(

ErrorCodes::EXPERIMENTAL_FEATURE_ERROR,

"Cannot create a type for subcolumn {} in Object data type: offsets_column has data inconsistent with nested_column. "

"Data size: {}, last offset: {}",

node.path.getPath(),

result_column->size(),

last_offset);

result_column = ColumnArray::create(result_column, *it);

result_type = std::make_shared<DataTypeArray>(result_type);

}

return {result_column, result_type, tuple_columns[0].array_dimensions};

}

auto [tuple_names, tuple_columns] = collect_tuple_elemets(node.children);

size_t num_elements = tuple_columns.size();

Columns tuple_elements_columns(num_elements);

DataTypes tuple_elements_types(num_elements);

for (size_t i = 0; i < tuple_columns.size(); ++i)

{

assert(tuple_columns[i].array_dimensions == tuple_columns[0].array_dimensions);

tuple_elements_columns[i] = tuple_columns[i].column;

tuple_elements_types[i] = tuple_columns[i].type;

}

auto result_column = ColumnTuple::create(tuple_elements_columns);

auto result_type = std::make_shared<DataTypeTuple>(tuple_elements_types, tuple_names);

return {result_column, result_type, tuple_columns[0].array_dimensions};

{

std::vector<PathInData::Parts> new_path_parts;

DataTypes new_types;

PathInDataBuilder builder;

flattenTupleImpl(builder, type, new_path_parts, new_types);

PathsInData new_paths(new_path_parts.begin(), new_path_parts.end());

return {new_paths, new_types};

{

Columns new_columns;

Columns offsets_columns;

flattenTupleImpl(column, new_columns, offsets_columns);

return ColumnTuple::create(new_columns);

{

assert(paths.size() == tuple_types.size());

Columns tuple_columns;

tuple_columns.reserve(tuple_types.size());

for (const auto & type : tuple_types)

tuple_columns.emplace_back(type->createColumn());

return unflattenTuple(paths, tuple_types, tuple_columns).second;

{

const auto & subcolumns = column.getSubcolumns();

if (subcolumns.empty())

{

auto type = std::make_shared<DataTypeTuple>(

DataTypes{std::make_shared<DataTypeUInt8>()},

Names{ColumnObjectDeprecated::COLUMN_NAME_DUMMY});

return {type->createColumn()->cloneResized(column.size()), type};

}

PathsInData paths;

DataTypes types;

Columns columns;

paths.reserve(subcolumns.size());

types.reserve(subcolumns.size());

columns.reserve(subcolumns.size());

for (const auto & entry : subcolumns)

{

paths.emplace_back(entry->path);

types.emplace_back(entry->data.getLeastCommonType());

columns.emplace_back(entry->data.getFinalizedColumnPtr());

}

return unflattenTuple(paths, types, columns);

const PathsInData & paths,

const DataTypes & tuple_types,

const Columns & tuple_columns)

{

assert(paths.size() == tuple_types.size());

assert(paths.size() == tuple_columns.size());

if (paths.empty())

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, "Cannot unflatten empty Tuple");

/// We add all paths to the subcolumn tree and then create a type from it.

/// The tree stores column, type and number of array dimensions

/// for each intermediate node.

SubcolumnsTreeWithColumns tree;

for (size_t i = 0; i < paths.size(); ++i)

{

auto column = tuple_columns[i];

auto type = tuple_types[i];

const auto & parts = paths[i].getParts();

size_t num_parts = parts.size();

size_t pos = 0;

tree.add(paths[i], [&](Node::Kind kind, bool exists) -> std::shared_ptr<Node>

{

if (pos >= num_parts)

throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR,

"Not enough name parts for path {}. Expected at least {}, got {}",

paths[i].getPath(), pos + 1, num_parts);

size_t array_dimensions = kind == Node::NESTED ? 1 : parts[pos].anonymous_array_level;

ColumnWithTypeAndDimensions current_column{column, type, array_dimensions};

/// Get type and column for next node.

if (array_dimensions)

{

type = reduceNumberOfDimensions(type, array_dimensions);

column = reduceNumberOfDimensions(column, array_dimensions);

}

++pos;

if (exists)

return nullptr;

return kind == Node::SCALAR

? std::make_shared<Node>(kind, current_column, paths[i])

: std::make_shared<Node>(kind, current_column);

});

}

auto [column, type, _] = createTypeFromNode(tree.getRoot());

return std::make_pair(std::move(column), std::move(type));

{

auto & select = query->as<ASTSelectQuery &>();

if (!select.with())

select.setExpression(ASTSelectQuery::Expression::WITH, std::make_shared<ASTExpressionList>());

/// TODO: avoid materialize

auto node = makeASTFunction("materialize",

makeASTFunction("CAST",

std::make_shared<ASTLiteral>(data_type->getDefault()),

std::make_shared<ASTLiteral>(data_type->getName())));

node->alias = column_name;

node->prefer_alias_to_column_name = true;

select.with()->children.push_back(std::move(node));

const ColumnsDescription & expected_columns,

const ColumnsDescription & available_columns

)

{

NamesAndTypes missed_names_types;

/// Find all subcolumns that are in @expected_columns, but not in @available_columns.

for (const auto & column : available_columns)

{

auto expected_column = expected_columns.getColumn(GetColumnsOptions::All, column.name);

/// Extract all paths from both descriptions to easily check existence of subcolumns.

auto [available_paths, available_types] = flattenTuple(column.type);

auto [expected_paths, expected_types] = flattenTuple(expected_column.type);

auto extract_names_and_types = [&column](const auto & paths, const auto & types)

{

NamesAndTypes res;

res.reserve(paths.size());

for (size_t i = 0; i < paths.size(); ++i)

{

auto full_name = Nested::concatenateName(column.name, paths[i].getPath());

res.emplace_back(full_name, types[i]);

}

::sort(res.begin(), res.end());

return res;

};

auto available_names_types = extract_names_and_types(available_paths, available_types);

auto expected_names_types = extract_names_and_types(expected_paths, expected_types);

std::set_difference(

expected_names_types.begin(), expected_names_types.end(),

available_names_types.begin(), available_names_types.end(),

std::back_inserter(missed_names_types),

[](const auto & lhs, const auto & rhs) { return lhs.name < rhs.name; });

}

return missed_names_types;

const ColumnsDescription & expected_columns,

const ColumnsDescription & available_columns,

ASTPtr query)

{

NamesAndTypes missed_names_types = calculateMissedSubcolumns(expected_columns, available_columns);

if (missed_names_types.empty())

return;

IdentifierNameSet identifiers;