|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +#include <gtest/gtest.h> |
| 21 | + |
| 22 | +#include <algorithm> |
| 23 | +#include <cstdio> |
| 24 | +#include <memory> |
| 25 | +#include <string> |
| 26 | +#include <tuple> |
| 27 | +#include <vector> |
| 28 | + |
| 29 | +#include "common/path.h" |
| 30 | +#include "reader/qds_without_timegenerator.h" |
| 31 | +#include "reader/tsfile_reader.h" |
| 32 | +#include "writer/tsfile_writer.h" |
| 33 | + |
| 34 | +namespace storage { |
| 35 | +namespace { |
| 36 | + |
| 37 | +using namespace common; |
| 38 | + |
| 39 | +// Leading and intermediate all-null value pages must not prevent reading |
| 40 | +// subsequent pages, regardless of the value codec or write API. |
| 41 | +class AlignedNullPageReadTest |
| 42 | + : public ::testing::TestWithParam<std::tuple<TSDataType, uint32_t, int>> { |
| 43 | + protected: |
| 44 | + void SetUp() override { |
| 45 | + ASSERT_EQ(libtsfile_init(), E_OK); |
| 46 | + saved_page_rows_ = g_config_value_.page_writer_max_point_num_; |
| 47 | + saved_block_memory_ = g_config_value_.tsblock_max_memory_; |
| 48 | + g_config_value_.page_writer_max_point_num_ = std::get<1>(GetParam()); |
| 49 | + // The larger pages span multiple result blocks. |
| 50 | + g_config_value_.tsblock_max_memory_ = 4096; |
| 51 | + file_ = "aligned_null_page_" + |
| 52 | + std::to_string(static_cast<int>(std::get<0>(GetParam()))) + |
| 53 | + "_" + std::to_string(std::get<1>(GetParam())) + "_" + |
| 54 | + std::to_string(std::get<2>(GetParam())) + ".tsfile"; |
| 55 | + std::remove(file_.c_str()); |
| 56 | + } |
| 57 | + |
| 58 | + void TearDown() override { |
| 59 | + std::remove(file_.c_str()); |
| 60 | + g_config_value_.page_writer_max_point_num_ = saved_page_rows_; |
| 61 | + g_config_value_.tsblock_max_memory_ = saved_block_memory_; |
| 62 | + libtsfile_destroy(); |
| 63 | + } |
| 64 | + |
| 65 | + bool is_null(int row) const { |
| 66 | + int page = row / std::get<1>(GetParam()); |
| 67 | + return page == 0 || page == 2 || page == 3 || page == 6; |
| 68 | + } |
| 69 | + |
| 70 | + void add_value(TsRecord& record, int row) { |
| 71 | + switch (std::get<0>(GetParam())) { |
| 72 | + case INT32: |
| 73 | + record.add_point("value", static_cast<int32_t>(1000 + row)); |
| 74 | + break; |
| 75 | + case INT64: |
| 76 | + record.add_point("value", static_cast<int64_t>(1000 + row)); |
| 77 | + break; |
| 78 | + case FLOAT: |
| 79 | + record.add_point("value", static_cast<float>(1000.25 + row)); |
| 80 | + break; |
| 81 | + case DOUBLE: |
| 82 | + record.add_point("value", 1000.25 + row); |
| 83 | + break; |
| 84 | + case STRING: |
| 85 | + record_string_ = "value-" + std::to_string(row); |
| 86 | + record.add_point("value", String(record_string_)); |
| 87 | + break; |
| 88 | + default: |
| 89 | + FAIL() << "Unexpected type"; |
| 90 | + } |
| 91 | + } |
| 92 | + |
| 93 | + void add_value(Tablet& tablet, int index, int row) { |
| 94 | + switch (std::get<0>(GetParam())) { |
| 95 | + case INT32: |
| 96 | + ASSERT_EQ(tablet.add_value(index, 1, |
| 97 | + static_cast<int32_t>(1000 + row)), |
| 98 | + E_OK); |
| 99 | + break; |
| 100 | + case INT64: |
| 101 | + ASSERT_EQ(tablet.add_value(index, 1, |
| 102 | + static_cast<int64_t>(1000 + row)), |
| 103 | + E_OK); |
| 104 | + break; |
| 105 | + case FLOAT: |
| 106 | + ASSERT_EQ(tablet.add_value(index, 1, |
| 107 | + static_cast<float>(1000.25 + row)), |
| 108 | + E_OK); |
| 109 | + break; |
| 110 | + case DOUBLE: |
| 111 | + ASSERT_EQ(tablet.add_value(index, 1, 1000.25 + row), E_OK); |
| 112 | + break; |
| 113 | + case STRING: { |
| 114 | + std::string value = "value-" + std::to_string(row); |
| 115 | + ASSERT_EQ(tablet.add_value(index, 1, value.c_str()), E_OK); |
| 116 | + break; |
| 117 | + } |
| 118 | + default: |
| 119 | + FAIL() << "Unexpected type"; |
| 120 | + } |
| 121 | + } |
| 122 | + |
| 123 | + void check_value(Field* field, int row) { |
| 124 | + if (is_null(row)) { |
| 125 | + EXPECT_EQ(field->type_, NULL_TYPE) << "row=" << row; |
| 126 | + return; |
| 127 | + } |
| 128 | + ASSERT_EQ(field->type_, std::get<0>(GetParam())) << "row=" << row; |
| 129 | + switch (field->type_) { |
| 130 | + case INT32: |
| 131 | + EXPECT_EQ(field->value_.ival_, 1000 + row); |
| 132 | + break; |
| 133 | + case INT64: |
| 134 | + EXPECT_EQ(field->value_.lval_, 1000 + row); |
| 135 | + break; |
| 136 | + case FLOAT: |
| 137 | + EXPECT_FLOAT_EQ(field->value_.fval_, 1000.25f + row); |
| 138 | + break; |
| 139 | + case DOUBLE: |
| 140 | + EXPECT_DOUBLE_EQ(field->value_.dval_, 1000.25 + row); |
| 141 | + break; |
| 142 | + case STRING: |
| 143 | + EXPECT_EQ(field->get_string_value()->to_std_string(), |
| 144 | + "value-" + std::to_string(row)); |
| 145 | + break; |
| 146 | + default: |
| 147 | + FAIL() << "Unexpected type"; |
| 148 | + } |
| 149 | + } |
| 150 | + |
| 151 | + std::string file_; |
| 152 | + std::string record_string_; |
| 153 | + uint32_t saved_page_rows_ = 0; |
| 154 | + uint32_t saved_block_memory_ = 0; |
| 155 | +}; |
| 156 | + |
| 157 | +TEST_P(AlignedNullPageReadTest, |
| 158 | + ReadsValuesAfterLeadingAndIntermediateNullPages) { |
| 159 | + const TSDataType type = std::get<0>(GetParam()); |
| 160 | + const TSEncoding encoding = type == STRING ? DICTIONARY : GORILLA; |
| 161 | + const std::string device = "root.null_pages"; |
| 162 | + const int rows = 7 * std::get<1>(GetParam()); |
| 163 | + const int mode = std::get<2>(GetParam()); |
| 164 | + { |
| 165 | + TsFileWriter writer; |
| 166 | + ASSERT_EQ(writer.open(file_), E_OK); |
| 167 | + std::vector<MeasurementSchema*> schemas{ |
| 168 | + new MeasurementSchema("anchor", INT64, PLAIN, UNCOMPRESSED), |
| 169 | + new MeasurementSchema("value", type, encoding, UNCOMPRESSED)}; |
| 170 | + ASSERT_EQ(writer.register_aligned_timeseries(device, schemas), E_OK); |
| 171 | + for (int row = 0; row < rows;) { |
| 172 | + if (mode == 0 || (mode == 2 && row % 3 == 0)) { |
| 173 | + TsRecord record(row, device); |
| 174 | + record.add_point("anchor", static_cast<int64_t>(row)); |
| 175 | + if (is_null(row)) |
| 176 | + record.points_.emplace_back("value"); |
| 177 | + else |
| 178 | + add_value(record, row); |
| 179 | + ASSERT_EQ(writer.write_record_aligned(record), E_OK); |
| 180 | + ++row; |
| 181 | + } else { |
| 182 | + const int count = std::min(11, rows - row); |
| 183 | + auto schema = |
| 184 | + std::make_shared<std::vector<MeasurementSchema>>(); |
| 185 | + schema->emplace_back("anchor", INT64, PLAIN, UNCOMPRESSED); |
| 186 | + schema->emplace_back("value", type, encoding, UNCOMPRESSED); |
| 187 | + Tablet tablet(device, schema, count); |
| 188 | + ASSERT_EQ(tablet.err_code_, E_OK); |
| 189 | + for (int index = 0; index < count; ++index) { |
| 190 | + ASSERT_EQ(tablet.add_timestamp(index, row + index), E_OK); |
| 191 | + ASSERT_EQ(tablet.add_value( |
| 192 | + index, 0, static_cast<int64_t>(row + index)), |
| 193 | + E_OK); |
| 194 | + if (!is_null(row + index)) |
| 195 | + add_value(tablet, index, row + index); |
| 196 | + } |
| 197 | + ASSERT_EQ(writer.write_tablet_aligned(tablet), E_OK); |
| 198 | + row += count; |
| 199 | + } |
| 200 | + } |
| 201 | + ASSERT_EQ(writer.flush(), E_OK); |
| 202 | + ASSERT_EQ(writer.close(), E_OK); |
| 203 | + } |
| 204 | + |
| 205 | + TsFileReader reader; |
| 206 | + ASSERT_EQ(reader.open(file_), E_OK); |
| 207 | + ResultSet* result = nullptr; |
| 208 | + std::vector<Path> paths{Path(device + ".anchor"), Path(device + ".value")}; |
| 209 | + ASSERT_EQ(reader.query(QueryExpression::create(paths, nullptr), result), |
| 210 | + E_OK); |
| 211 | + int row = 0; |
| 212 | + bool more = false; |
| 213 | + while (true) { |
| 214 | + ASSERT_EQ(result->next(more), E_OK); |
| 215 | + if (!more) break; |
| 216 | + auto* record = result->get_row_record(); |
| 217 | + ASSERT_LT(row, rows); |
| 218 | + EXPECT_EQ(record->get_timestamp(), row); |
| 219 | + EXPECT_EQ(record->get_field(1)->value_.lval_, row); |
| 220 | + check_value(record->get_field(2), row); |
| 221 | + ++row; |
| 222 | + } |
| 223 | + EXPECT_EQ(row, rows); |
| 224 | + reader.destroy_query_data_set(result); |
| 225 | + ASSERT_EQ(reader.close(), E_OK); |
| 226 | +} |
| 227 | + |
| 228 | +INSTANTIATE_TEST_SUITE_P( |
| 229 | + CodecsAndPageBoundaries, AlignedNullPageReadTest, |
| 230 | + ::testing::Combine(::testing::Values(INT32, INT64, FLOAT, DOUBLE, STRING), |
| 231 | + ::testing::Values(1u, 7u, 256u), |
| 232 | + ::testing::Values(0, 1, 2))); |
| 233 | + |
| 234 | +} // namespace |
| 235 | +} // namespace storage |
0 commit comments