Skip to content

Commit fa36c3f

Browse files
authored
Merge pull request #160 from lomik/simplify_index_reverse
Simplify logic of usage reversed index query
2 parents beb5a30 + 0edbb08 commit fa36c3f

File tree

5 files changed

+171
-146
lines changed

5 files changed

+171
-146
lines changed

config/config.go

Lines changed: 57 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -85,34 +85,54 @@ type Common struct {
8585
MemoryReturnInterval *Duration `toml:"memory-return-interval" json:"memory-return-interval"`
8686
}
8787

88-
type NValue struct {
88+
// IndexReverseRule contains rules to use direct or reversed request to index table
89+
type IndexReverseRule struct {
8990
Suffix string `toml:"suffix" json:"suffix"`
9091
Prefix string `toml:"prefix" json:"prefix"`
9192
RegexStr string `toml:"regex" json:"regex"`
9293
Regex *regexp.Regexp `toml:"-" json:"-"`
93-
Value int `toml:"reverse" json:"reverse"`
94+
Reverse string `toml:"reverse" json:"reverse"`
9495
}
9596

97+
// IndexReverses is a slise of ptrs to IndexReverseRule
98+
type IndexReverses []*IndexReverseRule
99+
100+
const (
101+
IndexAuto = iota
102+
IndexDirect = iota
103+
IndexReversed = iota
104+
)
105+
106+
// IndexReverse maps setting name to value
107+
var IndexReverse = map[string]uint8{
108+
"direct": IndexDirect,
109+
"auto": IndexAuto,
110+
"reversed": IndexReversed,
111+
}
112+
113+
// IndexReverseNames contains valid names for index-reverse setting
114+
var IndexReverseNames = []string{"auto", "direct", "reversed"}
115+
96116
type ClickHouse struct {
97-
Url string `toml:"url" json:"url"`
98-
DataTimeout *Duration `toml:"data-timeout" json:"data-timeout"`
99-
TreeTable string `toml:"tree-table" json:"tree-table"`
100-
DateTreeTable string `toml:"date-tree-table" json:"date-tree-table"`
101-
DateTreeTableVersion int `toml:"date-tree-table-version" json:"date-tree-table-version"`
102-
IndexTable string `toml:"index-table" json:"index-table"`
103-
IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"`
104-
IndexReverseDepth int `toml:"index-reverse-depth" json:"index-reverse-depth"`
105-
IndexUseReverses []*NValue `toml:"index-reverses" json:"index-reverses"`
106-
IndexTimeout *Duration `toml:"index-timeout" json:"index-timeout"`
107-
TaggedTable string `toml:"tagged-table" json:"tagged-table"`
108-
TaggedAutocompleDays int `toml:"tagged-autocomplete-days" json:"tagged-autocomplete-days"`
109-
ReverseTreeTable string `toml:"reverse-tree-table" json:"reverse-tree-table"`
110-
TreeTimeout *Duration `toml:"tree-timeout" json:"tree-timeout"`
111-
TagTable string `toml:"tag-table" json:"tag-table"`
112-
ExtraPrefix string `toml:"extra-prefix" json:"extra-prefix"`
113-
ConnectTimeout *Duration `toml:"connect-timeout" json:"connect-timeout"`
114-
DataTableLegacy string `toml:"data-table" json:"data-table"`
115-
RollupConfLegacy string `toml:"rollup-conf" json:"-"`
117+
Url string `toml:"url" json:"url"`
118+
DataTimeout *Duration `toml:"data-timeout" json:"data-timeout"`
119+
TreeTable string `toml:"tree-table" json:"tree-table"`
120+
DateTreeTable string `toml:"date-tree-table" json:"date-tree-table"`
121+
DateTreeTableVersion int `toml:"date-tree-table-version" json:"date-tree-table-version"`
122+
IndexTable string `toml:"index-table" json:"index-table"`
123+
IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"`
124+
IndexReverse string `toml:"index-reverse" json:"index-reverse"`
125+
IndexReverses IndexReverses `toml:"index-reverses" json:"index-reverses"`
126+
IndexTimeout *Duration `toml:"index-timeout" json:"index-timeout"`
127+
TaggedTable string `toml:"tagged-table" json:"tagged-table"`
128+
TaggedAutocompleDays int `toml:"tagged-autocomplete-days" json:"tagged-autocomplete-days"`
129+
ReverseTreeTable string `toml:"reverse-tree-table" json:"reverse-tree-table"`
130+
TreeTimeout *Duration `toml:"tree-timeout" json:"tree-timeout"`
131+
TagTable string `toml:"tag-table" json:"tag-table"`
132+
ExtraPrefix string `toml:"extra-prefix" json:"extra-prefix"`
133+
ConnectTimeout *Duration `toml:"connect-timeout" json:"connect-timeout"`
134+
DataTableLegacy string `toml:"data-table" json:"data-table"`
135+
RollupConfLegacy string `toml:"rollup-conf" json:"-"`
116136
// Sets the maximum for maxDataPoints parameter.
117137
MaxDataPoints int `toml:"max-data-points" json:"max-data-points"`
118138
// InternalAggregation controls if ClickHouse itself or graphite-clickhouse aggregates points to proper retention
@@ -219,10 +239,10 @@ func New() *Config {
219239
TreeTimeout: &Duration{
220240
Duration: time.Minute,
221241
},
222-
IndexTable: "",
223-
IndexUseDaily: true,
224-
IndexReverseDepth: 1,
225-
IndexUseReverses: []*NValue{},
242+
IndexTable: "",
243+
IndexUseDaily: true,
244+
IndexReverse: "auto",
245+
IndexReverses: IndexReverses{},
226246
IndexTimeout: &Duration{
227247
Duration: time.Minute,
228248
},
@@ -259,16 +279,21 @@ func New() *Config {
259279
return cfg
260280
}
261281

262-
func IndexUseReversesValidate(u []*NValue) error {
282+
// Compile checks if IndexReverseRule are valid in the IndexReverses and compiles regexps if set
283+
func (ir IndexReverses) Compile() error {
263284
var err error
264-
for i, n := range u {
285+
for i, n := range ir {
265286
if len(n.RegexStr) > 0 {
266287
if n.Regex, err = regexp.Compile(n.RegexStr); err != nil {
267288
return err
268289
}
269290
} else if len(n.Prefix) == 0 && len(n.Suffix) == 0 {
270291
return fmt.Errorf("empthy index-use-reverses[%d] rule", i)
271292
}
293+
if _, ok := IndexReverse[n.Reverse]; !ok {
294+
return fmt.Errorf("%s is not valid value for index-reverses.reverse", n.Reverse)
295+
}
296+
272297
}
273298
return nil
274299
}
@@ -349,7 +374,11 @@ func ReadConfig(filename string) (*Config, error) {
349374
}
350375
}
351376

352-
err = IndexUseReversesValidate(cfg.ClickHouse.IndexUseReverses)
377+
if _, ok := IndexReverse[cfg.ClickHouse.IndexReverse]; !ok {
378+
return nil, fmt.Errorf("%s is not valid value for index-reverse", cfg.ClickHouse.IndexReverse)
379+
}
380+
381+
err = cfg.ClickHouse.IndexReverses.Compile()
353382
if err != nil {
354383
return nil, err
355384
}

finder/finder.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ func newPlainFinder(ctx context.Context, config *config.Config, query string, fr
4343
config.ClickHouse.Url,
4444
config.ClickHouse.IndexTable,
4545
config.ClickHouse.IndexUseDaily,
46-
config.ClickHouse.IndexReverseDepth,
47-
config.ClickHouse.IndexUseReverses,
46+
config.ClickHouse.IndexReverse,
47+
config.ClickHouse.IndexReverses,
4848
clickhouse.Options{
4949
Timeout: config.ClickHouse.IndexTimeout.Value(),
5050
ConnectTimeout: config.ClickHouse.ConnectTimeout.Value(),

finder/index.go

Lines changed: 43 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,32 @@ const ReverseTreeLevelOffset = 30000
1919

2020
const DefaultTreeDate = "1970-02-12"
2121

22+
const (
23+
queryAuto = config.IndexAuto
24+
queryDirect = config.IndexDirect
25+
queryReversed = config.IndexReversed
26+
)
27+
2228
type IndexFinder struct {
2329
url string // clickhouse dsn
2430
table string // graphite_tree table
2531
opts clickhouse.Options // timeout, connectTimeout
2632
dailyEnabled bool
27-
reverseDepth int
28-
revUse []*config.NValue
33+
confReverse uint8
34+
confReverses config.IndexReverses
35+
reverse uint8 // calculated in IndexFinder.useReverse only once
2936
body []byte // clickhouse response body
30-
useReverse bool
3137
useDaily bool
3238
}
3339

34-
func NewIndex(url string, table string, dailyEnabled bool, reverseDepth int, reverseUse []*config.NValue, opts clickhouse.Options) Finder {
40+
func NewIndex(url string, table string, dailyEnabled bool, reverse string, reverses config.IndexReverses, opts clickhouse.Options) Finder {
3541
return &IndexFinder{
3642
url: url,
3743
table: table,
3844
opts: opts,
3945
dailyEnabled: dailyEnabled,
40-
reverseDepth: reverseDepth,
41-
revUse: reverseUse,
46+
confReverse: config.IndexReverse[reverse],
47+
confReverses: reverses,
4248
}
4349
}
4450

@@ -53,83 +59,53 @@ func (idx *IndexFinder) where(query string, levelOffset int) *where.Where {
5359
return w
5460
}
5561

56-
func useReverse(query string) bool {
57-
p := strings.LastIndexByte(query, '.')
58-
59-
if !where.HasWildcard(query) || p < 0 || p >= len(query)-1 || where.HasWildcard(query[p+1:]) {
60-
return false
61-
}
62-
return true
63-
}
64-
65-
func reverseSuffixDepth(query string, defaultReverseDepth int, revUse []*config.NValue) int {
66-
for i := range revUse {
67-
if len(revUse[i].Prefix) > 0 && !strings.HasPrefix(query, revUse[i].Prefix) {
62+
func (idx *IndexFinder) checkReverses(query string) uint8 {
63+
for _, rule := range idx.confReverses {
64+
if len(rule.Prefix) > 0 && !strings.HasPrefix(query, rule.Prefix) {
6865
continue
6966
}
70-
if len(revUse[i].Suffix) > 0 && !strings.HasSuffix(query, revUse[i].Suffix) {
67+
if len(rule.Suffix) > 0 && !strings.HasSuffix(query, rule.Suffix) {
7168
continue
7269
}
73-
if revUse[i].Regex != nil && revUse[i].Regex.FindStringIndex(query) == nil {
70+
if rule.Regex != nil && rule.Regex.FindStringIndex(query) == nil {
7471
continue
7572
}
76-
return revUse[i].Value
73+
return config.IndexReverse[rule.Reverse]
7774
}
78-
return defaultReverseDepth
75+
return idx.confReverse
7976
}
8077

81-
func useReverseDepth(query string, reverseDepth int, revUse []*config.NValue) bool {
82-
if reverseDepth == -1 {
78+
func (idx *IndexFinder) useReverse(query string) bool {
79+
if idx.reverse == queryDirect {
8380
return false
81+
} else if idx.reverse == queryReversed {
82+
return true
8483
}
8584

86-
w := where.IndexWildcardOrDot(query)
87-
if w == -1 {
88-
return false
89-
} else if query[w] == '.' {
90-
reverseDepth = reverseSuffixDepth(query, reverseDepth, revUse)
91-
if reverseDepth == 0 {
92-
return false
93-
} else if reverseDepth == 1 {
94-
return useReverse(query)
95-
}
96-
} else {
97-
reverseDepth = 1
85+
if idx.reverse = idx.checkReverses(query); idx.reverse != queryAuto {
86+
return idx.useReverse(query)
9887
}
9988

100-
w = where.IndexReverseWildcard(query)
89+
w := where.IndexWildcard(query)
10190
if w == -1 {
102-
return false
103-
}
104-
p := len(query)
105-
if w == p-1 {
106-
return false
91+
idx.reverse = queryDirect
92+
return idx.useReverse(query)
10793
}
108-
depth := 0
109-
110-
for {
111-
e := strings.LastIndexByte(query[w:p], '.')
112-
if e < 0 {
113-
break
114-
} else if e < len(query)-1 {
115-
if where.HasWildcard(query[w+e+1 : p]) {
116-
break
117-
}
118-
depth++
119-
if depth >= reverseDepth {
120-
return true
121-
}
122-
if e == 0 {
123-
break
124-
}
125-
}
126-
p = w + e - 1
94+
firstWildcardNode := strings.Count(query[:w], ".")
95+
96+
w = where.IndexLastWildcard(query)
97+
lastWildcardNode := strings.Count(query[w:], ".")
98+
99+
if firstWildcardNode < lastWildcardNode {
100+
idx.reverse = queryReversed
101+
return idx.useReverse(query)
127102
}
128-
return false
103+
idx.reverse = queryDirect
104+
return idx.useReverse(query)
129105
}
130106

131107
func (idx *IndexFinder) Execute(ctx context.Context, query string, from int64, until int64) (err error) {
132-
idx.useReverse = useReverseDepth(query, idx.reverseDepth, idx.revUse)
108+
idx.useReverse(query)
133109

134110
if idx.dailyEnabled && from > 0 && until > 0 {
135111
idx.useDaily = true
@@ -139,18 +115,18 @@ func (idx *IndexFinder) Execute(ctx context.Context, query string, from int64, u
139115

140116
var levelOffset int
141117
if idx.useDaily {
142-
if idx.useReverse {
118+
if idx.useReverse(query) {
143119
levelOffset = ReverseLevelOffset
144120
}
145121
} else {
146-
if idx.useReverse {
122+
if idx.useReverse(query) {
147123
levelOffset = ReverseTreeLevelOffset
148124
} else {
149125
levelOffset = TreeLevelOffset
150126
}
151127
}
152128

153-
if idx.useReverse {
129+
if idx.useReverse(query) {
154130
query = ReverseString(query)
155131
}
156132

@@ -205,7 +181,7 @@ func (idx *IndexFinder) makeList(onlySeries bool) [][]byte {
205181

206182
rows = rows[:len(rows)-skip]
207183

208-
if idx.useReverse {
184+
if idx.useReverse("") {
209185
for i := 0; i < len(rows); i++ {
210186
rows[i] = ReverseBytes(rows[i])
211187
}

0 commit comments

Comments
 (0)