Skip to content

Commit 81c7f32

Browse files
therealpandeytherealpandey
authored
feat(prometheus): create a dedicated prometheus package (#7397)
1 parent 2cbd873 commit 81c7f32

File tree

44 files changed

+1109
-698
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1109
-698
lines changed

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ go-run-enterprise: ## Runs the enterprise go backend server
7474
--use-logs-new-schema true \
7575
--use-trace-new-schema true
7676

77+
.PHONY: go-test
78+
go-test: ## Runs go unit tests
79+
@go test -race ./...
80+
7781
.PHONY: go-run-community
7882
go-run-community: ## Runs the community go backend server
7983
@SIGNOZ_INSTRUMENTATION_LOGS_LEVEL=debug \

‎conf/example.yaml‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ sqlstore:
7272
# The path to the SQLite database file.
7373
path: /var/lib/signoz/signoz.db
7474

75-
7675
##################### APIServer #####################
7776
apiserver:
7877
timeout:
@@ -91,20 +90,29 @@ apiserver:
9190
- /api/v1/version
9291
- /
9392

94-
9593
##################### TelemetryStore #####################
9694
telemetrystore:
97-
# Specifies the telemetrystore provider to use.
98-
provider: clickhouse
9995
# Maximum number of idle connections in the connection pool.
10096
max_idle_conns: 50
10197
# Maximum number of open connections to the database.
10298
max_open_conns: 100
10399
# Maximum time to wait for a connection to be established.
104100
dial_timeout: 5s
101+
# Specifies the telemetrystore provider to use.
102+
provider: clickhouse
105103
clickhouse:
106-
# The DSN to use for ClickHouse.
107-
dsn: http://localhost:9000
104+
# The DSN to use for clickhouse.
105+
dsn: tcp://localhost:9000
106+
107+
##################### Prometheus #####################
108+
prometheus:
109+
active_query_tracker:
110+
# Whether to enable the active query tracker.
111+
enabled: true
112+
# The path to use for the active query tracker.
113+
path: ""
114+
# The maximum number of concurrent queries.
115+
max_concurrent: 20
108116

109117
##################### Alertmanager #####################
110118
alertmanager:
@@ -117,7 +125,7 @@ alertmanager:
117125
# The poll interval for periodically syncing the alertmanager with the config in the store.
118126
poll_interval: 1m
119127
# The URL under which Alertmanager is externally reachable (for example, if Alertmanager is served via a reverse proxy). Used for generating relative and absolute links back to Alertmanager itself.
120-
external_url: http://localhost:9093
128+
external_url: http://localhost:8080
121129
# The global configuration for the alertmanager. All the exahustive fields can be found in the upstream: https://github.com/prometheus/alertmanager/blob/efa05feffd644ba4accb526e98a8c6545d26a783/config/config.go#L833
122130
global:
123131
# ResolveTimeout is the time after which an alert is declared resolved if it has not been updated.

‎ee/query-service/app/db/reader.go‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ import (
88
"github.com/jmoiron/sqlx"
99

1010
"github.com/SigNoz/signoz/pkg/cache"
11+
"github.com/SigNoz/signoz/pkg/prometheus"
1112
basechr "github.com/SigNoz/signoz/pkg/query-service/app/clickhouseReader"
1213
"github.com/SigNoz/signoz/pkg/query-service/interfaces"
14+
"github.com/SigNoz/signoz/pkg/telemetrystore"
1315
)
1416

1517
type ClickhouseReader struct {
@@ -20,23 +22,19 @@ type ClickhouseReader struct {
2022

2123
func NewDataConnector(
2224
localDB *sqlx.DB,
23-
ch clickhouse.Conn,
24-
promConfigPath string,
25+
telemetryStore telemetrystore.TelemetryStore,
26+
prometheus prometheus.Prometheus,
2527
lm interfaces.FeatureLookup,
2628
cluster string,
2729
useLogsNewSchema bool,
2830
useTraceNewSchema bool,
2931
fluxIntervalForTraceDetail time.Duration,
3032
cache cache.Cache,
3133
) *ClickhouseReader {
32-
chReader := basechr.NewReader(localDB, ch, promConfigPath, lm, cluster, useLogsNewSchema, useTraceNewSchema, fluxIntervalForTraceDetail, cache)
34+
chReader := basechr.NewReader(localDB, telemetryStore, prometheus, lm, cluster, useLogsNewSchema, useTraceNewSchema, fluxIntervalForTraceDetail, cache)
3335
return &ClickhouseReader{
34-
conn: ch,
36+
conn: telemetryStore.ClickhouseDB(),
3537
appdb: localDB,
3638
ClickHouseReader: chReader,
3739
}
3840
}
39-
40-
func (r *ClickhouseReader) Start(readerReady chan bool) {
41-
r.ClickHouseReader.Start(readerReady)
42-
}

‎ee/query-service/app/server.go‎

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,14 @@ import (
1818
"github.com/SigNoz/signoz/ee/query-service/constants"
1919
"github.com/SigNoz/signoz/ee/query-service/dao"
2020
"github.com/SigNoz/signoz/ee/query-service/integrations/gateway"
21-
"github.com/SigNoz/signoz/ee/query-service/interfaces"
2221
"github.com/SigNoz/signoz/ee/query-service/rules"
2322
"github.com/SigNoz/signoz/pkg/alertmanager"
2423
"github.com/SigNoz/signoz/pkg/http/middleware"
24+
"github.com/SigNoz/signoz/pkg/prometheus"
2525
"github.com/SigNoz/signoz/pkg/query-service/auth"
2626
"github.com/SigNoz/signoz/pkg/signoz"
2727
"github.com/SigNoz/signoz/pkg/sqlstore"
28+
"github.com/SigNoz/signoz/pkg/telemetrystore"
2829
"github.com/SigNoz/signoz/pkg/types"
2930
"github.com/SigNoz/signoz/pkg/types/authtypes"
3031
"github.com/SigNoz/signoz/pkg/web"
@@ -49,7 +50,6 @@ import (
4950
"github.com/SigNoz/signoz/pkg/query-service/healthcheck"
5051
baseint "github.com/SigNoz/signoz/pkg/query-service/interfaces"
5152
basemodel "github.com/SigNoz/signoz/pkg/query-service/model"
52-
pqle "github.com/SigNoz/signoz/pkg/query-service/pqlEngine"
5353
baserules "github.com/SigNoz/signoz/pkg/query-service/rules"
5454
"github.com/SigNoz/signoz/pkg/query-service/telemetry"
5555
"github.com/SigNoz/signoz/pkg/query-service/utils"
@@ -137,27 +137,23 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
137137

138138
// set license manager as feature flag provider in dao
139139
modelDao.SetFlagProvider(lm)
140-
readerReady := make(chan bool)
141140

142141
fluxIntervalForTraceDetail, err := time.ParseDuration(serverOptions.FluxIntervalForTraceDetail)
143142
if err != nil {
144143
return nil, err
145144
}
146145

147-
var reader interfaces.DataConnector
148-
qb := db.NewDataConnector(
146+
reader := db.NewDataConnector(
149147
serverOptions.SigNoz.SQLStore.SQLxDB(),
150-
serverOptions.SigNoz.TelemetryStore.ClickHouseDB(),
151-
serverOptions.PromConfigPath,
148+
serverOptions.SigNoz.TelemetryStore,
149+
serverOptions.SigNoz.Prometheus,
152150
lm,
153151
serverOptions.Cluster,
154152
serverOptions.UseLogsNewSchema,
155153
serverOptions.UseTraceNewSchema,
156154
fluxIntervalForTraceDetail,
157155
serverOptions.SigNoz.Cache,
158156
)
159-
go qb.Start(readerReady)
160-
reader = qb
161157

162158
skipConfig := &basemodel.SkipConfig{}
163159
if serverOptions.SkipTopLvlOpsPath != "" {
@@ -176,9 +172,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
176172
c = cache.NewCache(cacheOpts)
177173
}
178174

179-
<-readerReady
180175
rm, err := makeRulesManager(
181-
serverOptions.PromConfigPath,
182176
serverOptions.RuleRepoURL,
183177
serverOptions.SigNoz.SQLStore.SQLxDB(),
184178
reader,
@@ -189,6 +183,8 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
189183
serverOptions.UseTraceNewSchema,
190184
serverOptions.SigNoz.Alertmanager,
191185
serverOptions.SigNoz.SQLStore,
186+
serverOptions.SigNoz.TelemetryStore,
187+
serverOptions.SigNoz.Prometheus,
192188
)
193189

194190
if err != nil {
@@ -233,7 +229,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
233229
}
234230

235231
// start the usagemanager
236-
usageManager, err := usage.New(modelDao, lm.GetRepo(), serverOptions.SigNoz.TelemetryStore.ClickHouseDB(), serverOptions.Config.TelemetryStore.ClickHouse.DSN)
232+
usageManager, err := usage.New(modelDao, lm.GetRepo(), serverOptions.SigNoz.TelemetryStore.ClickhouseDB(), serverOptions.Config.TelemetryStore.Clickhouse.DSN)
237233
if err != nil {
238234
return nil, err
239235
}
@@ -304,7 +300,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
304300
&opAmpModel.AllAgents, agentConfMgr,
305301
)
306302

307-
errorList := qb.PreloadMetricsMetadata(context.Background())
303+
errorList := reader.PreloadMetricsMetadata(context.Background())
308304
for _, er := range errorList {
309305
zap.L().Error("failed to preload metrics metadata", zap.Error(er))
310306
}
@@ -537,7 +533,6 @@ func (s *Server) Stop() error {
537533
}
538534

539535
func makeRulesManager(
540-
promConfigPath,
541536
ruleRepoURL string,
542537
db *sqlx.DB,
543538
ch baseint.Reader,
@@ -548,16 +543,13 @@ func makeRulesManager(
548543
useTraceNewSchema bool,
549544
alertmanager alertmanager.Alertmanager,
550545
sqlstore sqlstore.SQLStore,
546+
telemetryStore telemetrystore.TelemetryStore,
547+
prometheus prometheus.Prometheus,
551548
) (*baserules.Manager, error) {
552-
// create engine
553-
pqle, err := pqle.FromConfigPath(promConfigPath)
554-
if err != nil {
555-
return nil, fmt.Errorf("failed to create pql engine : %v", err)
556-
}
557-
558549
// create manager opts
559550
managerOpts := &baserules.ManagerOptions{
560-
PqlEngine: pqle,
551+
TelemetryStore: telemetryStore,
552+
Prometheus: prometheus,
561553
RepoURL: ruleRepoURL,
562554
DBConn: db,
563555
Context: context.Background(),

‎ee/query-service/interfaces/connector.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,5 @@ import (
77
// Connector defines methods for interaction
88
// with o11y data. for example - clickhouse
99
type DataConnector interface {
10-
Start(readerReady chan bool)
1110
baseint.Reader
1211
}

‎ee/query-service/main.go‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ import (
1818
"github.com/SigNoz/signoz/pkg/types/authtypes"
1919
"github.com/SigNoz/signoz/pkg/version"
2020

21-
prommodel "github.com/prometheus/common/model"
22-
2321
"go.uber.org/zap"
2422
"go.uber.org/zap/zapcore"
2523
)
@@ -32,10 +30,6 @@ func initZapLog() *zap.Logger {
3230
return logger
3331
}
3432

35-
func init() {
36-
prommodel.NameValidationScheme = prommodel.UTF8Validation
37-
}
38-
3933
func main() {
4034
var promConfigPath, skipTopLvlOpsPath string
4135

@@ -89,6 +83,7 @@ func main() {
8983
MaxIdleConns: maxIdleConns,
9084
MaxOpenConns: maxOpenConns,
9185
DialTimeout: dialTimeout,
86+
Config: promConfigPath,
9287
})
9388
if err != nil {
9489
zap.L().Fatal("Failed to create config", zap.Error(err))
@@ -110,7 +105,7 @@ func main() {
110105
signoz.NewTelemetryStoreProviderFactories(),
111106
)
112107
if err != nil {
113-
zap.L().Fatal("Failed to create signoz struct", zap.Error(err))
108+
zap.L().Fatal("Failed to create signoz", zap.Error(err))
114109
}
115110

116111
jwtSecret := os.Getenv("SIGNOZ_JWT_SECRET")

‎ee/query-service/rules/manager.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func PrepareTaskFunc(opts baserules.PrepareTaskOptions) (baserules.Task, error)
4848
opts.Rule,
4949
opts.Logger,
5050
opts.Reader,
51-
opts.ManagerOpts.PqlEngine,
51+
opts.ManagerOpts.Prometheus,
5252
baserules.WithSQLStore(opts.SQLStore),
5353
)
5454

@@ -145,7 +145,7 @@ func TestNotification(opts baserules.PrepareTestRuleOptions) (int, *basemodel.Ap
145145
parsedRule,
146146
opts.Logger,
147147
opts.Reader,
148-
opts.ManagerOpts.PqlEngine,
148+
opts.ManagerOpts.Prometheus,
149149
baserules.WithSendAlways(),
150150
baserules.WithSendUnmatched(),
151151
baserules.WithSQLStore(opts.SQLStore),

0 commit comments

Comments
 (0)