diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c0092351bc47603494bb7ae28515d8c9a946a99..0ff9d396e2b632df57d6cdce0d7a75d401c28774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Split image registry and repository in Helm chart - (BREAKING) Internally Sloth (not k8s) prometheusServiceLevel uses k8s `k8s.io/apimachinery/pkg/util/yaml` lib for unmarshaling YAML instead of `gopkg.in/yaml.v2`. - Core SLO validation and SLO rules generation migrated to SLO plugins. +- (BREAKING) `--sli-plugins-path`, `--slo-plugins-path`, `-m` args and it's env vars `SLOTH_SLI_PLUGINS_PATH`and `SLOTH_SLO_PLUGINS_PATH` have been removed in favor or `--plugins-path`, `-p` and it's env var `SLOTH_PLUGINS_PATH` that discovers and loads SLI and SLO plugins with a single flag. ### Added diff --git a/cmd/sloth/commands/generate.go b/cmd/sloth/commands/generate.go index 5bf337a6a284e762e16191e9719776dd25aa62ad..1a3aa750af674231c13c9c699b6b5dd449cd3066 100644 --- a/cmd/sloth/commands/generate.go +++ b/cmd/sloth/commands/generate.go @@ -41,8 +41,7 @@ type generateCommand struct { disableAlerts bool disableOptimizedRules bool extraLabels map[string]string - sliPluginsPaths []string - sloPluginsPaths []string + pluginsPaths []string sloPeriodWindowsPath string sloPeriod string } @@ -59,8 +58,7 @@ func NewGenerateCommand(app *kingpin.Application) Command { cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels) cmd.Flag("disable-recordings", "Disables recording rules generation.").BoolVar(&c.disableRecordings) cmd.Flag("disable-alerts", "Disables alert rules generation.").BoolVar(&c.disableAlerts) - cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths) - cmd.Flag("slo-plugins-path", "The path to SLO plugins a.k.a SLO generation middlewares (can be repeated).").Short('m').StringsVar(&c.sloPluginsPaths) + cmd.Flag("plugins-path", "The path to SLI and SLO plugins (can be repeated).").Short('p').StringsVar(&c.pluginsPaths) cmd.Flag("slo-period-windows-path", "The directory path to custom SLO period windows catalog (replaces default ones).").StringVar(&c.sloPeriodWindowsPath) cmd.Flag("default-slo-period", "The default SLO period windows to be used for the SLOs.").Default("30d").StringVar(&c.sloPeriod) cmd.Flag("disable-optimized-rules", "If enabled it will disable optimized generated rules.").BoolVar(&c.disableOptimizedRules) @@ -112,13 +110,8 @@ func (g generateCommand) Run(ctx context.Context, config RootConfig) error { "out": g.slosOut, }) - // Load plugins - pluginSLIRepo, err := createSLIPluginLoader(ctx, logger, g.sliPluginsPaths) - if err != nil { - return err - } - - pluginSLORepo, err := createSLOPluginLoader(ctx, logger, g.sloPluginsPaths) + // Load plugins. + pluginsRepo, err := createPluginLoader(ctx, logger, g.pluginsPaths) if err != nil { return err } @@ -143,8 +136,8 @@ func (g generateCommand) Run(ctx context.Context, config RootConfig) error { } // Create Spec loaders. - promYAMLLoader := storageio.NewSlothPrometheusYAMLSpecLoader(pluginSLIRepo, sloPeriod) - kubeYAMLLoader := storageio.NewK8sSlothPrometheusYAMLSpecLoader(pluginSLIRepo, sloPeriod) + promYAMLLoader := storageio.NewSlothPrometheusYAMLSpecLoader(pluginsRepo, sloPeriod) + kubeYAMLLoader := storageio.NewK8sSlothPrometheusYAMLSpecLoader(pluginsRepo, sloPeriod) openSLOYAMLLoader := storageio.NewOpenSLOYAMLSpecLoader(sloPeriod) // Get SLO targets. @@ -257,7 +250,7 @@ func (g generateCommand) Run(ctx context.Context, config RootConfig) error { disableAlerts: g.disableAlerts, disableOptimizedRules: g.disableOptimizedRules, extraLabels: g.extraLabels, - sloPluginRepo: pluginSLORepo, + sloPluginRepo: pluginsRepo, } for _, genTarget := range genTargets { @@ -318,7 +311,7 @@ type generator struct { disableAlerts bool disableOptimizedRules bool extraLabels map[string]string - sloPluginRepo *storagefs.FileSLOPluginRepo + sloPluginRepo *storagefs.FilePluginRepo } // GeneratePrometheus generates the SLOs based on a raw regular Prometheus spec format input and outs a Prometheus raw yaml. diff --git a/cmd/sloth/commands/helpers.go b/cmd/sloth/commands/helpers.go index c0b4c49edc67a9ab2ae8e5d0e2279fce89eeb9a4..72c05b19a08f74bac049c39a0ee52f778b0f4a5d 100644 --- a/cmd/sloth/commands/helpers.go +++ b/cmd/sloth/commands/helpers.go @@ -42,21 +42,7 @@ func splitYAML(data []byte) []string { return nonEmptyData } -func createSLIPluginLoader(ctx context.Context, logger log.Logger, paths []string) (*storagefs.FileSLIPluginRepo, error) { - config := storagefs.FileSLIPluginRepoConfig{ - Paths: paths, - PluginLoader: pluginenginesli.PluginLoader, - Logger: logger, - } - sliPluginRepo, err := storagefs.NewFileSLIPluginRepo(config) - if err != nil { - return nil, fmt.Errorf("could not create file SLI plugin repository: %w", err) - } - - return sliPluginRepo, nil -} - -func createSLOPluginLoader(ctx context.Context, logger log.Logger, paths []string) (*storagefs.FileSLOPluginRepo, error) { +func createPluginLoader(ctx context.Context, logger log.Logger, paths []string) (*storagefs.FilePluginRepo, error) { fss := []fs.FS{ plugin.EmbeddedDefaultSLOPlugins, } @@ -64,12 +50,12 @@ func createSLOPluginLoader(ctx context.Context, logger log.Logger, paths []strin fss = append(fss, os.DirFS(p)) } - sliPluginRepo, err := storagefs.NewFileSLOPluginRepo(logger, pluginengineslo.PluginLoader, fss...) + pluginsRepo, err := storagefs.NewFilePluginRepo(logger, pluginenginesli.PluginLoader, pluginengineslo.PluginLoader, fss...) if err != nil { - return nil, fmt.Errorf("could not create file SLO plugin repository: %w", err) + return nil, fmt.Errorf("could not create file SLO and SLI plugins repository: %w", err) } - return sliPluginRepo, nil + return pluginsRepo, nil } func discoverSLOManifests(logger log.Logger, exclude, include *regexp.Regexp, path string) ([]string, error) { diff --git a/cmd/sloth/commands/k8scontroller.go b/cmd/sloth/commands/k8scontroller.go index e57a834cabeb16dc501d877a15094910527029b5..087d37ca0d9e8122695130cbd77996672e4bd669 100644 --- a/cmd/sloth/commands/k8scontroller.go +++ b/cmd/sloth/commands/k8scontroller.go @@ -69,8 +69,7 @@ type kubeControllerCommand struct { hotReloadPath string hotReloadAddr string metricsListenAddr string - sliPluginsPaths []string - sloPluginsPaths []string + pluginsPaths []string sloPeriodWindowsPath string sloPeriod string disableOptimizedRules bool @@ -97,8 +96,7 @@ func NewKubeControllerCommand(app *kingpin.Application) Command { cmd.Flag("hot-reload-addr", "The listen address for hot-reloading components that allow it.").Default(":8082").StringVar(&c.hotReloadAddr) cmd.Flag("hot-reload-path", "The webhook path for hot-reloading components that allow it.").Default("/-/reload").StringVar(&c.hotReloadPath) cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels) - cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths) - cmd.Flag("slo-plugins-path", "The path to SLO plugins a.k.a SLO generation middlewares (can be repeated).").Short('m').StringsVar(&c.sloPluginsPaths) + cmd.Flag("plugins-path", "The path to SLI and SLO plugins (can be repeated).").Short('p').StringsVar(&c.pluginsPaths) cmd.Flag("slo-period-windows-path", "The directory path to custom SLO period windows catalog (replaces default ones).").StringVar(&c.sloPeriodWindowsPath) cmd.Flag("default-slo-period", "The default SLO period windows to be used for the SLOs.").Default("30d").StringVar(&c.sloPeriod) cmd.Flag("disable-optimized-rules", "If enabled it will disable optimized generated rules.").BoolVar(&c.disableOptimizedRules) @@ -118,12 +116,7 @@ func (k kubeControllerCommand) Run(ctx context.Context, config RootConfig) error sloPeriod := time.Duration(sp) // Plugins. - pluginSLIRepo, err := createSLIPluginLoader(ctx, logger, k.sliPluginsPaths) - if err != nil { - return err - } - - pluginSLORepo, err := createSLOPluginLoader(ctx, logger, k.sloPluginsPaths) + pluginsRepo, err := createPluginLoader(ctx, logger, k.pluginsPaths) if err != nil { return err } @@ -169,11 +162,7 @@ func (k kubeControllerCommand) Run(ctx context.Context, config RootConfig) error { // Set SLI plugin repository reloader. reloadManager.Add(1000, reload.ReloaderFunc(func(ctx context.Context, id string) error { - return pluginSLIRepo.Reload(ctx) - })) - - reloadManager.Add(1000, reload.ReloaderFunc(func(ctx context.Context, id string) error { - return pluginSLORepo.Reload(ctx) + return pluginsRepo.Reload(ctx) })) ctx, cancel := context.WithCancel(ctx) @@ -356,7 +345,7 @@ func (k kubeControllerCommand) Run(ctx context.Context, config RootConfig) error MetadataRulesGenSLOPlugin: metaRuleGen, AlertRulesGenSLOPlugin: alertRuleGen, ValidateSLOPlugin: validatePlugin, - SLOPluginGetter: pluginSLORepo, + SLOPluginGetter: pluginsRepo, Logger: generatorLogger{Logger: logger}, }) if err != nil { @@ -366,7 +355,7 @@ func (k kubeControllerCommand) Run(ctx context.Context, config RootConfig) error // Create handler. controllerConfig := kubecontroller.HandlerConfig{ Generator: generator, - SpecLoader: storageio.NewK8sSlothPrometheusCRSpecLoader(pluginSLIRepo, sloPeriod), + SpecLoader: storageio.NewK8sSlothPrometheusCRSpecLoader(pluginsRepo, sloPeriod), Repository: kuberepo, KubeStatusStorer: kuberepo, ExtraLabels: k.extraLabels, diff --git a/cmd/sloth/commands/validate.go b/cmd/sloth/commands/validate.go index f752e6276bc225372982529c959d021747957aea..06628dccc121b86b06c75c8ef08fdfddffcd3e05 100644 --- a/cmd/sloth/commands/validate.go +++ b/cmd/sloth/commands/validate.go @@ -22,8 +22,7 @@ type validateCommand struct { slosExcludeRegex string slosIncludeRegex string extraLabels map[string]string - sliPluginsPaths []string - sloPluginsPaths []string + pluginsPaths []string sloPeriodWindowsPath string sloPeriod string } @@ -36,8 +35,7 @@ func NewValidateCommand(app *kingpin.Application) Command { cmd.Flag("fs-exclude", "Filter regex to ignore matched discovered SLO file paths.").Short('e').StringVar(&c.slosExcludeRegex) cmd.Flag("fs-include", "Filter regex to include matched discovered SLO file paths, everything else will be ignored. Exclude has preference.").Short('n').StringVar(&c.slosIncludeRegex) cmd.Flag("extra-labels", "Extra labels that will be added to all the generated Prometheus rules ('key=value' form, can be repeated).").Short('l').StringMapVar(&c.extraLabels) - cmd.Flag("sli-plugins-path", "The path to SLI plugins (can be repeated), if not set it disable plugins support.").Short('p').StringsVar(&c.sliPluginsPaths) - cmd.Flag("slo-plugins-path", "The path to SLO plugins a.k.a SLO generation middlewares (can be repeated).").Short('m').StringsVar(&c.sloPluginsPaths) + cmd.Flag("plugins-path", "The path to SLI and SLO plugins (can be repeated).").Short('p').StringsVar(&c.pluginsPaths) cmd.Flag("slo-period-windows-path", "The directory path to custom SLO period windows catalog (replaces default ones).").StringVar(&c.sloPeriodWindowsPath) cmd.Flag("default-slo-period", "The default SLO period windows to be used for the SLOs.").Default("30d").StringVar(&c.sloPeriod) @@ -83,12 +81,7 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { } // Load plugins. - pluginSLIRepo, err := createSLIPluginLoader(ctx, logger, v.sliPluginsPaths) - if err != nil { - return err - } - - pluginSLORepo, err := createSLOPluginLoader(ctx, logger, v.sloPluginsPaths) + pluginsRepo, err := createPluginLoader(ctx, logger, v.pluginsPaths) if err != nil { return err } @@ -113,8 +106,8 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { } // Create Spec loaders. - promYAMLLoader := storageio.NewSlothPrometheusYAMLSpecLoader(pluginSLIRepo, sloPeriod) - kubeYAMLLoader := storageio.NewK8sSlothPrometheusYAMLSpecLoader(pluginSLIRepo, sloPeriod) + promYAMLLoader := storageio.NewSlothPrometheusYAMLSpecLoader(pluginsRepo, sloPeriod) + kubeYAMLLoader := storageio.NewK8sSlothPrometheusYAMLSpecLoader(pluginsRepo, sloPeriod) openSLOYAMLLoader := storageio.NewOpenSLOYAMLSpecLoader(sloPeriod) // For every file load the data and start the validation process: @@ -134,7 +127,7 @@ func (v validateCommand) Run(ctx context.Context, config RootConfig) error { logger: log.Noop, windowsRepo: windowsRepo, extraLabels: v.extraLabels, - sloPluginRepo: pluginSLORepo, + sloPluginRepo: pluginsRepo, } // Prepare file validation result and start validation result for every SLO in the file. diff --git a/deploy/kubernetes/helm/sloth/templates/deployment.yaml b/deploy/kubernetes/helm/sloth/templates/deployment.yaml index 3f6ac1070aa3e2d421c24ab3c04d754de482f56c..f856e0928c000d270a8d408a02b5cf6013352a2b 100644 --- a/deploy/kubernetes/helm/sloth/templates/deployment.yaml +++ b/deploy/kubernetes/helm/sloth/templates/deployment.yaml @@ -52,7 +52,7 @@ spec: - --extra-labels={{ $key }}={{ $val }} {{- end}} {{- if .Values.commonPlugins.enabled }} - - --sli-plugins-path=/plugins + - --plugins-path=/plugins {{- end }} {{- with .Values.sloth.defaultSloPeriod }} - --default-slo-period={{ . }} diff --git a/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_custom.yaml b/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_custom.yaml index 538270f2f3ebf79fb9a78f778beaafe0bc435581..ad3832caac29b6bf9595b4333e4fe6d98c88fb13 100644 --- a/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_custom.yaml +++ b/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_custom.yaml @@ -48,7 +48,7 @@ spec: - --label-selector=x=y,z!=y - --extra-labels=k1=v1 - --extra-labels=k2=v2 - - --sli-plugins-path=/plugins + - --plugins-path=/plugins - --disable-optimized-rules - --logger=default ports: diff --git a/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_default.yaml b/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_default.yaml index 40805140d7952725234ec2622cea7489852649db..65091427ddce4e8d7ae69e574a6e7fc1b726a0b3 100644 --- a/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_default.yaml +++ b/deploy/kubernetes/helm/sloth/tests/testdata/output/deployment_default.yaml @@ -35,7 +35,7 @@ spec: image: ghcr.io/slok/sloth:v0.12.0 args: - kubernetes-controller - - --sli-plugins-path=/plugins + - --plugins-path=/plugins - --logger=default ports: - containerPort: 8081 diff --git a/deploy/kubernetes/raw/sloth-with-common-plugins.yaml b/deploy/kubernetes/raw/sloth-with-common-plugins.yaml index 3c56d2ce7d6341108aaefb4083679374e395c95b..2250377276887902be46fb628e02370faeeb0114 100644 --- a/deploy/kubernetes/raw/sloth-with-common-plugins.yaml +++ b/deploy/kubernetes/raw/sloth-with-common-plugins.yaml @@ -88,7 +88,7 @@ spec: image: ghcr.io/slok/sloth:v0.12.0 args: - kubernetes-controller - - --sli-plugins-path=/plugins + - --plugins-path=/plugins - --logger=default ports: - containerPort: 8081 diff --git a/internal/storage/fs/plugin.go b/internal/storage/fs/plugin.go new file mode 100644 index 0000000000000000000000000000000000000000..5c782f6c6bef7e4feff5fe29bf494daee63183c5 --- /dev/null +++ b/internal/storage/fs/plugin.go @@ -0,0 +1,170 @@ +package fs + +import ( + "context" + "fmt" + "io/fs" + "regexp" + "sync" + + "github.com/slok/sloth/internal/log" + pluginenginesli "github.com/slok/sloth/internal/pluginengine/sli" + pluginengineslo "github.com/slok/sloth/internal/pluginengine/slo" + commonerrors "github.com/slok/sloth/pkg/common/errors" +) + +type SLIPluginLoader interface { + LoadRawSLIPlugin(ctx context.Context, src string) (*pluginenginesli.SLIPlugin, error) +} + +//go:generate mockery --case underscore --output fsmock --outpkg fsmock --name SLIPluginLoader + +type SLOPluginLoader interface { + LoadRawPlugin(ctx context.Context, src string) (*pluginengineslo.Plugin, error) +} + +//go:generate mockery --case underscore --output fsmock --outpkg fsmock --name SLOPluginLoader + +type FilePluginRepo struct { + fss []fs.FS + sloPluginLoader SLOPluginLoader + sliPluginLoader SLIPluginLoader + sloPluginCache map[string]pluginengineslo.Plugin + sliPluginCache map[string]pluginenginesli.SLIPlugin + logger log.Logger + mu sync.RWMutex +} + +// NewFilePluginRepo returns a new FilePluginRepo that loads SLI and SLO plugins from the given file system. +func NewFilePluginRepo(logger log.Logger, sliPluginLoader SLIPluginLoader, sloPluginLoader SLOPluginLoader, fss ...fs.FS) (*FilePluginRepo, error) { + r := &FilePluginRepo{ + fss: fss, + sliPluginLoader: sliPluginLoader, + sloPluginLoader: sloPluginLoader, + sloPluginCache: map[string]pluginengineslo.Plugin{}, + sliPluginCache: map[string]pluginenginesli.SLIPlugin{}, + logger: logger, + } + + err := r.Reload(context.Background()) + if err != nil { + return nil, fmt.Errorf("could not load plugins: %w", err) + } + + return r, nil +} + +var pluginNameRegex = regexp.MustCompile("plugin.go$") + +func (r *FilePluginRepo) Reload(ctx context.Context) error { + sloPlugins, sliPlugins, err := r.loadPlugins(ctx, r.fss...) + if err != nil { + return fmt.Errorf("could not load plugins: %w", err) + } + + // Set loaded plugins. + r.mu.Lock() + r.sloPluginCache = sloPlugins + r.sliPluginCache = sliPlugins + r.mu.Unlock() + + r.logger.WithValues(log.Kv{"slo-plugins": len(sloPlugins), "sli-plugins": len(sliPlugins)}).Infof("Plugins loaded") + return nil +} + +func (r *FilePluginRepo) GetSLOPlugin(ctx context.Context, id string) (*pluginengineslo.Plugin, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + p, ok := r.sloPluginCache[id] + if !ok { + return nil, fmt.Errorf("plugin %q not found: %w", id, commonerrors.ErrNotFound) + } + + return &p, nil +} + +func (r *FilePluginRepo) ListSLOPlugins(ctx context.Context) (map[string]pluginengineslo.Plugin, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + return r.sloPluginCache, nil +} + +func (r *FilePluginRepo) GetSLIPlugin(ctx context.Context, id string) (*pluginenginesli.SLIPlugin, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + p, ok := r.sliPluginCache[id] + if !ok { + return nil, fmt.Errorf("plugin %q not found: %w", id, commonerrors.ErrNotFound) + } + + return &p, nil +} + +func (r *FilePluginRepo) ListSLIPlugins(ctx context.Context) (map[string]pluginenginesli.SLIPlugin, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + return r.sliPluginCache, nil +} + +func (r *FilePluginRepo) loadPlugins(ctx context.Context, fss ...fs.FS) (map[string]pluginengineslo.Plugin, map[string]pluginenginesli.SLIPlugin, error) { + sloPlugins := map[string]pluginengineslo.Plugin{} + sliPlugins := map[string]pluginenginesli.SLIPlugin{} + + for _, f := range fss { + err := fs.WalkDir(f, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + if !pluginNameRegex.MatchString(path) { + return nil + } + + pluginDataBytes, err := fs.ReadFile(f, path) + if err != nil { + return fmt.Errorf("could not read %q plugin data: %w", path, err) + } + pluginData := string(pluginDataBytes) + + // Try SLI plugin if not, SLO plugin. + sliPlugin, sliErr := r.sliPluginLoader.LoadRawSLIPlugin(ctx, pluginData) + if sliErr == nil { + _, ok := sliPlugins[sliPlugin.ID] + if ok { + return fmt.Errorf("plugin %q already loaded", sliPlugin.ID) + } + sliPlugins[sliPlugin.ID] = *sliPlugin + r.logger.WithValues(log.Kv{"sli-plugin-id": sliPlugin.ID}).Debugf("SLI plugin discovered and loaded") + return nil + } + + // Try SLO plugin. + sloPlugin, sloErr := r.sloPluginLoader.LoadRawPlugin(ctx, pluginData) + if sloErr == nil { + _, ok := sloPlugins[sloPlugin.ID] + if ok { + return fmt.Errorf("plugin %q already loaded", sloPlugin.ID) + } + sloPlugins[sloPlugin.ID] = *sloPlugin + r.logger.WithValues(log.Kv{"slo-plugin-id": sloPlugin.ID}).Debugf("SLO plugin discovered and loaded") + return nil + } + + r.logger.Errorf("could not load %q as SLI or SLO plugin: (SLI plugin error: %s | SLO plugin error: %s)", path, sliErr, sloErr) + + return nil + }) + if err != nil { + return nil, nil, fmt.Errorf("could not walk dir: %w", err) + } + } + + return sloPlugins, sliPlugins, nil +} diff --git a/internal/storage/fs/plugin_test.go b/internal/storage/fs/plugin_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d8329fd72777c096b250b8c43c17b7bf18527ca6 --- /dev/null +++ b/internal/storage/fs/plugin_test.go @@ -0,0 +1,364 @@ +package fs_test + +import ( + "fmt" + "io/fs" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/slok/sloth/internal/log" + pluginenginesli "github.com/slok/sloth/internal/pluginengine/sli" + pluginengineslo "github.com/slok/sloth/internal/pluginengine/slo" + storagefs "github.com/slok/sloth/internal/storage/fs" + "github.com/slok/sloth/internal/storage/fs/fsmock" +) + +func TestFilePluginRepoListSLOPlugins(t *testing.T) { + tests := map[string]struct { + fss func() []fs.FS + mock func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) + expPlugins map[string]pluginengineslo.Plugin + expLoadErr bool + expErr bool + }{ + "Having no files, should return empty list of plugins.": { + fss: func() []fs.FS { return nil }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) {}, + expPlugins: map[string]pluginengineslo.Plugin{}, + }, + + "Having plugins in multiple FS and directories, should return all plugins.": { + fss: func() []fs.FS { + m1 := make(fstest.MapFS) + m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + m1["m1/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p3")} + + m2 := make(fstest.MapFS) + m2["m2/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p4")} + m2["m2/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p5")} + m2["m2/plugin-test.go"] = &fstest.MapFile{Data: []byte("p8")} // Ignored. + + m3 := make(fstest.MapFS) + m3["m3/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p6")} + m3["m3/plx/pl3/plugin.yaml"] = &fstest.MapFile{Data: []byte("p7")} // Ignored. + + return []fs.FS{m1, m2, m3} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p3").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p4").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p5").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p6").Once().Return(nil, fmt.Errorf("something")) + + mslopl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(&pluginengineslo.Plugin{ID: "p2"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p3").Once().Return(&pluginengineslo.Plugin{ID: "p3"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p4").Once().Return(&pluginengineslo.Plugin{ID: "p4"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p5").Once().Return(&pluginengineslo.Plugin{ID: "p5"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p6").Once().Return(&pluginengineslo.Plugin{ID: "p6"}, nil) + }, + expPlugins: map[string]pluginengineslo.Plugin{ + "p1": {ID: "p1"}, + "p2": {ID: "p2"}, + "p3": {ID: "p3"}, + "p4": {ID: "p4"}, + "p5": {ID: "p5"}, + "p6": {ID: "p6"}, + }, + }, + + "Having a plugin loaded with the same ID multiple times should fail.": { + fss: func() []fs.FS { + m1 := make(fstest.MapFS) + m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + + return []fs.FS{m1} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) + + mslopl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) + + }, + expLoadErr: true, + }, + + "Having an error while loading a plugin, should not fail but don't load the failed plugin.": { + fss: func() []fs.FS { + m1 := make(fstest.MapFS) + m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + + return []fs.FS{m1} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) + + mslopl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) + + }, + expPlugins: map[string]pluginengineslo.Plugin{ + "p1": {ID: "p1"}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + mslopl := fsmock.NewSLOPluginLoader(t) + mslipl := fsmock.NewSLIPluginLoader(t) + test.mock(mslopl, mslipl) + + // Create repository and load plugins. + repo, err := storagefs.NewFilePluginRepo(log.Noop, mslipl, mslopl, test.fss()...) + if test.expLoadErr { + assert.Error(err) + return + } + assert.NoError(err) + + plugins, err := repo.ListSLOPlugins(t.Context()) + if test.expErr { + assert.Error(err) + } else if assert.NoError(err) { + assert.Equal(test.expPlugins, plugins) + } + }) + } +} + +func TestFilePluginRepoGetSLOPlugin(t *testing.T) { + tests := map[string]struct { + pluginID string + fss func() []fs.FS + mock func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) + expPlugin pluginengineslo.Plugin + expErr bool + }{ + "Having no files, should fail.": { + pluginID: "test", + fss: func() []fs.FS { return nil }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) {}, + expErr: true, + }, + + "Getting a correct plugin, should return the plugin.": { + pluginID: "p2", + fss: func() []fs.FS { + m := make(fstest.MapFS) + m["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + return []fs.FS{m} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(nil, fmt.Errorf("something")) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) + + mslopl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) + mslopl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(&pluginengineslo.Plugin{ID: "p2"}, nil) + }, + expPlugin: pluginengineslo.Plugin{ID: "p2"}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + mslopl := fsmock.NewSLOPluginLoader(t) + mslipl := fsmock.NewSLIPluginLoader(t) + test.mock(mslopl, mslipl) + + // Create repository and load plugins. + repo, err := storagefs.NewFilePluginRepo(log.Noop, mslipl, mslopl, test.fss()...) + require.NoError(err) + + plugin, err := repo.GetSLOPlugin(t.Context(), test.pluginID) + if test.expErr { + assert.Error(err) + } else if assert.NoError(err) { + assert.Equal(test.expPlugin, *plugin) + } + }) + } +} + +func TestFilePluginRepoListSLIPlugins(t *testing.T) { + tests := map[string]struct { + fss func() []fs.FS + mock func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) + expPlugins map[string]pluginenginesli.SLIPlugin + expLoadErr bool + expErr bool + }{ + "Having no files, should return empty list of plugins.": { + fss: func() []fs.FS { return nil }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) {}, + expPlugins: map[string]pluginenginesli.SLIPlugin{}, + }, + + "Having plugins in multiple FS and directories, should return all plugins.": { + fss: func() []fs.FS { + m1 := make(fstest.MapFS) + m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + m1["m1/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p3")} + + m2 := make(fstest.MapFS) + m2["m2/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p4")} + m2["m2/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p5")} + m2["m2/plugin-test.go"] = &fstest.MapFile{Data: []byte("p8")} // Ignored. + + m3 := make(fstest.MapFS) + m3["m3/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p6")} + m3["m3/plx/pl3/plugin.yaml"] = &fstest.MapFile{Data: []byte("p7")} // Ignored. + + return []fs.FS{m1, m2, m3} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(&pluginenginesli.SLIPlugin{ID: "p1"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(&pluginenginesli.SLIPlugin{ID: "p2"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p3").Once().Return(&pluginenginesli.SLIPlugin{ID: "p3"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p4").Once().Return(&pluginenginesli.SLIPlugin{ID: "p4"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p5").Once().Return(&pluginenginesli.SLIPlugin{ID: "p5"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p6").Once().Return(&pluginenginesli.SLIPlugin{ID: "p6"}, nil) + }, + expPlugins: map[string]pluginenginesli.SLIPlugin{ + "p1": {ID: "p1"}, + "p2": {ID: "p2"}, + "p3": {ID: "p3"}, + "p4": {ID: "p4"}, + "p5": {ID: "p5"}, + "p6": {ID: "p6"}, + }, + }, + + "Having a plugin loaded with the same ID multiple times should fail.": { + fss: func() []fs.FS { + m1 := make(fstest.MapFS) + m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + + return []fs.FS{m1} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(&pluginenginesli.SLIPlugin{ID: "p1"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(&pluginenginesli.SLIPlugin{ID: "p1"}, nil) + }, + expLoadErr: true, + }, + + "Having an error while loading a plugin, should not fail but don't load the failed plugin.": { + fss: func() []fs.FS { + m1 := make(fstest.MapFS) + m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + + return []fs.FS{m1} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(&pluginenginesli.SLIPlugin{ID: "p1"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) + mslopl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) + + }, + expPlugins: map[string]pluginenginesli.SLIPlugin{ + "p1": {ID: "p1"}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + + mslopl := fsmock.NewSLOPluginLoader(t) + mslipl := fsmock.NewSLIPluginLoader(t) + test.mock(mslopl, mslipl) + + // Create repository and load plugins. + repo, err := storagefs.NewFilePluginRepo(log.Noop, mslipl, mslopl, test.fss()...) + if test.expLoadErr { + assert.Error(err) + return + } + assert.NoError(err) + + plugins, err := repo.ListSLIPlugins(t.Context()) + if test.expErr { + assert.Error(err) + } else if assert.NoError(err) { + assert.Equal(test.expPlugins, plugins) + } + }) + } +} + +func TestFilePluginRepoGetSLIPlugin(t *testing.T) { + tests := map[string]struct { + pluginID string + fss func() []fs.FS + mock func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) + expPlugin pluginenginesli.SLIPlugin + expErr bool + }{ + "Having no files, should fail.": { + pluginID: "test", + fss: func() []fs.FS { return nil }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) {}, + expErr: true, + }, + + "Getting a correct plugin, should return the plugin.": { + pluginID: "p2", + fss: func() []fs.FS { + m := make(fstest.MapFS) + m["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} + m["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} + return []fs.FS{m} + }, + mock: func(mslopl *fsmock.SLOPluginLoader, mslipl *fsmock.SLIPluginLoader) { + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p1").Once().Return(&pluginenginesli.SLIPlugin{ID: "p1"}, nil) + mslipl.On("LoadRawSLIPlugin", mock.Anything, "p2").Once().Return(&pluginenginesli.SLIPlugin{ID: "p2"}, nil) + }, + expPlugin: pluginenginesli.SLIPlugin{ID: "p2"}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + mslopl := fsmock.NewSLOPluginLoader(t) + mslipl := fsmock.NewSLIPluginLoader(t) + test.mock(mslopl, mslipl) + + // Create repository and load plugins. + repo, err := storagefs.NewFilePluginRepo(log.Noop, mslipl, mslopl, test.fss()...) + require.NoError(err) + + plugin, err := repo.GetSLIPlugin(t.Context(), test.pluginID) + if test.expErr { + assert.Error(err) + } else if assert.NoError(err) { + assert.Equal(test.expPlugin, *plugin) + } + }) + } +} diff --git a/internal/storage/fs/sli_plugin.go b/internal/storage/fs/sli_plugin.go deleted file mode 100644 index efa236025d74405df2c17ca9c3c21fbe9234cae1..0000000000000000000000000000000000000000 --- a/internal/storage/fs/sli_plugin.go +++ /dev/null @@ -1,197 +0,0 @@ -package fs - -import ( - "context" - "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" - "sync" - - "github.com/slok/sloth/internal/log" - pluginenginesli "github.com/slok/sloth/internal/pluginengine/sli" -) - -type SLIPluginLoader interface { - LoadRawSLIPlugin(ctx context.Context, src string) (*pluginenginesli.SLIPlugin, error) -} - -//go:generate mockery --case underscore --output fsmock --outpkg fsmock --name SLIPluginLoader - -// FileManager knows how to manage files. -// TODO(slok): Use fs.FS. -type FileManager interface { - FindFiles(ctx context.Context, root string, matcher *regexp.Regexp) (paths []string, err error) - ReadFile(ctx context.Context, path string) (data []byte, err error) -} - -//go:generate mockery --case underscore --output fsmock --outpkg fsmock --name FileManager - -type fileManager struct{} - -func (f fileManager) FindFiles(ctx context.Context, root string, matcher *regexp.Regexp) ([]string, error) { - paths := []string{} - err := filepath.Walk(root, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - return nil - } - - if matcher.MatchString(path) { - paths = append(paths, path) - } - - return nil - }) - - if err != nil { - return nil, fmt.Errorf("could not find files recursively: %w", err) - } - - return paths, nil -} - -func (f fileManager) ReadFile(_ context.Context, path string) ([]byte, error) { - return os.ReadFile(path) -} - -type FileSLIPluginRepoConfig struct { - FileManager FileManager - Paths []string - PluginLoader SLIPluginLoader - Logger log.Logger -} - -func (c *FileSLIPluginRepoConfig) defaults() error { - if c.FileManager == nil { - c.FileManager = fileManager{} - } - - if c.PluginLoader == nil { - c.PluginLoader = pluginenginesli.PluginLoader - } - - if c.Logger == nil { - c.Logger = log.Noop - } - c.Logger = c.Logger.WithValues(log.Kv{"svc": "storage.FileSLIPlugin"}) - - return nil -} - -func NewFileSLIPluginRepo(config FileSLIPluginRepoConfig) (*FileSLIPluginRepo, error) { - err := config.defaults() - if err != nil { - return nil, fmt.Errorf("invalid configuration: %w", err) - } - - f := &FileSLIPluginRepo{ - fileManager: config.FileManager, - pluginLoader: config.PluginLoader, - paths: config.Paths, - logger: config.Logger, - } - - err = f.Reload(context.Background()) - if err != nil { - return nil, fmt.Errorf("could not load plugins: %w", err) - } - - return f, nil -} - -// FileSLIPluginRepo will provide the plugins loaded from files. -// To be able to provide a simple and safe plugin system to the user we have set some -// rules/requirements that a plugin must implement: -// -// - The plugin must be in a `plugin.go` file inside a directory. -// - All the plugin must be in the `plugin.go` file. -// - The plugin can't import anything apart from the Go standard library. -// - `reflect` and `unsafe` packages can't be used. -// -// These rules provide multiple things: -// - Easy discovery of plugins without the need to provide extra data (import paths, path sanitization...). -// - Safety because we don't allow adding external packages easily. -// - Force keeping the plugins simple, small and without smart code. -// - Force avoiding DRY in small plugins and embrace WET to have independent plugins. -type FileSLIPluginRepo struct { - pluginLoader SLIPluginLoader - fileManager FileManager - paths []string - plugins map[string]pluginenginesli.SLIPlugin - mu sync.RWMutex - logger log.Logger -} - -var sliPluginNameRegex = regexp.MustCompile("plugin.go$") - -// Reload will reload all the plugins again from the paths. -func (f *FileSLIPluginRepo) Reload(ctx context.Context) error { - // Discover plugins. - paths := map[string]struct{}{} - for _, path := range f.paths { - discoveredPaths, err := f.fileManager.FindFiles(ctx, path, sliPluginNameRegex) - if err != nil { - return fmt.Errorf("could not discover SLI plugins: %w", err) - } - for _, dPath := range discoveredPaths { - paths[dPath] = struct{}{} - } - } - - // Load the plugins. - plugins := map[string]pluginenginesli.SLIPlugin{} - for path := range paths { - pluginData, err := f.fileManager.ReadFile(ctx, path) - if err != nil { - return fmt.Errorf("could not read %q plugin data: %w", path, err) - } - - // Create the plugin. - plugin, err := f.pluginLoader.LoadRawSLIPlugin(ctx, string(pluginData)) - if err != nil { - return fmt.Errorf("could not load %q plugin: %w", path, err) - } - - // Check collision. - _, ok := plugins[plugin.ID] - if ok { - return fmt.Errorf("2 or more plugins with the same %q ID have been loaded", plugin.ID) - } - - plugins[plugin.ID] = *plugin - f.logger.WithValues(log.Kv{"plugin-id": plugin.ID, "plugin-path": path}).Debugf("SLI plugin loaded") - } - - // Set loaded plugins. - f.mu.Lock() - f.plugins = plugins - f.mu.Unlock() - - f.logger.WithValues(log.Kv{"plugins": len(plugins)}).Infof("SLI plugins loaded") - - return nil -} - -func (f *FileSLIPluginRepo) ListSLIPlugins(ctx context.Context) (map[string]pluginenginesli.SLIPlugin, error) { - f.mu.RLock() - defer f.mu.RUnlock() - - return f.plugins, nil -} - -func (f *FileSLIPluginRepo) GetSLIPlugin(ctx context.Context, id string) (*pluginenginesli.SLIPlugin, error) { - f.mu.RLock() - defer f.mu.RUnlock() - - p, ok := f.plugins[id] - if !ok { - return nil, fmt.Errorf("plugin %q missing", id) - } - - return &p, nil -} diff --git a/internal/storage/fs/sli_plugin_test.go b/internal/storage/fs/sli_plugin_test.go deleted file mode 100644 index cd3ce8c10608770e7eb15e289593cd1663a5c9de..0000000000000000000000000000000000000000 --- a/internal/storage/fs/sli_plugin_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package fs_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - pluginenginesli "github.com/slok/sloth/internal/pluginengine/sli" - "github.com/slok/sloth/internal/storage/fs" - "github.com/slok/sloth/internal/storage/fs/fsmock" -) - -func TestSLIPluginRepoListSLIPlugins(t *testing.T) { - tests := map[string]struct { - mock func(mfm *fsmock.FileManager, mpl *fsmock.SLIPluginLoader) - expPlugins map[string]pluginenginesli.SLIPlugin - expErr bool - }{ - "Having no files, should return empty list of plugins.": { - mock: func(mfm *fsmock.FileManager, mpl *fsmock.SLIPluginLoader) { - mfm.On("FindFiles", mock.Anything, "./", mock.Anything).Once().Return([]string{}, nil) - }, - expPlugins: map[string]pluginenginesli.SLIPlugin{}, - }, - - "Having multiple files, should return multiple plugins.": { - mock: func(mfm *fsmock.FileManager, mpl *fsmock.SLIPluginLoader) { - mfm.On("FindFiles", mock.Anything, "./", mock.Anything).Once().Return([]string{ - "./test_plugin_1.go", - "./test_plugin_2.go", - "./test2/test_plugin_3.go", - "./test3/test4/test_plugin_4.go", - }, nil) - - mfm.On("ReadFile", mock.Anything, "./test_plugin_1.go").Once().Return([]byte(`test1`), nil) - mfm.On("ReadFile", mock.Anything, "./test_plugin_2.go").Once().Return([]byte(`test2`), nil) - mfm.On("ReadFile", mock.Anything, "./test2/test_plugin_3.go").Once().Return([]byte(`test3`), nil) - mfm.On("ReadFile", mock.Anything, "./test3/test4/test_plugin_4.go").Once().Return([]byte(`test4`), nil) - - mpl.On("LoadRawSLIPlugin", mock.Anything, "test1").Once().Return(&pluginenginesli.SLIPlugin{ID: "test1"}, nil) - mpl.On("LoadRawSLIPlugin", mock.Anything, "test2").Once().Return(&pluginenginesli.SLIPlugin{ID: "test2"}, nil) - mpl.On("LoadRawSLIPlugin", mock.Anything, "test3").Once().Return(&pluginenginesli.SLIPlugin{ID: "test3"}, nil) - mpl.On("LoadRawSLIPlugin", mock.Anything, "test4").Once().Return(&pluginenginesli.SLIPlugin{ID: "test4"}, nil) - }, - expPlugins: map[string]pluginenginesli.SLIPlugin{ - "test1": {ID: "test1"}, - "test2": {ID: "test2"}, - "test3": {ID: "test3"}, - "test4": {ID: "test4"}, - }, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - mfm := fsmock.NewFileManager(t) - mpl := fsmock.NewSLIPluginLoader(t) - test.mock(mfm, mpl) - - // Create repository and load plugins. - config := fs.FileSLIPluginRepoConfig{ - FileManager: mfm, - PluginLoader: mpl, - Paths: []string{"./"}, - } - repo, err := fs.NewFileSLIPluginRepo(config) - require.NoError(err) - - plugins, err := repo.ListSLIPlugins(t.Context()) - if test.expErr { - assert.Error(err) - } else if assert.NoError(err) { - assert.Equal(test.expPlugins, plugins) - } - }) - } -} - -func TestSLIPluginRepoGetSLIPlugin(t *testing.T) { - tests := map[string]struct { - pluginID string - mock func(mfm *fsmock.FileManager, mpl *fsmock.SLIPluginLoader) - expPlugin pluginenginesli.SLIPlugin - expErr bool - }{ - "Having a missing plugin, should fail.": { - pluginID: "test3", - mock: func(mfm *fsmock.FileManager, mpl *fsmock.SLIPluginLoader) { - mfm.On("FindFiles", mock.Anything, "./", mock.Anything).Once().Return([]string{ - "./test_plugin_1.go", - "./test_plugin_2.go", - }, nil) - - mfm.On("ReadFile", mock.Anything, "./test_plugin_1.go").Once().Return([]byte(`test1`), nil) - mfm.On("ReadFile", mock.Anything, "./test_plugin_2.go").Once().Return([]byte(`test2`), nil) - - mpl.On("LoadRawSLIPlugin", mock.Anything, "test1").Once().Return(&pluginenginesli.SLIPlugin{ID: "test1"}, nil) - mpl.On("LoadRawSLIPlugin", mock.Anything, "test2").Once().Return(&pluginenginesli.SLIPlugin{ID: "test2"}, nil) - }, - expErr: true, - }, - - "Having a correct plugin, should return the plugin.": { - pluginID: "test2", - mock: func(mfm *fsmock.FileManager, mpl *fsmock.SLIPluginLoader) { - mfm.On("FindFiles", mock.Anything, "./", mock.Anything).Once().Return([]string{ - "./test_plugin_1.go", - "./test_plugin_2.go", - }, nil) - - mfm.On("ReadFile", mock.Anything, "./test_plugin_1.go").Once().Return([]byte(`test1`), nil) - mfm.On("ReadFile", mock.Anything, "./test_plugin_2.go").Once().Return([]byte(`test2`), nil) - - mpl.On("LoadRawSLIPlugin", mock.Anything, "test1").Once().Return(&pluginenginesli.SLIPlugin{ID: "test1"}, nil) - mpl.On("LoadRawSLIPlugin", mock.Anything, "test2").Once().Return(&pluginenginesli.SLIPlugin{ID: "test2"}, nil) - }, - expPlugin: pluginenginesli.SLIPlugin{ID: "test2"}, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - mfm := fsmock.NewFileManager(t) - mpl := fsmock.NewSLIPluginLoader(t) - test.mock(mfm, mpl) - - // Create repository and load plugins. - config := fs.FileSLIPluginRepoConfig{ - FileManager: mfm, - PluginLoader: mpl, - Paths: []string{"./"}, - } - repo, err := fs.NewFileSLIPluginRepo(config) - require.NoError(err) - - plugin, err := repo.GetSLIPlugin(t.Context(), test.pluginID) - if test.expErr { - assert.Error(err) - } else if assert.NoError(err) { - assert.Equal(test.expPlugin, *plugin) - } - }) - } -} diff --git a/internal/storage/fs/slo_plugin.go b/internal/storage/fs/slo_plugin.go deleted file mode 100644 index b1f17395f57330b0156005455a94776a30ea9363..0000000000000000000000000000000000000000 --- a/internal/storage/fs/slo_plugin.go +++ /dev/null @@ -1,124 +0,0 @@ -package fs - -import ( - "context" - "fmt" - "io/fs" - "regexp" - "sync" - - "github.com/slok/sloth/internal/log" - pluginengineslo "github.com/slok/sloth/internal/pluginengine/slo" - commonerrors "github.com/slok/sloth/pkg/common/errors" -) - -type SLOPluginLoader interface { - LoadRawPlugin(ctx context.Context, src string) (*pluginengineslo.Plugin, error) -} - -//go:generate mockery --case underscore --output fsmock --outpkg fsmock --name SLOPluginLoader - -type FileSLOPluginRepo struct { - fss []fs.FS - pluginLoader SLOPluginLoader - cache map[string]pluginengineslo.Plugin - logger log.Logger - mu sync.RWMutex -} - -// NewFileSLOPluginRepo returns a new FileSLOPluginRepo that loads plugins from the given file system. -// The plugin file should be called "plugin.go". -func NewFileSLOPluginRepo(logger log.Logger, pluginLoader SLOPluginLoader, fss ...fs.FS) (*FileSLOPluginRepo, error) { - r := &FileSLOPluginRepo{ - fss: fss, - pluginLoader: pluginLoader, - cache: map[string]pluginengineslo.Plugin{}, - logger: logger, - } - - err := r.Reload(context.Background()) - if err != nil { - return nil, fmt.Errorf("could not load plugins: %w", err) - } - - return r, nil -} - -var sloPluginNameRegex = regexp.MustCompile("plugin.go$") - -func (r *FileSLOPluginRepo) Reload(ctx context.Context) error { - plugins, err := r.loadPlugins(ctx, r.fss...) - if err != nil { - return fmt.Errorf("could not load plugins: %w", err) - } - - // Set loaded plugins. - r.mu.Lock() - r.cache = plugins - r.mu.Unlock() - - r.logger.WithValues(log.Kv{"plugins": len(plugins)}).Infof("SLO plugins loaded") - return nil -} - -func (r *FileSLOPluginRepo) GetSLOPlugin(ctx context.Context, id string) (*pluginengineslo.Plugin, error) { - r.mu.RLock() - defer r.mu.RUnlock() - - p, ok := r.cache[id] - if !ok { - return nil, fmt.Errorf("plugin %q not found: %w", id, commonerrors.ErrNotFound) - } - - return &p, nil -} - -func (r *FileSLOPluginRepo) ListSLOPlugins(ctx context.Context) (map[string]pluginengineslo.Plugin, error) { - r.mu.RLock() - defer r.mu.RUnlock() - - return r.cache, nil -} - -func (r *FileSLOPluginRepo) loadPlugins(ctx context.Context, fss ...fs.FS) (map[string]pluginengineslo.Plugin, error) { - allPlugins := map[string]pluginengineslo.Plugin{} - - for _, f := range fss { - err := fs.WalkDir(f, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - - if !sloPluginNameRegex.MatchString(path) { - return nil - } - - pluginData, err := fs.ReadFile(f, path) - if err != nil { - return fmt.Errorf("could not read %q plugin data: %w", path, err) - } - - plugin, err := r.pluginLoader.LoadRawPlugin(ctx, string(pluginData)) - if err != nil { - return fmt.Errorf("could not load %q plugin: %w", path, err) - } - - _, ok := allPlugins[plugin.ID] - if ok { - return fmt.Errorf("plugin %q already loaded", plugin.ID) - } - allPlugins[plugin.ID] = *plugin - r.logger.WithValues(log.Kv{"plugin-id": plugin.ID}).Debugf("SLO plugin discovered and loaded") - - return nil - }) - if err != nil { - return nil, fmt.Errorf("could not walk dir: %w", err) - } - } - - return allPlugins, nil -} diff --git a/internal/storage/fs/slo_plugin_test.go b/internal/storage/fs/slo_plugin_test.go deleted file mode 100644 index 8a36bfa276dc86aba88ca6bb570daf326b6c4b34..0000000000000000000000000000000000000000 --- a/internal/storage/fs/slo_plugin_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package fs_test - -import ( - "fmt" - "io/fs" - "testing" - "testing/fstest" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/slok/sloth/internal/log" - pluginengineslo "github.com/slok/sloth/internal/pluginengine/slo" - storagefs "github.com/slok/sloth/internal/storage/fs" - "github.com/slok/sloth/internal/storage/fs/fsmock" -) - -func TestFileSLOPluginRepoListSLOPlugins(t *testing.T) { - tests := map[string]struct { - fss func() []fs.FS - mock func(mpl *fsmock.SLOPluginLoader) - expPlugins map[string]pluginengineslo.Plugin - expLoadErr bool - expErr bool - }{ - "Having no files, should return empty list of plugins.": { - fss: func() []fs.FS { return nil }, - mock: func(mpl *fsmock.SLOPluginLoader) {}, - expPlugins: map[string]pluginengineslo.Plugin{}, - }, - - "Having plugins in multiple FS and directories, should return all plugins.": { - fss: func() []fs.FS { - m1 := make(fstest.MapFS) - m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} - m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} - m1["m1/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p3")} - - m2 := make(fstest.MapFS) - m2["m2/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p4")} - m2["m2/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p5")} - m2["m2/plugin-test.go"] = &fstest.MapFile{Data: []byte("p8")} // Ignored. - - m3 := make(fstest.MapFS) - m3["m3/plx/pl3/plugin.go"] = &fstest.MapFile{Data: []byte("p6")} - m3["m3/plx/pl3/plugin.yaml"] = &fstest.MapFile{Data: []byte("p7")} // Ignored. - - return []fs.FS{m1, m2, m3} - }, - mock: func(mpl *fsmock.SLOPluginLoader) { - mpl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(&pluginengineslo.Plugin{ID: "p2"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p3").Once().Return(&pluginengineslo.Plugin{ID: "p3"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p4").Once().Return(&pluginengineslo.Plugin{ID: "p4"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p5").Once().Return(&pluginengineslo.Plugin{ID: "p5"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p6").Once().Return(&pluginengineslo.Plugin{ID: "p6"}, nil) - }, - expPlugins: map[string]pluginengineslo.Plugin{ - "p1": {ID: "p1"}, - "p2": {ID: "p2"}, - "p3": {ID: "p3"}, - "p4": {ID: "p4"}, - "p5": {ID: "p5"}, - "p6": {ID: "p6"}, - }, - }, - - "Having a plugin loaded with the same ID multiple times should fail.": { - fss: func() []fs.FS { - m1 := make(fstest.MapFS) - m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} - m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} - - return []fs.FS{m1} - }, - mock: func(mpl *fsmock.SLOPluginLoader) { - mpl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) - - }, - expLoadErr: true, - }, - - "Having an error while loading a plugin, should fail.": { - fss: func() []fs.FS { - m1 := make(fstest.MapFS) - m1["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} - m1["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} - - return []fs.FS{m1} - }, - mock: func(mpl *fsmock.SLOPluginLoader) { - mpl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(nil, fmt.Errorf("something")) - - }, - expLoadErr: true, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - - mpl := fsmock.NewSLOPluginLoader(t) - test.mock(mpl) - - // Create repository and load plugins. - repo, err := storagefs.NewFileSLOPluginRepo(log.Noop, mpl, test.fss()...) - if test.expLoadErr { - assert.Error(err) - return - } - assert.NoError(err) - - plugins, err := repo.ListSLOPlugins(t.Context()) - if test.expErr { - assert.Error(err) - } else if assert.NoError(err) { - assert.Equal(test.expPlugins, plugins) - } - }) - } -} - -func TestFileSLOPluginRepoGetSLOPlugin(t *testing.T) { - tests := map[string]struct { - pluginID string - fss func() []fs.FS - mock func(mpl *fsmock.SLOPluginLoader) - expPlugin pluginengineslo.Plugin - expErr bool - }{ - "Having no files, should return empty list of plugins.": { - pluginID: "test", - fss: func() []fs.FS { return nil }, - mock: func(mpl *fsmock.SLOPluginLoader) {}, - expErr: true, - }, - - "Getting a correct plugin, should return the plugin.": { - pluginID: "p2", - fss: func() []fs.FS { - m := make(fstest.MapFS) - m["m1/pl1/plugin.go"] = &fstest.MapFile{Data: []byte("p1")} - m["m1/pl2/plugin.go"] = &fstest.MapFile{Data: []byte("p2")} - return []fs.FS{m} - }, - mock: func(mpl *fsmock.SLOPluginLoader) { - mpl.On("LoadRawPlugin", mock.Anything, "p1").Once().Return(&pluginengineslo.Plugin{ID: "p1"}, nil) - mpl.On("LoadRawPlugin", mock.Anything, "p2").Once().Return(&pluginengineslo.Plugin{ID: "p2"}, nil) - }, - expPlugin: pluginengineslo.Plugin{ID: "p2"}, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - mpl := fsmock.NewSLOPluginLoader(t) - test.mock(mpl) - - // Create repository and load plugins. - repo, err := storagefs.NewFileSLOPluginRepo(log.Noop, mpl, test.fss()...) - require.NoError(err) - - plugin, err := repo.GetSLOPlugin(t.Context(), test.pluginID) - if test.expErr { - assert.Error(err) - } else if assert.NoError(err) { - assert.Equal(test.expPlugin, *plugin) - } - }) - } -} diff --git a/test/integration/k8scontroller/k8scontroller_test.go b/test/integration/k8scontroller/k8scontroller_test.go index 65658473bc5bdf28944742f5d0039862f942d686..0fe15e1d2366f14f93232b7f53c3f4bdb3633cee 100644 --- a/test/integration/k8scontroller/k8scontroller_test.go +++ b/test/integration/k8scontroller/k8scontroller_test.go @@ -224,7 +224,7 @@ func TestKubernetesControllerPromOperatorGenerate(t *testing.T) { args := []string{ "--metrics-listen-addr=:0", "--hot-reload-addr=:0", - "--sli-plugins-path=./", + "--plugins-path=./", fmt.Sprintf("--namespace=%s", ns), fmt.Sprintf("--default-slo-period=%s", test.sloPeriod), } diff --git a/test/integration/prometheus/helpers.go b/test/integration/prometheus/helpers.go index 8c0371a7fa0a14a7296caf2399669773f0708a29..d7d6937419fc1d62607031947a7e1a64b9358a03 100644 --- a/test/integration/prometheus/helpers.go +++ b/test/integration/prometheus/helpers.go @@ -48,8 +48,7 @@ func NewConfig(t *testing.T) Config { func RunSlothGenerate(ctx context.Context, config Config, cmdArgs string) (stdout, stderr []byte, err error) { env := []string{ - fmt.Sprintf("SLOTH_SLI_PLUGINS_PATH=%s", "./sli_plugins"), - fmt.Sprintf("SLOTH_SLO_PLUGINS_PATH=%s", "./slo_plugins"), + fmt.Sprintf("SLOTH_PLUGINS_PATH=%s", "./plugins"), } return testutils.RunSloth(ctx, env, config.Binary, fmt.Sprintf("generate %s", cmdArgs), true) @@ -57,8 +56,7 @@ func RunSlothGenerate(ctx context.Context, config Config, cmdArgs string) (stdou func RunSlothValidate(ctx context.Context, config Config, cmdArgs string) (stdout, stderr []byte, err error) { env := []string{ - fmt.Sprintf("SLOTH_SLI_PLUGINS_PATH=%s", "./sli_plugins"), - fmt.Sprintf("SLOTH_SLO_PLUGINS_PATH=%s", "./slo_plugins"), + fmt.Sprintf("SLOTH_PLUGINS_PATH=%s", "./plugins"), } return testutils.RunSloth(ctx, env, config.Binary, fmt.Sprintf("validate %s", cmdArgs), true) diff --git a/test/integration/prometheus/sli_plugins/plugin1/plugin.go b/test/integration/prometheus/plugins/sli/plugin1/plugin.go similarity index 100% rename from test/integration/prometheus/sli_plugins/plugin1/plugin.go rename to test/integration/prometheus/plugins/sli/plugin1/plugin.go diff --git a/test/integration/prometheus/slo_plugins/plugin1/plugin.go b/test/integration/prometheus/plugins/slo/plugin1/plugin.go similarity index 100% rename from test/integration/prometheus/slo_plugins/plugin1/plugin.go rename to test/integration/prometheus/plugins/slo/plugin1/plugin.go