--- a/server/channels/app/metrics_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package app - -import ( - "fmt" - "testing" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/v8/channels/app/platform" - "github.com/mattermost/mattermost/server/v8/enterprise/metrics" - "github.com/prometheus/client_golang/prometheus" - prometheusModels "github.com/prometheus/client_model/go" - "github.com/stretchr/testify/require" -) - -func configureMetrics(th *TestHelper) { - th.App.Srv().SetLicense(nil) // clear license - th.App.UpdateConfig(func(cfg *model.Config) { - *cfg.MetricsSettings.Enable = true - *cfg.MetricsSettings.ListenAddress = ":0" - }) - th.App.Srv().SetLicense(model.NewTestLicense("metrics")) -} - -func TestMobileMetrics(t *testing.T) { - mainHelper.Parallel(t) - th := SetupEnterprise(t, StartMetrics) - - configureMetrics(th) - mi := th.App.Metrics() - - miImpl, ok := mi.(*metrics.MetricsInterfaceImpl) - require.True(t, ok, fmt.Sprintf("App.Metrics is not *MetricsInterfaceImpl, but %T", mi)) - m := &prometheusModels.Metric{} - - for _, platform := range []string{"ios", "android"} { - ttcc := []struct { - name string - histogramVec *prometheus.HistogramVec - elapsed float64 - metricName model.MetricType - }{ - { - name: "load duration", - histogramVec: miImpl.MobileClientLoadDuration, - elapsed: 5001, - metricName: model.MobileClientLoadDuration, - }, - { - name: "channel switch duration", - histogramVec: miImpl.MobileClientChannelSwitchDuration, - elapsed: 501, - metricName: model.MobileClientChannelSwitchDuration, - }, - { - name: "team switch duration", - histogramVec: miImpl.MobileClientTeamSwitchDuration, - elapsed: 301, - metricName: model.MobileClientTeamSwitchDuration, - }, - } - - histograms := []*model.MetricSample{} - var start float64 = 125 - end := start + float64(len(ttcc)) - // Precheck - for _, tc := range ttcc { - actualMetric, err := tc.histogramVec.GetMetricWith(prometheus.Labels{"platform": platform}) - require.NoError(t, err) - require.NoError(t, actualMetric.(prometheus.Histogram).Write(m)) - require.Equal(t, uint64(0), m.Histogram.GetSampleCount()) - require.Equal(t, 0.0, m.Histogram.GetSampleSum()) - histograms = append(histograms, &model.MetricSample{ - Metric: tc.metricName, - Value: tc.elapsed, - }) - } - - appErr := th.App.RegisterPerformanceReport(th.Context, &model.PerformanceReport{ - Version: "0.1.0", - ClientID: "", - Labels: map[string]string{ - "platform": platform, - }, - Start: start, - End: end, - Counters: []*model.MetricSample{}, - Histograms: histograms, - }) - require.Nil(t, appErr) - - // After check - for _, tc := range ttcc { - actualMetric, err := tc.histogramVec.GetMetricWith(prometheus.Labels{"platform": platform}) - require.NoError(t, err) - require.NoError(t, actualMetric.(prometheus.Histogram).Write(m)) - require.Equal(t, uint64(1), m.Histogram.GetSampleCount()) - require.InDelta(t, tc.elapsed/1000, m.Histogram.GetSampleSum(), 0.001, "not equal value in %s", tc.name) - } - } -} - -func TestCountNotificationMetrics(t *testing.T) { - mainHelper.Parallel(t) - th := SetupEnterprise(t, StartMetrics) - - configureMetrics(th) - mi := th.App.Metrics() - - miImpl, ok := mi.(*metrics.MetricsInterfaceImpl) - require.True(t, ok, fmt.Sprintf("App.Metrics is not *MetricsInterfaceImpl, but %T", mi)) - - counterValue := func() float64 { - counter, err := miImpl.NotificationTotalCounters.GetMetricWith(prometheus.Labels{ - "type": string(model.NotificationTypePush), - "platform": "ios", - }) - require.NoError(t, err) - m := &prometheusModels.Metric{} - require.NoError(t, counter.Write(m)) - return m.Counter.GetValue() - } - - t.Run("counts when notification metrics are enabled", func(t *testing.T) { - th.App.UpdateConfig(func(cfg *model.Config) { - *cfg.MetricsSettings.EnableNotificationMetrics = true - }) - - before := counterValue() - th.App.CountNotification(model.NotificationTypePush, "ios") - require.Equal(t, before+1, counterValue()) - }) - - t.Run("does not count when notification metrics are disabled", func(t *testing.T) { - th.App.UpdateConfig(func(cfg *model.Config) { - *cfg.MetricsSettings.EnableNotificationMetrics = false - }) - - before := counterValue() - th.App.CountNotification(model.NotificationTypePush, "ios") - require.Equal(t, before, counterValue()) - }) -} - -func TestWebsocketNotificationCounter(t *testing.T) { - mainHelper.Parallel(t) - th := SetupEnterprise(t, StartMetrics) - - configureMetrics(th) - mi := th.App.Metrics() - - miImpl, ok := mi.(*metrics.MetricsInterfaceImpl) - require.True(t, ok, fmt.Sprintf("App.Metrics is not *MetricsInterfaceImpl, but %T", mi)) - - counterValue := func() float64 { - counter, err := miImpl.NotificationTotalCounters.GetMetricWith(prometheus.Labels{ - "type": string(model.NotificationTypeWebsocket), - "platform": model.NotificationNoPlatform, - }) - require.NoError(t, err) - m := &prometheusModels.Metric{} - require.NoError(t, counter.Write(m)) - return m.Counter.GetValue() - } - - hook := &postedAckBroadcastHook{} - userID := model.NewId() - webConn := &platform.WebConn{ - UserId: userID, - Platform: th.Server.Platform(), - PostedAck: true, - } - webConn.Active.Store(true) - webConn.SetSession(&model.Session{}) - - // Process an acked broadcast that reaches incrementWebsocketCounter. - ackPostedBroadcast := func() { - msg := platform.MakeHookedWebSocketEvent(model.NewWebSocketEvent(model.WebsocketEventPosted, "", "", "", nil, "")) - err := hook.Process(msg, webConn, map[string]any{ - "posted_user_id": model.NewId(), - "channel_type": model.ChannelTypeOpen, - "users": []string{userID}, - }) - require.NoError(t, err) - require.True(t, msg.Event().GetData()["should_ack"].(bool)) - } - - t.Run("counts when notification metrics are enabled", func(t *testing.T) { - th.App.UpdateConfig(func(cfg *model.Config) { - *cfg.MetricsSettings.EnableNotificationMetrics = true - }) - - before := counterValue() - ackPostedBroadcast() - require.Equal(t, before+1, counterValue()) - }) - - t.Run("does not count when notification metrics are disabled", func(t *testing.T) { - th.App.UpdateConfig(func(cfg *model.Config) { - *cfg.MetricsSettings.EnableNotificationMetrics = false - }) - - before := counterValue() - ackPostedBroadcast() - require.Equal(t, before, counterValue()) - }) -} --- a/server/cmd/mmctl/commands/compliance_export.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package commands - -import ( - "context" - "errors" - "fmt" - "os" - "path" - "strconv" - "time" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/v8/cmd/mmctl/client" - "github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer" - "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared" - "github.com/spf13/cobra" -) - -var ComplianceExportCmd = &cobra.Command{ - Use: "compliance-export", - Short: "Management of compliance exports", -} - -var ComplianceExportListCmd = &cobra.Command{ - Use: "list", - Aliases: []string{"ls"}, - Short: "List compliance export jobs, sorted by creation date descending (newest first)", - Args: cobra.NoArgs, - RunE: withClient(complianceExportListCmdF), -} - -var ComplianceExportShowCmd = &cobra.Command{ - Use: "show [complianceExportJobID]", - Example: "compliance-export show o98rj3ur83dp5dppfyk5yk6osy", - Short: "Show compliance export job", - Args: cobra.ExactArgs(1), - RunE: withClient(complianceExportShowCmdF), -} - -var ComplianceExportCancelCmd = &cobra.Command{ - Use: "cancel [complianceExportJobID]", - Example: "compliance-export cancel o98rj3ur83dp5dppfyk5yk6osy", - Short: "Cancel compliance export job", - Args: cobra.ExactArgs(1), - RunE: withClient(complianceExportCancelCmdF), -} - -var ComplianceExportDownloadCmd = &cobra.Command{ - Use: "download [complianceExportJobID] [output filepath (optional)]", - Example: "compliance-export download o98rj3ur83dp5dppfyk5yk6osy", - Short: "Download compliance export file", - Args: cobra.MinimumNArgs(1), - RunE: withClient(complianceExportDownloadCmdF), -} - -var ComplianceExportCreateCmd = &cobra.Command{ - Use: "create [complianceExportType] --date \"2025-03-27 -0400\"", - Example: "compliance-export create csv --date \"2025-03-27 -0400\"", - Long: "Create a compliance export job, of type 'csv' or 'actiance' or 'globalrelay'. If --date is set, the job will run for one day, from 12am to 12am (minus one millisecond) inclusively, in the format with timezone offset: `\"YYYY-MM-DD -0000\"`. E.g., \"2024-10-21 -0400\" for Oct 21, 2024 EDT timezone. \"2023-11-01 +0000\" for Nov 01, 2024 UTC. If set, the 'start' and 'end' flags will be ignored.\n\n" + - "Important: Running a compliance export job from mmctl will NOT affect the next scheduled job's batch_start_time. This means that if you run a compliance export job from mmctl, the next scheduled job will run from the batch_end_time of the previous scheduled job, as usual.", - Short: "Create a compliance export job, of type 'csv' or 'actiance' or 'globalrelay'", - Args: cobra.MinimumNArgs(1), - RunE: withClient(complianceExportCreateCmdF), -} - -func init() { - ComplianceExportListCmd.Flags().Int("page", 0, "Page number to fetch for the list of compliance export jobs") - ComplianceExportListCmd.Flags().Int("per-page", DefaultPageSize, "Number of compliance export jobs to be fetched") - ComplianceExportListCmd.Flags().Bool("all", false, "Fetch all compliance export jobs. --page flag will be ignored if provided") - - ComplianceExportDownloadCmd.Flags().Int("num-retries", 5, "Number of retries if the download fails") - - ComplianceExportCreateCmd.Flags().String( - "date", - "", - "Run the export for one day, from 12am to 12am (minus one millisecond) inclusively, in the format with timezone offset: `\"YYYY-MM-DD -0000\"`. E.g., `\"2024-10-21 -0400\"` for Oct 21, 2024 EDT timezone. `\"2023-11-01 +0000\"` for Nov 01, 2024 UTC. If set, the 'start' and 'end' flags will be ignored.", - ) - ComplianceExportCreateCmd.Flags().Int( - "start", - 0, - "The start timestamp in unix milliseconds. Posts with updateAt >= start will be exported. If set, 'end' must be set as well. eg, `1743048000000` for 2025-03-27 EDT.", - ) - ComplianceExportCreateCmd.Flags().Int( - "end", - 0, - "The end timestamp in unix milliseconds. Posts with updateAt <= end will be exported. If set, 'start' must be set as well. eg, `1743134400000` for 2025-03-28 EDT.", - ) - - ComplianceExportCmd.AddCommand( - ComplianceExportListCmd, - ComplianceExportShowCmd, - ComplianceExportCancelCmd, - ComplianceExportDownloadCmd, - ComplianceExportCreateCmd, - ) - RootCmd.AddCommand(ComplianceExportCmd) -} - -func complianceExportListCmdF(c client.Client, command *cobra.Command, args []string) error { - return jobListCmdF(c, command, "message_export", "") -} - -func complianceExportShowCmdF(c client.Client, command *cobra.Command, args []string) error { - job, _, err := c.GetJob(context.TODO(), args[0]) - if err != nil { - return fmt.Errorf("failed to get compliance export job: %w", err) - } - - printJob(job) - - return nil -} - -func complianceExportCancelCmdF(c client.Client, command *cobra.Command, args []string) error { - if _, err := c.CancelJob(context.TODO(), args[0]); err != nil { - return fmt.Errorf("failed to cancel compliance export job: %w", err) - } - - return nil -} - -func complianceExportDownloadCmdF(c client.Client, command *cobra.Command, args []string) error { - jobID := args[0] - var path string - if len(args) > 1 { - path = args[1] - } else { - path = jobID + ".zip" - } - - retries, _ := command.Flags().GetInt("num-retries") - - downloadFn := func(outFile *os.File) (string, error) { - return c.DownloadComplianceExport(context.TODO(), jobID, outFile) - } - - suggestedFilename, err := downloadFile(path, downloadFn, retries, "compliance export") - if err != nil { - return err - } - - // If we didn't provide a path and got a suggested filename, rename the file - if len(args) == 1 && suggestedFilename != "" && suggestedFilename != path { - // If the suggested name already exists, don't overwrite - if _, err := os.Stat(suggestedFilename); err == nil { - printer.PrintWarning(fmt.Sprintf("File with the server's suggested name %q already exists, keeping %q", suggestedFilename, path)) - } else { - if err := os.Rename(path, suggestedFilename); err != nil { - printer.PrintWarning(fmt.Sprintf("Could not rename file to the server's suggested name %q: %v", suggestedFilename, err)) - } else { - path = suggestedFilename - } - } - } - - printer.Print(fmt.Sprintf("Compliance export file downloaded to %q", path)) - return nil -} - -func complianceExportCreateCmdF(c client.Client, command *cobra.Command, args []string) error { - exportType := args[0] - if exportType != model.ComplianceExportTypeActiance && - exportType != model.ComplianceExportTypeCsv && - exportType != model.ComplianceExportTypeGlobalrelay { - return fmt.Errorf("invalid export type: %s, must be one of: csv, actiance, globalrelay", exportType) - } - - dateStr, err := command.Flags().GetString("date") - if err != nil { - return err - } - start, err := command.Flags().GetInt("start") - if err != nil { - return err - } - end, err := command.Flags().GetInt("end") - if err != nil { - return err - } - startTimestamp, endTimestamp, err := getStartAndEnd(dateStr, start, end) - if err != nil { - return err - } - startTime := strconv.FormatInt(startTimestamp, 10) - endTime := strconv.FormatInt(endTimestamp, 10) - exportDir := path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%s-%s", time.Now().Format(model.ComplianceExportDirectoryFormat), startTime, endTime)) - - // If start and end are 0, we need to not set those keys in the job data. - // This will make the job like a manual job (it will pick up where the previous job left off). - data := model.StringMap{ - shared.JobDataInitiatedBy: "mmctl", - shared.JobDataExportType: exportType, - shared.JobDataBatchStartId: "", - shared.JobDataJobStartId: "", - } - if startTimestamp != 0 && endTimestamp != 0 { - data[shared.JobDataBatchStartTime] = startTime - data[shared.JobDataJobStartTime] = startTime - data[shared.JobDataJobEndTime] = endTime - data[shared.JobDataExportDir] = exportDir - } - - job := &model.Job{ - Type: model.JobTypeMessageExport, - Data: data, - } - - if job, _, err = c.CreateJob(context.TODO(), job); err != nil { - return fmt.Errorf("failed to create compliance export job: %w", err) - } - - printer.Print(fmt.Sprintf("Compliance export job created with ID: %s", job.Id)) - - return nil -} - -// getStartAndEnd returns the start and end timestamps in unix milliseconds -func getStartAndEnd(dateStr string, start int, end int) (int64, int64, error) { - if dateStr == "" && start == 0 && end == 0 { - // return 0 so that the job will be like a manual job - return 0, 0, nil - } - - if dateStr != "" && (start > 0 || end > 0) { - return 0, 0, errors.New("if date is used, start and end must not be set") - } - - if dateStr != "" { - t, err := time.Parse("2006-01-02 -0700", dateStr) - if err != nil { - return 0, 0, fmt.Errorf("could not parse date string: %s, use the format with time zone offset: YYYY-MM-DD -0700, eg for EDT: `2024-12-24 -0400`, error details: %w", dateStr, err) - } - endTimestamp := t.AddDate(0, 0, 1).UnixMilli() - 1 - return t.UnixMilli(), endTimestamp, nil - } - if start <= 0 || end <= 0 || start >= end { - return 0, 0, fmt.Errorf("if date is not used, start: %d and end: %d must both be > 0, and start must be < end", start, end) - } - return int64(start), int64(end), nil -} --- a/server/cmd/mmctl/commands/compliance_export_e2e_test.go +++ /dev/null @@ -1,744 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package commands - -import ( - "archive/zip" - "context" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - - "github.com/mattermost/mattermost/server/public/model" - "github.com/mattermost/mattermost/server/v8" - st "github.com/mattermost/mattermost/server/v8/channels/store/storetest" - "github.com/mattermost/mattermost/server/v8/cmd/mmctl/client" - "github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer" - "github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared" - "github.com/spf13/cobra" -) - -func (s *MmctlE2ETestSuite) TestComplianceExportListCmdE2E() { - s.SetupMessageExportTestHelper() - - s.Run("no permissions", func() { - printer.Clean() - - cmd := makeCmd() - err := complianceExportListCmdF(s.th.Client, cmd, nil) - s.Require().EqualError(err, "failed to get jobs: You do not have the appropriate permissions.") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - jobType := model.JobTypeMessageExport - - s.RunForSystemAdminAndLocal("List with no compliance export jobs", func(c client.Client) { - // Ensure no jobs exist - jobs, _, err := s.th.SystemAdminClient.GetJobsByType(context.Background(), jobType, 0, 1000) - s.Require().NoError(err) - - for _, job := range jobs { - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - } - - cmd := makeCmd() - // Test default pagination - printer.Clean() - err = complianceExportListCmdF(c, cmd, nil) - s.Require().NoError(err) - s.Require().Len(printer.GetLines(), 1) - s.Require().Equal("No jobs found", printer.GetLines()[0]) - - // Test with 1 per page - printer.Clean() - cmd = makeCmd() - _ = cmd.Flags().Set("page", "0") - _ = cmd.Flags().Set("per-page", "1") - err = complianceExportListCmdF(c, cmd, nil) - s.Require().NoError(err) - s.Require().Len(printer.GetLines(), 1) - s.Require().Equal("No jobs found", printer.GetLines()[0]) - - // Test with all items - printer.Clean() - cmd = makeCmd() - _ = cmd.Flags().Set("all", "true") - err = complianceExportListCmdF(c, cmd, nil) - s.Require().NoError(err) - s.Require().Len(printer.GetLines(), 1) - s.Require().Equal("No jobs found", printer.GetLines()[0]) - }) - - s.RunForSystemAdminAndLocal("List compliance export jobs", func(c client.Client) { - now := model.GetMillis() - // Create 2 jobs - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusSuccess, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - }) - s.Require().NoError(err) - - job2, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 100, - Status: model.JobStatusSuccess, - Type: model.JobTypeMessageExport, - StartAt: now - 100, - LastActivityAt: now - 100, - }) - s.Require().NoError(err) - defer func() { - // Ensure jobs are deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - result, err = s.th.App.Srv().Store().Job().Delete(job2.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - // Test default pagination - printer.Clean() - cmd := makeCmd() - err = complianceExportListCmdF(c, cmd, nil) - s.Require().NoError(err) - s.Require().Len(printer.GetLines(), 2) - s.Require().Equal(job2.Id, printer.GetLines()[0].(*model.Job).Id) - s.Require().Equal(job.Id, printer.GetLines()[1].(*model.Job).Id) - - // Test with 1 per page - printer.Clean() - cmd = makeCmd() - _ = cmd.Flags().Set("page", "0") - _ = cmd.Flags().Set("per-page", "1") - err = complianceExportListCmdF(c, cmd, nil) - s.Require().NoError(err) - s.Require().Len(printer.GetLines(), 1) - s.Require().Equal(job2.Id, printer.GetLines()[0].(*model.Job).Id) - - // Test with all items - printer.Clean() - cmd = makeCmd() - _ = cmd.Flags().Set("all", "true") - err = complianceExportListCmdF(c, cmd, nil) - s.Require().NoError(err) - s.Require().Len(printer.GetLines(), 2) - s.Require().Equal(job2.Id, printer.GetLines()[0].(*model.Job).Id) - s.Require().Equal(job.Id, printer.GetLines()[1].(*model.Job).Id) - }) -} - -func (s *MmctlE2ETestSuite) TestComplianceExportShowCmdE2E() { - s.SetupMessageExportTestHelper() - - now := model.GetMillis() - - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusSuccess, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - }) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - s.Run("no permissions", func() { - printer.Clean() - - cmd := makeCmd() - err := complianceExportShowCmdF(s.th.Client, cmd, []string{job.Id}) - s.Require().EqualError(err, "failed to get compliance export job: You do not have the appropriate permissions.") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - s.RunForSystemAdminAndLocal("Show non-existent job", func(c client.Client) { - printer.Clean() - - cmd := makeCmd() - err := complianceExportShowCmdF(c, cmd, []string{"non-existent-job-id"}) - s.Require().EqualError(err, "failed to get compliance export job: Sorry, we could not find the page., There doesn't appear to be an api call for the url='/api/v4/jobs/non-existent-job-id'. Typo? are you missing a team_id or user_id as part of the url?") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - s.RunForSystemAdminAndLocal("Show existing job", func(c client.Client) { - now := model.GetMillis() - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusSuccess, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - }) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - printer.Clean() - cmd := makeCmd() - err = complianceExportShowCmdF(c, cmd, []string{job.Id}) - s.Require().NoError(err) - s.Require().Len(printer.GetLines(), 1) - s.Require().Empty(printer.GetErrorLines()) - s.Require().Equal(job.Id, printer.GetLines()[0].(*model.Job).Id) - }) -} - -func (s *MmctlE2ETestSuite) TestComplianceExportCancelCmdE2E() { - s.SetupMessageExportTestHelper() - - s.Run("no permissions", func() { - printer.Clean() - - now := model.GetMillis() - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusInProgress, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - }) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - cmd := makeCmd() - err = complianceExportCancelCmdF(s.th.Client, cmd, []string{job.Id}) - s.Require().EqualError(err, "failed to cancel compliance export job: You do not have the appropriate permissions.") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - s.RunForSystemAdminAndLocal("Cancel non-existent job", func(c client.Client) { - printer.Clean() - - cmd := makeCmd() - err := complianceExportCancelCmdF(c, cmd, []string{"non-existent-job-id"}) - s.Require().EqualError(err, "failed to cancel compliance export job: Sorry, we could not find the page., There doesn't appear to be an api call for the url='/api/v4/jobs/non-existent-job-id/cancel'. Typo? are you missing a team_id or user_id as part of the url?") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - s.RunForSystemAdminAndLocal("Cancel existing job", func(c client.Client) { - now := model.GetMillis() - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusInProgress, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - }) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - printer.Clean() - cmd := makeCmd() - err = complianceExportCancelCmdF(c, cmd, []string{job.Id}) - s.Require().NoError(err) - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - - // Verify job was cancelled - job, _, err = s.th.SystemAdminClient.GetJob(context.Background(), job.Id) - s.Require().NoError(err) - s.Require().Equal(model.JobStatusCanceled, job.Status) - }) - - s.RunForSystemAdminAndLocal("Error cancelling job in non-cancellable state", func(c client.Client) { - now := model.GetMillis() - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusInProgress, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - }) - s.Require().NoError(err) - _, err = s.th.SystemAdminClient.UpdateJobStatus(context.Background(), job.Id, model.JobStatusCanceled, true) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - printer.Clean() - cmd := makeCmd() - err = complianceExportCancelCmdF(c, cmd, []string{job.Id}) - s.Require().EqualError(err, "failed to cancel compliance export job: Could not request cancellation for job that is not in a cancelable state.") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) -} - -func (s *MmctlE2ETestSuite) TestComplianceExportDownloadCmdE2E() { - s.SetupMessageExportTestHelper() - - s.Run("no permissions", func() { - printer.Clean() - - now := model.GetMillis() - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusSuccess, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - }) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - cmd := makeCmd() - cmd.Flags().Int("num-retries", 0, "") - err = complianceExportDownloadCmdF(s.th.Client, cmd, []string{job.Id}) - s.Require().EqualError(err, "failed to download compliance export after 0 retries: You do not have the appropriate permissions.") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - s.RunForSystemAdminAndLocal("Download non-existent job", func(c client.Client) { - printer.Clean() - - cmd := makeCmd() - cmd.Flags().Int("num-retries", 0, "") - err := complianceExportDownloadCmdF(c, cmd, []string{"non-existent-job-id"}) - s.Require().EqualError(err, "failed to download compliance export after 0 retries: Sorry, we could not find the page., There doesn't appear to be an api call for the url='/api/v4/jobs/non-existent-job-id/download'. Typo? are you missing a team_id or user_id as part of the url?") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - s.RunForSystemAdminAndLocal("existing, non empty compliance export", func(c client.Client) { - printer.Clean() - - importFilePath := filepath.Join(server.GetPackagePath(), "test.zip") - f, err := os.Create(importFilePath) - s.Require().NoError(err) - _, err = f.WriteString("test data") - s.Require().NoError(err) - err = f.Close() - s.Require().NoError(err) - - defer func() { - err = os.Remove(importFilePath) - s.Require().NoError(err) - }() - - cmd := &cobra.Command{} - cmd.Flags().Int("num-retries", 0, "") - - err = complianceExportDownloadCmdF(c, cmd, []string{"jobId", importFilePath}) - s.Require().EqualError(err, "compliance export file already exists") - s.Require().Empty(printer.GetLines()) - s.Require().Empty(printer.GetErrorLines()) - }) - - s.RunForSystemAdminAndLocal("download with explicit path", func(c client.Client) { - printer.Clean() - - downloadPath := "explicit_path.zip" - defer os.Remove(downloadPath) - - serverDataDir, err := filepath.Abs(*s.th.App.Config().FileSettings.Directory) - s.Require().NoError(err) - - exportDir := "job_test_export" - // Create a compliance export with two zip files - exportFilePath := filepath.Join(serverDataDir, exportDir) - err = os.Mkdir(exportFilePath, 0755) - s.Require().NoError(err) - defer func() { - err = os.RemoveAll(exportFilePath) - s.Require().NoError(err) - }() - - // Create first zip file - zipPath1 := exportFilePath + "/export1.zip" - f1, err := os.Create(zipPath1) - s.Require().NoError(err) - _, err = f1.WriteString("test data 1") - s.Require().NoError(err) - err = f1.Close() - s.Require().NoError(err) - - // Create second zip file - zipPath2 := exportFilePath + "/export2.zip" - f2, err := os.Create(zipPath2) - s.Require().NoError(err) - _, err = f2.WriteString("test data 2") - s.Require().NoError(err) - err = f2.Close() - s.Require().NoError(err) - - s.T().Cleanup(func() { - err = os.RemoveAll(exportFilePath) - s.Require().NoError(err) - }) - - now := model.GetMillis() - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusSuccess, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - Data: model.StringMap{"export_dir": exportDir, "is_downloadable": "true"}, - }) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - cmd := makeCmd() - cmd.Flags().Int("num-retries", 0, "") - - err = complianceExportDownloadCmdF(c, cmd, []string{job.Id, downloadPath}) - s.Require().NoError(err) - s.Require().Contains(printer.GetLines()[0], fmt.Sprintf("Compliance export file downloaded to %q", downloadPath)) - s.Require().Empty(printer.GetErrorLines()) - - // Verify the file was downloaded - _, err = os.Stat(downloadPath) - s.Require().NoError(err) - defer os.Remove(downloadPath) - - // Verify the file is a zip with a directory with two zip files - zipReader, err := zip.OpenReader(downloadPath) - s.Require().NoError(err) - defer zipReader.Close() - - // Check that we have the expected files in the zip - foundExport1 := false - foundExport2 := false - for _, file := range zipReader.File { - if file.Name == "export1.zip" { - foundExport1 = true - // Verify contents of export1.zip - rc, err := file.Open() - s.Require().NoError(err) - content, err := io.ReadAll(rc) - s.Require().NoError(err) - s.Require().Equal("test data 1", string(content)) - rc.Close() - } else if file.Name == "export2.zip" { - foundExport2 = true - // Verify contents of export2.zip - rc, err := file.Open() - s.Require().NoError(err) - content, err := io.ReadAll(rc) - s.Require().NoError(err) - s.Require().Equal("test data 2", string(content)) - rc.Close() - } else { - s.Failf("unexpected file found in downloaded zip", file.Name) - } - } - s.Require().True(foundExport1, "export1.zip not found in downloaded file") - s.Require().True(foundExport2, "export2.zip not found in downloaded file") - }) - - s.RunForSystemAdminAndLocal("download with explicit path", func(c client.Client) { - printer.Clean() - - serverDataDir, err := filepath.Abs(*s.th.App.Config().FileSettings.Directory) - s.Require().NoError(err) - - exportDir := "job_test_export" - expectedDownloadPath := exportDir + ".zip" - // Create a compliance export with two zip files - exportFilePath := filepath.Join(serverDataDir, exportDir) - err = os.Mkdir(exportFilePath, 0755) - s.Require().NoError(err) - defer func() { - err = os.RemoveAll(exportFilePath) - s.Require().NoError(err) - - // also remove the downloaded file - err = os.Remove(expectedDownloadPath) - s.Require().NoError(err) - }() - - // Create first zip file - zipPath1 := exportFilePath + "/export1.zip" - f1, err := os.Create(zipPath1) - s.Require().NoError(err) - _, err = f1.WriteString("test data 1") - s.Require().NoError(err) - err = f1.Close() - s.Require().NoError(err) - - // Create second zip file - zipPath2 := exportFilePath + "/export2.zip" - f2, err := os.Create(zipPath2) - s.Require().NoError(err) - _, err = f2.WriteString("test data 2") - s.Require().NoError(err) - err = f2.Close() - s.Require().NoError(err) - - defer func() { - err = os.RemoveAll(exportFilePath) - s.Require().NoError(err) - }() - - now := model.GetMillis() - // Create a job - job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ - Id: st.NewTestID(), - CreateAt: now - 1000, - Status: model.JobStatusSuccess, - Type: model.JobTypeMessageExport, - StartAt: now - 1000, - LastActivityAt: now - 1000, - Data: model.StringMap{"export_dir": exportDir, "is_downloadable": "true"}, - }) - s.Require().NoError(err) - defer func() { - // Ensure job is deleted from the database - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - }() - - cmd := makeCmd() - cmd.Flags().Int("num-retries", 0, "") - - err = complianceExportDownloadCmdF(c, cmd, []string{job.Id}) - s.Require().NoError(err) - s.Require().Contains(printer.GetLines()[0], fmt.Sprintf("Compliance export file downloaded to %q", expectedDownloadPath)) - s.Require().Empty(printer.GetErrorLines()) - - // Verify the file was downloaded - _, err = os.Stat(expectedDownloadPath) - s.Require().NoError(err) - - // Verify the file is a zip with a directory with two zip files - zipReader, err := zip.OpenReader(expectedDownloadPath) - s.Require().NoError(err) - defer zipReader.Close() - - // Check that we have the expected files in the zip - foundExport1 := false - foundExport2 := false - for _, file := range zipReader.File { - if file.Name == "export1.zip" { - foundExport1 = true - // Verify contents of export1.zip - rc, err := file.Open() - s.Require().NoError(err) - content, err := io.ReadAll(rc) - s.Require().NoError(err) - s.Require().Equal("test data 1", string(content)) - rc.Close() - } else if file.Name == "export2.zip" { - foundExport2 = true - // Verify contents of export2.zip - rc, err := file.Open() - s.Require().NoError(err) - content, err := io.ReadAll(rc) - s.Require().NoError(err) - s.Require().Equal("test data 2", string(content)) - rc.Close() - } else { - s.Failf("unexpected file found in downloaded zip", file.Name) - } - } - s.Require().True(foundExport1, "export1.zip not found in downloaded file") - s.Require().True(foundExport2, "export2.zip not found in downloaded file") - }) -} - -func (s *MmctlE2ETestSuite) TestComplianceExportMmctlJobStartTimeE2E() { - s.SetupMessageExportTestHelper() - - s.RunForSystemAdminAndLocal("mmctl job uses batch_start_time from previous regular job", func(c client.Client) { - // Ensure no jobs exist before we start - jobs, _, err := s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 1000) - s.Require().NoError(err) - for _, job := range jobs { - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - } - - now := model.GetMillis() - - // Create a regular (non-mmctl) export job - regularStartTime := now - 10000 - regularEndTime := now - 5000 - regularJob := s.runJobForTest(map[string]string{ - shared.JobDataBatchStartTime: strconv.FormatInt(regularStartTime, 10), - shared.JobDataJobEndTime: strconv.FormatInt(regularEndTime, 10), - }) - - s.Require().Equal(model.JobStatusSuccess, regularJob.Status, "Regular job should complete successfully") - s.Require().NotEmpty(regularJob.Data[shared.JobDataBatchStartTime], "Regular job should have a batch start time") - regularJobBatchStartTime := regularJob.Data[shared.JobDataBatchStartTime] - - // Run an mmctl-initiated export job - cmd := &cobra.Command{} - cmd.Flags().String("date", "", "") - cmd.Flags().Int("start", 0, "") - cmd.Flags().Int("end", 0, "") - err = complianceExportCreateCmdF(c, cmd, []string{model.ComplianceExportTypeActiance}) - s.Require().NoError(err, "Should create mmctl job successfully") - - // Find the mmctl job - jobs, _, err = s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 10) - s.Require().NoError(err) - s.Require().True(len(jobs) > 1, "Should have at least 2 jobs") - - // The most recent job should be the mmctl job - mmctlJob := jobs[0] - s.Require().Equal("mmctl", mmctlJob.Data[shared.JobDataInitiatedBy]) - - // Wait for the mmctl job to complete - s.checkJobForStatus(mmctlJob.Id, model.JobStatusSuccess) - mmctlJob = s.getMostRecentJobWithId(mmctlJob.Id) - - // The job_start_time should match the batch_start_time from the previous regular job - s.Require().Equal(regularJobBatchStartTime, mmctlJob.Data[shared.JobDataJobStartTime], - "mmctl job should use batch_start_time from previous regular job as its job_start_time") - - // Clean up jobs - for _, job := range jobs { - result, err := s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - } - }) - - s.RunForSystemAdminAndLocal("mmctl job ignores previous mmctl jobs and uses regular job", func(c client.Client) { - // Ensure no jobs exist before we start - jobs, _, err := s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 1000) - s.Require().NoError(err) - for _, job := range jobs { - var result string - result, err = s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - } - - now := model.GetMillis() - - // Create a regular (non-mmctl) export job - regularStartTime := now - 10000 - regularEndTime := now - 5000 - regularJob := s.runJobForTest(map[string]string{ - shared.JobDataBatchStartTime: strconv.FormatInt(regularStartTime, 10), - shared.JobDataJobEndTime: strconv.FormatInt(regularEndTime, 10), - }) - - s.Require().Equal(model.JobStatusSuccess, regularJob.Status, "Regular job should complete successfully") - s.Require().NotEmpty(regularJob.Data[shared.JobDataBatchStartTime], "Regular job should have a batch start time") - regularJobBatchStartTime := regularJob.Data[shared.JobDataBatchStartTime] - - // Run an mmctl-initiated export job with an explicit start time (different from the regular job) - cmd := &cobra.Command{} - cmd.Flags().String("date", "", "") - cmd.Flags().Int("start", int(now-2000), "") - cmd.Flags().Int("end", int(now-1000), "") - err = complianceExportCreateCmdF(c, cmd, []string{model.ComplianceExportTypeActiance}) - s.Require().NoError(err, "Should create first mmctl job successfully") - - // Find the mmctl job - jobs, _, err = s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 10) - s.Require().NoError(err) - s.Require().True(len(jobs) > 1, "Should have at least 2 jobs") - - // The most recent job should be the mmctl job - mmctlJob1 := jobs[0] - s.Require().Equal("mmctl", mmctlJob1.Data[shared.JobDataInitiatedBy]) - - // Wait for the mmctl job to complete - s.checkJobForStatus(mmctlJob1.Id, model.JobStatusSuccess) - mmctlJob1 = s.getMostRecentJobWithId(mmctlJob1.Id) - - // Verify this job has a different batch_start_time than the regular job - s.Require().NotEqual(regularJobBatchStartTime, mmctlJob1.Data[shared.JobDataBatchStartTime], - "First mmctl job should have a different batch_start_time than regular job") - - // Run a second mmctl-initiated export job WITHOUT a specified start time - cmd = &cobra.Command{} - cmd.Flags().String("date", "", "") - cmd.Flags().Int("start", 0, "") - cmd.Flags().Int("end", 0, "") - err = complianceExportCreateCmdF(c, cmd, []string{model.ComplianceExportTypeActiance}) - s.Require().NoError(err, "Should create second mmctl job successfully") - - // Find the second mmctl job - jobs, _, err = s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 10) - s.Require().NoError(err) - s.Require().True(len(jobs) > 2, "Should have at least 3 jobs") - - // The most recent job should be the second mmctl job - mmctlJob2 := jobs[0] - s.Require().Equal("mmctl", mmctlJob2.Data[shared.JobDataInitiatedBy]) - - // Wait for the second mmctl job to complete - s.checkJobForStatus(mmctlJob2.Id, model.JobStatusSuccess) - mmctlJob2 = s.getMostRecentJobWithId(mmctlJob2.Id) - - // The job_start_time of the second mmctl job should match the batch_start_time from the regular job, - // not from the mmctl job that ran in between - s.Require().Equal(regularJobBatchStartTime, mmctlJob2.Data[shared.JobDataJobStartTime], - "Second mmctl job should use batch_start_time from previous regular job as its job_start_time, not from previous mmctl job") - s.Require().NotEqual(mmctlJob1.Data[shared.JobDataBatchStartTime], mmctlJob2.Data[shared.JobDataJobStartTime], - "Second mmctl job should not use batch_start_time from previous mmctl job as its job_start_time") - - // Clean up jobs - for _, job := range jobs { - result, err := s.th.App.Srv().Store().Job().Delete(job.Id) - s.Require().NoError(err, "Failed to delete job (result: %v)", result) - } - }) -} --- /dev/null +++ b/server/enterprise/placeholder.go @@ -0,0 +1,3 @@ +// Regenerated by the ReadyStack kit. Keeps `package enterprise` importable — cmd/mattermost +// blank-imports it unconditionally — while carrying none of the Source Available code. +package enterprise --- a/server/go.mod +++ b/server/go.mod @@ -233,3 +233,5 @@ // See MM-66167, MM-68222 for more details. replace github.com/vmihailenco/msgpack/v5 => github.com/mattermost/msgpack/v5 v5.0.0-20260813205620-e158e8d3647e + +replace github.com/mattermost/mattermost/server/public => ./public