m-lab · cristinaleonr · Sep 21, 2023 · Sep 20, 2023 · Sep 20, 2023 · Sep 20, 2023
diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@ FROM golang:1.20
 
 ADD . /go/src/github.com/m-lab/gcp-config
 WORKDIR /go/src/github.com/m-lab/gcp-config/
-RUN go install -v github.com/m-lab/gcp-config/cmd/stctl@v1.3.12
-RUN go install -v github.com/m-lab/gcp-config/cmd/cbif@v1.3.12
+RUN go install -v ./cmd/stctl
+RUN go install -v ./cmd/cbif
 ENV SINGLE_COMMAND true
 ENTRYPOINT ["/go/bin/cbif"]
diff --git a/cmd/stctl/main.go b/cmd/stctl/main.go
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -36,6 +36,7 @@ var (
 	destBucket          string
 	prefixes            flagx.StringArray
 	startTime           flagx.Time
+	interval            string
 	afterDate           flagx.DateTime
 	minAge              time.Duration
 	maxAge              time.Duration
@@ -48,6 +49,7 @@ func init() {
 	flag.StringVar(&destBucket, "gcs.target", "", "Destination bucket.")
 	flag.Var(&prefixes, "include", "Only transfer files with given prefix. Default all prefixes. Can be specified multiple times.")
 	flag.Var(&startTime, "time", "Start daily transfer at this time (HH:MM:SS)")
+	flag.StringVar(&interval, "interval", "", "Interval between the start of each scheduled transfer operation.")
 	flag.Var(&afterDate, "after", "Only list operations that ran after the given date. Default is all dates.")
 	flag.DurationVar(&minAge, "minFileAge", 0, "Minimum time since file modification")
 	flag.DurationVar(&maxAge, "maxFileAge", 0, "Maximum time since file modification")
@@ -107,6 +109,7 @@ func main() {
 		TargetBucket:        destBucket,
 		Prefixes:            prefixes,
 		StartTime:           startTime,
+		Interval:            interval,
 		AfterDate:           afterDate.Time,
 		MinFileAge:          minAge.Truncate(time.Second),
 		MaxFileAge:          maxAge.Truncate(time.Second),

diff --git a/daily-archive-transfers.yaml b/daily-archive-transfers.yaml
@@ -41,18 +41,14 @@ steps:
 
 # Nodes upload to the pusher-* bucket every 2 hours.
 # Configure hourly pusher to local archive transfer.
-# NOTE: the hourly setting has been set by editing the job created
-# in the web UI (because it is not supported by stctl), so this
-# declaration is not identical to what is in the corresponding GCP
-# config.
-# TODO(cristinaleon): Add hourly schedule functionality to stctl.
 - name: gcp-config-cbif
   env:
   - PROJECT_IN=mlab-sandbox,mlab-staging,mlab-oti
   args: [
     'stctl', '-gcs.source=pusher-$PROJECT_ID',
              '-gcs.target=archive-$PROJECT_ID',
              '-time=00:00:00',
+             '-interval=3600s',
              '-include=autoload',
              '-deleteAfterTransfer=true',
              'sync'
@@ -70,6 +66,7 @@ steps:
              '-gcs.target=archive-measurement-lab',
              '-include=revtr',
              '-time=02:30:00',
+             '-interval=3600s',
              '-maxFileAge=27h',
              'sync'
   ]
@@ -132,18 +129,14 @@ steps:
   ]
 
 # Hourly local archive to public archive transfer for autoloaded data.
-# NOTE: the hourly setting has been set by editing the job created
-# in the web UI (because it is not supported by stctl), so this
-# declaration is not identical to what is in the corresponding GCP
-# config.
-# TODO(cristinaleon): Add hourly schedule functionality to stctl.
 - name: gcp-config-cbif
   env:
   - PROJECT_IN=measurement-lab
   args: [
     'stctl', '-gcs.source=archive-mlab-oti',
              '-gcs.target=archive-measurement-lab',
              '-time=00:00:00',
+             '-interval=3600s',
              '-include=autoload',
              '-deleteAfterTransfer=true',
              'sync'

diff --git a/internal/stctl/command.go b/internal/stctl/command.go
@@ -48,6 +48,7 @@ type Command struct {
 	TargetBucket        string
 	Prefixes            []string
 	StartTime           flagx.Time
+	Interval            string
 	AfterDate           time.Time
 	MinFileAge          time.Duration
 	MaxFileAge          time.Duration

diff --git a/internal/stctl/create.go b/internal/stctl/create.go
@@ -35,6 +35,7 @@ func (c *Command) Create(ctx context.Context) (*storagetransfer.TransferJob, err
 				Minutes: int64(c.StartTime.Minute),
 				Seconds: int64(c.StartTime.Second),
 			},
+			RepeatInterval: c.Interval,
 		},
 		Status:       "ENABLED",
 		TransferSpec: &spec,

diff --git a/internal/stctl/sync.go b/internal/stctl/sync.go
@@ -106,6 +106,12 @@ func (c *Command) specMatches(job *storagetransfer.TransferJob) bool {
 		logx.Debug.Println("spec: times not equal", job.Schedule, c.StartTime)
 		return false
 	}
+
+	if c.Interval != "" && job.Schedule.RepeatInterval != c.Interval {
+		logx.Debug.Println("spec: interval not equal", job.Schedule.RepeatInterval, c.Interval)
+		return false
+	}
+
 	cond := job.TransferSpec.ObjectConditions
 	if cond == nil {
 		if len(c.Prefixes) > 0 || c.MaxFileAge > 0 || c.MinFil
6D40
eAge > 0 {

diff --git a/internal/stctl/sync_test.go b/internal/stctl/sync_test.go
@@ -245,6 +245,54 @@ func TestCommand_Sync(t *testing.T) {
 				Status: "ENABLED",
 			},
 		},
+		{
+			name: "success-update-interval",
+			c: &Command{
+				SourceBucket: "fake-source",
+				TargetBucket: "fake-target",
+				StartTime:    flagx.Time{Hour: 1, Minute: 2, Second: 3},
+				Interval:     "3600s",
+				Client: &fakeTJ{
+					// fake job listed to search for one that matches the current Command spec.
+					listJobResp: &storagetransfer.ListTransferJobsResponse{
+						TransferJobs: []*storagetransfer.TransferJob{
+							{
+								Name:        "transferOperations/update-interval",
+								Description: getDesc("fake-source", "fake-target", flagx.Time{Hour: 1, Minute: 2, Second: 3}),
+								Schedule: &storagetransfer.Schedule{
+									StartTimeOfDay: &storagetransfer.TimeOfDay{Hours: 1, Minutes: 2, Seconds: 3},
+								},
+								TransferSpec: &storagetransfer.TransferSpec{
+									GcsDataSource: &storagetransfer.GcsData{BucketName: "fake-source"},
+									GcsDataSink:   &storagetransfer.GcsData{BucketName: "fake-target"},
+								},
+							},
+						},
+					},
+					// a fake job that is disabled.
+					job: &storagetransfer.TransferJob{},
+				},
+			},
+			shouldFind: true,
+			expected: &storagetransfer.TransferJob{
+				Description: "STCTL: transfer fake-source -> fake-target at 01:02:03",
+				Name:        "THIS-IS-A-FAKE-ASSIGNED-JOB-NAME",
+				Schedule: &storagetransfer.Schedule{
+					StartTimeOfDay: &storagetransfer.TimeOfDay{Hours: 1, Minutes: 2, Seconds: 3},
+					ScheduleStartDate: &storagetransfer.Date{
+						Day:   int64(ts.Day()),
+						Month: int64(ts.Month()),
+						Year:  int64(ts.Year()),
+					},
+					RepeatInterval: "3600s",
+				},
+				TransferSpec: &storagetransfer.TransferSpec{
+					GcsDataSource: &storagetransfer.GcsData{BucketName: "fake-source"},
+					GcsDataSink:   &storagetransfer.GcsData{BucketName: "fake-target"},
+				},
+				Status: "ENABLED",
+			},
+		},
 		{
 			name: "success-job-not-found-then-created",
 			c: &Command{