8000 copy/transform: refactor control msg-s; `num-workers`, `continue-on-err` · NVIDIA/aistore@8434a74 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content
8000

Commit 8434a74

Browse files
alex-aizmanalex-aizman
committed
copy/transform: refactor control msg-s; num-workers, continue-on-err
* move the latter from tco to (nested) tcb control * TODO: tcb to support `num-workers` Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent ad14a2a commit 8434a74

File tree

4 files changed

+35
-15
lines changed

4 files changed

+35
-15
lines changed

‎api/apc/multiobj.go

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Package apc: API control messages and constants
22
/*
3-
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
44
*/
55
package apc
66

@@ -106,13 +106,3 @@ type ArchiveMsg struct {
106106
AppendIfExists bool `json:"aate"` // adding a list or a range of objects to an existing archive
107107
ContinueOnError bool `json:"coer"` // on err, keep running arc xaction in a any given multi-object transaction
108108
}
109-
110-
// multi-object copy & transform
111-
// [NOTE] see cmn/api for cmn.TCOMsg (that also contains ToBck); see also TCBMsg
112-
type TCOMsg struct {
113-
TxnUUID string // (plstcx client, internal use)
114-
TCBMsg
115-
ListRange
116-
NumWorkers int `json:"num-workers"` // user-defined num concurrent workers; 0 - number of mountpaths (default); (-1) none
117-
ContinueOnError bool `json:"coer"`
118-
}

‎api/apc/tcb.go renamed to ‎api/apc/tcbtco.go

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Package apc: API control messages and constants
22
/*
3-
* Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
44
*/
55
package apc
66

@@ -11,7 +11,12 @@ import (
1111
"github.com/NVIDIA/aistore/cmn/cos"
1212
)
1313

14-
// copy & (offline) transform bucket to bucket
14+
// offline copy/transform: bucket-to-bucket and multi-object
15+
16+
// NOTE: see cmn/api for cmn.TCOMsg that also contains source and destination buckets
17+
18+
// TODO: `ContinueOnError` not implemented for the most part
19+
1520
type (
1621
CopyBckMsg struct {
1722
Prepend string `json:"prepend"` // destination naming, as in: dest-obj-name = Prepend + source-obj-name
@@ -25,16 +30,33 @@ type (
2530
Name string `json:"id,omitempty"`
2631
Timeout cos.Duration `json:"request_timeout,omitempty"`
2732
}
33+
34+
// bucket to bucket
2835
TCBMsg struct {
29-
// NOTE: objname extension ----------------------------------------------------------------------
36+
// Objname Extension ----------------------------------------------------------------------
3037
// - resulting object names will have this extension, if specified.
3138
// - if source bucket has two (or more) objects with the same base name but different extension,
3239
// specifying this field might cause unintended override.
33-
// - this field might not be any longer required - TODO review
40+
// - this field might not be any longer required
3441
Ext cos.StrKVs `json:"ext"`
3542

3643
Transform
3744
CopyBckMsg
45+
46+
// user-defined number of concurrent workers:
47+
// - 0 (default) - number of mountpaths
48+
// - (-1) - single thread, serial execution
49+
NumWorkers int `json:"num-workers"`
50+
51+
ContinueOnError bool `json:"coer"`
52+
}
53+
54+
// multi-object
55+
// (cmn.TCOMsg = TCOMsg + source and destination buckets)
56+
TCOMsg struct {
57+
TxnUUID string // (plstcx client; one control message)
58+
TCBMsg
59+
ListRange
3860
}
3961
)
4062

‎cmn/api.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@ import (
2020
"github.com/NVIDIA/aistore/cmn/nlog"
2121
)
2222

23+
// In this source: bucket props and assorted control messages that contain buckets, including:
24+
// - BsummResult
25+
// - ArchiveBckMsg
26+
// - TCOMsg
27+
2328
// Bprops - manageable, user-configurable, and inheritable (from cluster config).
2429
// Includes per-bucket user-configurable checksum, version, LRU, erasure-coding, and more.
2530
//

‎xact/xs/tcobjs.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ import (
3030
"github.com/NVIDIA/aistore/xact/xreg"
3131
)
3232

33+
// The flow: `BeginMsg`, `ContMsg`, and `doMsg`.
34+
// Single xaction to execute multiple list-range-prefix API calls for a given pair of (source, destination) buckets.
35+
3336
const PrefixTcoID = "tco-"
3437

3538
type (

0 commit comments

Comments
 (0)
0