8000 CLI: list-objects to show page number; show num-cached when applicable · NVIDIA/aistore@31a869c · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 31a869c

Browse files
alex-aizmanalex-aizman
committed
CLI: list-objects to show page number; show num-cached when applicable
* '--paged' option to count pages: 1, 2, 3, ... * following are some of the possible footer variations (examples): - Listed 12345 names - Listed 12345 names (in-cluster: 456) - Page 123: 1000 names (in-cluster: none) * docs: update docs/cli/bucket.md, add examples Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent 7e563bf commit 31a869c

File tree

2 files changed

+110
-20
lines changed

2 files changed

+110
-20
lines changed

‎cmd/cli/cli/ls.go

Lines changed: 46 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -450,7 +450,10 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
450450
}
451451
// list (and immediately show) pages, one page at a time
452452
if flagIsSet(c, pagedFlag) {
453-
pageCounter, toShow := 0, int(limit)
453+
var (
454+
pageCounter int
455+
toShow = int(limit)
456+
)
454457
for {
455458
if catOnly {
456459
now = mono.NanoTime()
@@ -469,7 +472,7 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
469472
toPrint = lst.Entries
470473
}
471474
err = printLso(c, toPrint, lstFilter, propsStr, nil /*_listed*/, now,
472-
addCachedCol, bck.IsRemote(), msg.IsFlagSet(apc.LsDiff))
475+
pageCounter+1, addCachedCol, bck.IsRemote(), msg.IsFlagSet(apc.LsDiff))
473476
if err != nil {
474477
return err
475478
}
@@ -514,7 +517,7 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch, printEmpt
514517
if len(lst.Entries) == 0 && !printEmpty {
515518
return fmt.Errorf("%s/%s not found", bck.Cname(""), msg.Prefix)
516519
}
517-
return printLso(c, lst.Entries, lstFilter, propsStr, _listed, now,
520+
return printLso(c, lst.Entries, lstFilter, propsStr, _listed, now, 0, /*npage*/
518521
addCachedCol, bck.IsRemote(), msg.IsFlagSet(apc.LsDiff))
519522
}
520523

@@ -575,9 +578,10 @@ func _setPage(c *cli.Context, bck cmn.Bck) (pageSize, maxPages, limit int64, err
575578
}
576579

577580
// NOTE: in addition to CACHED, may also dynamically add STATUS column
578-
func printLso(c *cli.Context, entries cmn.LsoEntries, lstFilter *lstFilter, props string, _listed *_listed, now int64,
581+
func printLso(c *cli.Context, entries cmn.LsoEntries, lstFilter *lstFilter, props string, _listed *_listed, now int64, npage int,
579582
addCachedCol, isRemote, addStatusCol bool) error {
580583
var (
584+
numCached = -1
581585
hideHeader = flagIsSet(c, noHeaderFlag)
582586
hideFooter = flagIsSet(c, noFooterFlag)
583587
matched, other = lstFilter.apply(entries)
@@ -588,33 +592,35 @@ func printLso(c *cli.Context, entries cmn.LsoEntries, lstFilter *lstFilter, prop
588592
}
589593

590594
propsList := splitCsv(props)
591-
if isRemote && !addStatusCol {
592-
if addCachedCol && !cos.StringInSlice(apc.GetPropsStatus, propsList) {
593-
for _, en := range entries {
594-
if en.IsAnyFlagSet(apc.EntryVerChanged | apc.EntryVerRemoved) {
595-
addStatusCol = true
596-
break
597-
}
598-
}
599-
}
600-
}
601595

602-
// validate props for typos
596+
// validate props
603597
for _, prop := range propsList {
604598
if _, ok := teb.ObjectPropsMap[prop]; !ok {
605599
return fmt.Errorf("unknown object property %q (expecting one of: %v)",
606600
prop, cos.StrKVs(teb.ObjectPropsMap).Keys())
607601
}
608602
}
609603

604+
// remote bucket: count cached; additional condition to add status
605+
if isRemote && addCachedCol {
606+
numCached = 0
607+
for _, en := range matched {
608+
if en.IsPresent() {
609+
numCached++
610+
}
611+
if en.IsAnyFlagSet(apc.EntryVerChanged | apc.EntryVerRemoved) {
612+
addStatusCol = true
613+
}
614+
}
615+
}
616+
610617
// count-and-time only
611618
if now > 0 {
612619
// unless caption already printed
613620
if _listed != nil && _listed.cptn {
614621
return nil
615622
}
616-
elapsed := teb.FormatDuration(mono.Since(now))
617-
fmt.Fprintln(c.App.Writer, listedText, cos.FormatBigInt(len(matched)), "names in", elapsed)
623+
lsCptn(c, npage, len(matched), numCached, mono.Since(now))
618624
return nil
619625
}
620626

@@ -633,7 +639,7 @@ func printLso(c *cli.Context, entries cmn.LsoEntries, lstFilter *lstFilter, prop
633639
}
634640

635641
if !hideFooter && len(matched) > 10 {
636-
fmt.Fprintln(c.App.Writer, fblue(listedText), cos.FormatBigInt(len(matched)), "names")
642+
lsCptn(c, npage, len(matched), numCached, 0)
637643
}
638644
if flagIsSet(c, showUnmatchedFlag) && len(other) > 0 {
639645
unmatched := fcyan("\nNames that didn't match: ") + strconv.Itoa(len(other))
@@ -645,6 +651,27 @@ func printLso(c *cli.Context, entries cmn.LsoEntries, lstFilter *lstFilter, prop
645651
return nil
646652
}
647653

654+
func lsCptn(c *cli.Context, npage, ntotal, ncached int, elapsed time.Duration) {
655+
var (
656+
prompt = listedText
657+
names = "names"
658+
)
659+
if npage > 0 {
660+
prompt = "Page " + strconv.Itoa(npage) + ":"
661+
}
662+
if ncached == 0 {
663+
names += " (in-cluster: none)"
664+
} else if ncached > 0 && ncached != ntotal {
665+
debug.Assert(ncached < ntotal, ncached, " vs ", ntotal)
666+
names += " (in-cluster: " + cos.FormatBigInt(ncached) + ")"
667+
}
668+
if elapsed > 0 {
669+
fmt.Fprintln(c.App.Writer, fblue(prompt), cos.FormatBigInt(ntotal), names, "in", teb.FormatDuration(elapsed))
670+
} else {
671+
fmt.Fprintln(c.App.Writer, fblue(prompt), cos.FormatBigInt(ntotal), names)
672+
}
673+
}
674+
648675
///////////////
649676
// lstFilter //
650677
///////////////
@@ -764,6 +791,7 @@ func (u *_listed) cb(lsoCounter *api.LsoCounter) {
764791
if lsoCounter.IsFinished() || (u.limit > 0 && u.limit <= lsoCounter.Count()) {
765792
u.done = true
766793
if !flagIsSet(u.c, noFooterFlag) {
794+
// (compare w/ lsCptn)
767795
elapsed := teb.FormatDuration(lsoCounter.Elapsed())
768796
fmt.Fprintf(u.c.App.Writer, "\r%s %s names in %s\n", listedText, cos.FormatBigInt(lsoCounter.Count()), elapsed)
769797
u.cptn = true

‎docs/cli/bucket.md

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -411,11 +411,24 @@ OPTIONS:
411411
| `--bytes` | `bool` | show sizes in bytes (ie., do not convert to KiB, MiB, GiB, etc.) | `false` |
412412
| `--name-only` | `bool` | fast request to retrieve only the names of objects in the bucket; if defined, all comma-separated fields in the `--props` flag will be ignored with only two exceptions: `name` and `status` | `false` |
413413

414+
### Footer Information:
415+
416+
When listing objects, a footer will be displayed showing:
417+
- Total number of objects listed
418+
- For remote buckets with `--cached` option: number of objects present in-cluster
419+
- For `--paged` option: current page number
420+
- For `--count-only` option: time elapsed to fetch the list
421+
422+
Examples of footer variations:
423+
* `Listed 12345 names`
424+
* `Listed 12345 names (in-cluster: 456)`
425+
* `Page 123: 1000 names (in-cluster: none)`
426+
414427
### Examples
415428

416429
#### List AIS and Cloud buckets with all defaults
417430

418-
List objects in the AIS bucket `bucket_name`.
431+
**1**. List objects in the AIS bucket `bucket_name`.
419432

420433
```console
421434
$ ais ls ais://bucket_name
@@ -425,7 +438,7 @@ shard-1.tar 16.00KiB
425438
...
426439
```
427440

428-
List objects in the remote bucket `bucket_name`.
441+
**2**. List objects in the remote bucket `bucket_name`.
429442

430443
```console
431444
ais ls aws://bucket_name
@@ -435,6 +448,55 @@ shard-1.tar 16.00KiB
435448
...
436449
```
437450

451+
**3**. List objects from a remote AIS cluster with a namespace:
452+
```
453+
$ ais ls ais://@Bghort1l#ml/bucket_name
454+
NAME SIZE VERSION
455+
shard-0.tar 16.00KiB 1
456+
shard-1.tar 16.00KiB 1
457+
...
458+
```
459+
460+
**4**. List objects with paged output (showing page numbers):
461+
```
462+
$ ais ls ais://bucket_name --paged --limit 100
463+
[... object listing ...]
464+
465+
Page 1: 100 names
466+
```
467+
468+
**5**. List cached objects from a remote bucket:
469+
```
470+
$ ais ls s3://bucket_name --cached
471+
[... listing of only in-cluster objects ...]
472+
473+
Listed 456789 names
474+
```
475+
476+
**6**. Count objects in a bucket:
477+
```
478+
$ ais ls s3://bucket_name/aprefix --count-only
479+
Listed 28,230 names in 5.62s
480+
```
481+
482+
**7**. Count objects with paged output:
483+
```
484+
$ ais ls s3://bucket_name/bprefix --count-only --paged
485+
486+
Page 1: 1,000 names in 772ms
487+
Page 2: 1,000 names in 180ms
488+
Page 3: 1,000 names in 265ms
489+
...
490+
Page 29: 230 names in 130ms
491+
```
492+
493+
### Notes:
494+
495+
- When using `--paged` with remote buckets, the footer will show both page number and in-cluster object count when applicable
496+
- The `--diff` option requires remote backends supporting some form of versioning (e.g., object version, checksum, and/or ETag)
497+
- For more information on working with archived content, see docs/archive.md
498+
- To fully synchronize in-cluster content with remote backend, see documentation on [out-of-band updates](/docs/out_of_band.md)
499+
438500
#### Include all properties
439501

440502
```console

0 commit comments

Comments
 (0)
0