for {
// FIXME: We can't use ListObjectsV2, it is not universally supported.
// (Ceph RGW supported ListObjectsV2 since v15.1.0, released 2020 Jan 30th)
// (as of 2020, DigitalOcean Spaces still does not support V2 - https://developers.digitalocean.com/documentation/spaces/#list-bucket-contents)
res, err := rs.svc.ListObjects(ctx, req)
if err != nil {
return errors.Trace(err)
}
for _, r := range res.Contents {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html#AmazonS3-ListObjects-response-NextMarker -
//
// `res.NextMarker` is populated only if we specify req.Delimiter.
// Aliyun OSS and minio will populate NextMarker no matter what,
// but this documented behavior does apply to AWS S3:
//
// "If response does not include the NextMarker and it is truncated,
// you can use the value of the last Key in the response as the marker
// in the subsequent request to get the next set of object keys."
req.Marker = r.Key
// when walk on specify directory, the result include storage.Prefix,
// which can not be reuse in other API(Open/Read) directly.
// so we use TrimPrefix to filter Prefix for next Open/Read.
path := strings.TrimPrefix(*r.Key, rs.options.Prefix)
// trim the prefix '/' to ensure that the path returned is consistent with the local storage
path = strings.TrimPrefix(path, "/")
itemSize := *r.Size
// filter out s3's empty directory items
if itemSize <= 0 && strings.HasSuffix(path, "/") {
log.Info("this path is an empty directory and cannot be opened in S3. Skip it", zap.String("path", path))
continue
}
if err = fn(path, itemSize); err != nil {
return errors.Trace(err)
}
}
if !aws.ToBool(res.IsTruncated) {
break
}
}
并且对于 v1/global_checkpoint 目录下的所有 .ts 文件，是调用 ListObjects （具体代码再 tidb/br/pkg/storage/s3.go 的 func WalkDir 中）获取列表，倘若不支持这个接口，也是无法拿到 metadata