diff --git a/common/lock_map.go b/common/lock_map.go index 753447941..c24924253 100644 --- a/common/lock_map.go +++ b/common/lock_map.go @@ -36,8 +36,7 @@ type LockMapItem struct { mtx sync.RWMutex downloadTime time.Time // track if file is in lazy open state - LazyOpen bool - SyncPending bool + LazyOpen bool } // Map holding locks for all the files diff --git a/common/types.go b/common/types.go index 70bc112f0..f32b7569f 100644 --- a/common/types.go +++ b/common/types.go @@ -99,6 +99,51 @@ func CloudfuseVersion_() string { return cloudfuseVersion_ } +// custom errors shared by different components +type CloudUnreachableError struct { + Message string + CloudStorageError error +} + +func NewCloudUnreachableError(originalError error) CloudUnreachableError { + return CloudUnreachableError{ + Message: "Failed to connect to cloud storage", + CloudStorageError: originalError, + } +} +func (e CloudUnreachableError) Error() string { + return fmt.Sprintf("%s. Here's why: %v", e.Message, e.CloudStorageError) +} +func (e CloudUnreachableError) Unwrap() error { + return e.CloudStorageError +} +func (e CloudUnreachableError) Is(target error) bool { + _, ok := target.(*CloudUnreachableError) + return ok +} + +type NoCachedDataError struct { + Message string + CacheError error +} + +func NewNoCachedDataError(originalError error) CloudUnreachableError { + return CloudUnreachableError{ + Message: "Failed to connect to cloud storage", + CloudStorageError: originalError, + } +} +func (e NoCachedDataError) Error() string { + return fmt.Sprintf("%s. Here's why: %v", e.Message, e.CacheError) +} +func (e NoCachedDataError) Unwrap() error { + return e.CacheError +} +func (e NoCachedDataError) Is(target error) bool { + _, ok := target.(*NoCachedDataError) + return ok +} + var DefaultWorkDir string var DefaultLogFilePath string var StatsConfigFilePath string diff --git a/component/attr_cache/attr_cache.go b/component/attr_cache/attr_cache.go index c3e0bf071..8e241c3e8 100644 --- a/component/attr_cache/attr_cache.go +++ b/component/attr_cache/attr_cache.go @@ -27,6 +27,7 @@ package attr_cache import ( "context" + "errors" "fmt" "os" "path" @@ -36,6 +37,7 @@ import ( "syscall" "time" + "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" "github.com/Seagate/cloudfuse/common/log" "github.com/Seagate/cloudfuse/internal" @@ -240,12 +242,12 @@ func (ac *AttrCache) deleteDirectory(path string, deletedAt time.Time) error { } // does the cache show this path as existing? -func (ac *AttrCache) pathExistsInCache(path string) bool { +func (ac *AttrCache) getItemIfExists(path string) *attrCacheItem { item, found := ac.cache.get(path) - if !found { - return false + if found && item.exists() { + return item } - return item.exists() + return nil } // returns the parent directory (without a trailing slash) @@ -449,7 +451,9 @@ func (ac *AttrCache) CreateDir(options internal.CreateDirOptions) error { exists: true, cachedAt: time.Now(), }) - // update flags for tracking directory existence + // this is a new directory, so we have a complete (empty) listing for it + newDirAttrCacheItem.listingComplete = true + // update flag for tracking directory existence if ac.cacheDirs { newDirAttrCacheItem.markInCloud(false) } @@ -534,15 +538,25 @@ func (ac *AttrCache) StreamDir( options.Name, numAdded, len(pathList)) } } - } - // add cached items in - if len(cachedPathList) > 0 { - log.Info( - "AttrCache::StreamDir : %s merging in %d list cache entries...", - options.Name, - len(cachedPathList), - ) - pathList = append(pathList, cachedPathList...) + } else if errors.Is(err, &common.CloudUnreachableError{}) { + // return expired cachedPathList + if cachedPathList != nil { + pathList = cachedPathList + nextToken = cachedToken + } else { + // return whatever entries we have (but only if the token is empty) + entry, found := ac.cache.get(options.Name) + if options.Token == "" && found && entry.listingComplete { + for _, v := range entry.children { + if v.exists() && v.valid() { + pathList = append(pathList, v.attr) + } + } + } else { + // the cloud is unavailable, and we have nothing to provide + err = common.NewNoCachedDataError(err) + } + } } // values should be returned in ascending order by key, without duplicates // sort @@ -559,7 +573,18 @@ func (ac *AttrCache) StreamDir( return a.Path == b.Path }, ) - ac.cacheListSegment(pathList, options.Name, options.Token, nextToken) + // cache the listing (if there was no error) + if err == nil { + // record when the directory was listed, an up to what token + // this will allow us to serve directory listings from this cache + ac.cacheListSegment(pathList, options.Name, options.Token, nextToken) + // if the listing is complete, record the fact that we have a complete listing + if nextToken == "" { + ac.markListingComplete(options.Name) + } + } else { + log.Err("AttrCache::StreamDir : %s encountered error [%v]", options.Name, err) + } log.Trace("AttrCache::StreamDir : %s returning %d entries", options.Name, len(pathList)) return pathList, nextToken, err } @@ -572,9 +597,8 @@ func (ac *AttrCache) fetchCachedDirList( path string, token string, ) ([]*internal.ObjAttr, string, error) { - var pathList []*internal.ObjAttr if !ac.cacheOnList { - return pathList, "", fmt.Errorf("cache on list is disabled") + return nil, "", fmt.Errorf("cache on list is disabled") } // start accessing the cache ac.cacheLock.RLock() @@ -583,25 +607,22 @@ func (ac *AttrCache) fetchCachedDirList( listDirCache, found := ac.cache.get(path) if !found { log.Warn("AttrCache::fetchCachedDirList : %s directory not found in cache", path) - return pathList, "", fmt.Errorf("%s directory not found in cache", path) + return nil, "", common.NewNoCachedDataError( + fmt.Errorf("%s directory not found in cache", path), + ) } // is the requested data cached? - if listDirCache.listCache == nil { - listDirCache.listCache = make(map[string]listCacheSegment) - } cachedListSegment, found := listDirCache.listCache[token] if !found { // the data for this token is not in the cache // don't provide cached data when new (uncached) data is being requested log.Info("AttrCache::fetchCachedDirList : %s listing segment %s not cached", path, token) - return pathList, "", fmt.Errorf("%s directory listing segment %s not cached", path, token) + return nil, "", fmt.Errorf("%s directory listing segment %s not cached", path, token) } // check timeout if time.Since(cachedListSegment.cachedAt).Seconds() >= float64(ac.cacheTimeout) { log.Info("AttrCache::fetchCachedDirList : %s listing segment %s cache expired", path, token) - // drop the invalid segment from the list cache - delete(listDirCache.listCache, token) - return pathList, "", fmt.Errorf( + return cachedListSegment.entries, "", fmt.Errorf( "%s directory listing segment %s cache expired", path, token, @@ -673,16 +694,19 @@ func (ac *AttrCache) cacheListSegment( } // add the new entry listDirItem.listCache[token] = newListCacheSegment - // scan the listing cache and remove expired entries - for k, v := range listDirItem.listCache { - if currTime.Sub(v.cachedAt).Seconds() >= float64(ac.cacheTimeout) { - delete(listDirItem.listCache, k) - } - } log.Trace("AttrCache::cacheListSegment : %s cached list entries \"%s\"-\"%s\" (%d items)", listDirPath, token, nextToken, len(pathList)) } +func (ac *AttrCache) markListingComplete(listDirPath string) { + ac.cacheLock.Lock() + defer ac.cacheLock.Unlock() + listDirItem, found := ac.cache.get(listDirPath) + if found { + listDirItem.listingComplete = true + } +} + // IsDirEmpty: Whether or not the directory is empty func (ac *AttrCache) IsDirEmpty(options internal.IsDirEmptyOptions) bool { log.Trace("AttrCache::IsDirEmpty : %s", options.Name) @@ -693,14 +717,15 @@ func (ac *AttrCache) IsDirEmpty(options internal.IsDirEmptyOptions) bool { "AttrCache::IsDirEmpty : %s Dir cache is disabled. Checking with container", options.Name, ) + // when offline, this will return false return ac.NextComponent().IsDirEmpty(options) } // Is the directory in our cache? ac.cacheLock.RLock() - pathInCache := ac.pathExistsInCache(options.Name) - ac.cacheLock.RUnlock() + defer ac.cacheLock.RUnlock() + item := ac.getItemIfExists(options.Name) // If the directory does not exist in the attribute cache then let the next component answer - if !pathInCache { + if item == nil { log.Debug( "AttrCache::IsDirEmpty : %s not found in attr_cache. Checking with container", options.Name, @@ -709,10 +734,15 @@ func (ac *AttrCache) IsDirEmpty(options internal.IsDirEmptyOptions) bool { } log.Debug("AttrCache::IsDirEmpty : %s found in attr_cache", options.Name) // Check if the cached directory is empty or not - if ac.anyContentsInCache(options.Name) { + if item.hasExistingChildren() { log.Debug("AttrCache::IsDirEmpty : %s has a subpath in attr_cache", options.Name) return false } + // do we have a complete listing? + if item.listingComplete { + // we know the directory is empty + return true + } // Dir is in cache but no contents are, so check with container log.Debug( "AttrCache::IsDirEmpty : %s children not found in cache. Checking with container", @@ -721,16 +751,10 @@ func (ac *AttrCache) IsDirEmpty(options internal.IsDirEmptyOptions) bool { return ac.NextComponent().IsDirEmpty(options) } -func (ac *AttrCache) anyContentsInCache(prefix string) bool { - ac.cacheLock.RLock() - defer ac.cacheLock.RUnlock() - - directory, found := ac.cache.get(prefix) - if found && directory.exists() { - for _, chldItem := range directory.children { - if chldItem.exists() { - return true - } +func (value *attrCacheItem) hasExistingChildren() bool { + for _, childItem := range value.children { + if childItem.exists() { + return true } } return false @@ -752,7 +776,7 @@ func (ac *AttrCache) RenameDir(options internal.RenameDirOptions) error { if ac.cacheDirs { // if attr_cache is tracking directories, validate this rename // First, check if the destination directory already exists - if ac.pathExistsInCache(options.Dst) { + if ac.getItemIfExists(options.Dst) != nil { return os.ErrExist } } else { @@ -1121,9 +1145,10 @@ func (ac *AttrCache) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr ac.cacheLock.Lock() defer ac.cacheLock.Unlock() - switch err { - case nil: + switch { + case err == nil: // Retrieved attributes so cache them + log.Debug("AttrCache::GetAttr : %s Caching record from cloud", options.Name) ac.cache.insert(insertOptions{ attr: pathAttr, exists: true, @@ -1132,13 +1157,43 @@ func (ac *AttrCache) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr if ac.cacheDirs { ac.markAncestorsInCloud(getParentDir(options.Name), time.Now()) } - case syscall.ENOENT: + case err == syscall.ENOENT: // cache this entity not existing + log.Debug("AttrCache::GetAttr : %s Caching ENOENT from cloud", options.Name) ac.cache.insert(insertOptions{ attr: internal.CreateObjAttr(options.Name, 0, time.Now()), exists: false, cachedAt: time.Now(), }) + case errors.Is(err, &common.CloudUnreachableError{}): + // the cloud connection is down + // do we have an entry, but it's expired? Let's serve that. + if found && value.valid() { + // Serve the request from the attribute cache + if !value.exists() { + log.Debug("AttrCache::GetAttr : %s ENOENT found in cache (offline)", options.Name) + return value.attr, errors.Join(syscall.ENOENT, err) + } else { + log.Debug("AttrCache::GetAttr : %s Entry found in cache (offline)", options.Name) + return value.attr, err + } + } else { + // we have no cached data about this item + // but do we have a complete listing for its parent directory? + entry, found := ac.cache.get(getParentDir(options.Name)) + if found && entry.listingComplete { + log.Debug("AttrCache::GetAttr : %s Not in directory (offline)", options.Name) + return nil, errors.Join(syscall.ENOENT, err) + } else { + // we have no way of knowing whether the requested item is in the directory in the cloud + // NOTE: + // the OS can call GetAttr on a file without listing its parent directory + // so having a valid file entry in cache does not mean we have a complete listing of its parent + // so we can't just check if the directory has any children as a proxy for whether it's been listed + log.Err("AttrCache::GetAttr : %s No cached data (offline)", options.Name) + return nil, common.NewNoCachedDataError(err) + } + } } return pathAttr, err } diff --git a/component/attr_cache/cacheMap.go b/component/attr_cache/cacheMap.go index 6fdea8309..0df0b1fc9 100644 --- a/component/attr_cache/cacheMap.go +++ b/component/attr_cache/cacheMap.go @@ -58,6 +58,8 @@ type attrCacheItem struct { attrFlag common.BitMap16 children map[string]*attrCacheItem parent *attrCacheItem + + listingComplete bool } // all cache entries are organized into this structure diff --git a/component/azstorage/azstorage.go b/component/azstorage/azstorage.go index 81035c6d6..9c0c3318b 100644 --- a/component/azstorage/azstorage.go +++ b/component/azstorage/azstorage.go @@ -27,7 +27,9 @@ package azstorage import ( "context" + "errors" "fmt" + "sync" "sync/atomic" "syscall" "time" @@ -39,6 +41,7 @@ import ( "github.com/Seagate/cloudfuse/internal/handlemap" "github.com/Seagate/cloudfuse/internal/stats_manager" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" ) @@ -49,6 +52,16 @@ type AzStorage struct { stConfig AzStorageConfig startTime time.Time listBlocked bool + state connectionState + ctx context.Context + cancelFn context.CancelFunc +} + +type connectionState struct { + sync.Mutex + lastConnectionAttempt *time.Time + firstOffline *time.Time + retryTicker *time.Ticker } const compName = "azstorage" @@ -194,6 +207,16 @@ func (az *AzStorage) Start(ctx context.Context) error { // create stats collector for azstorage azStatsCollector = stats_manager.NewStatsCollector(az.Name()) log.Debug("Starting azstorage stats collector") + // create a shared context for all cloud operations, with ability to cancel + az.ctx, az.cancelFn = context.WithCancel(ctx) + // create the retry ticker + az.state.retryTicker = time.NewTicker(time.Duration(az.stConfig.backoffTime) * time.Second) + az.state.retryTicker.Stop() // stop it for now, we will start it when we are offline + go func() { + for range az.state.retryTicker.C { + az.CloudConnected() + } + }() return nil } @@ -205,9 +228,108 @@ func (az *AzStorage) Stop() error { return nil } +// ------------------------- Connectivity check ------------------------------------------- + +// Online check +func (az *AzStorage) CloudConnected() bool { + log.Trace("AzStorage::CloudConnected") + connected := az.state.firstOffline == nil + // don't check the connection when it's up, or if we are not ready to retry + if connected || !az.timeToRetry() { + return connected + } + // check connection + ctx, cancelFun := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancelFun() + err := az.storage.ConnectionOkay(ctx) + log.Debug("AzStorage::CloudConnected : err is %v", err) + nowConnected := az.updateConnectionState(err) + return nowConnected +} + +func (az *AzStorage) timeToRetry() bool { + timeSinceLastAttempt := time.Since(*az.state.lastConnectionAttempt) + switch { + case timeSinceLastAttempt < time.Duration(az.stConfig.backoffTime)*time.Second: + // minimum delay before retrying + return false + case timeSinceLastAttempt > 90*time.Second: + // maximum delay + return true + default: + // when between the minimum and maximum delay, we use an exponential backoff + timeOfflineAtLastAttempt := az.state.lastConnectionAttempt.Sub(*az.state.firstOffline) + return timeSinceLastAttempt > timeOfflineAtLastAttempt + } +} + +func (az *AzStorage) updateConnectionState(err error) bool { + az.state.Lock() + defer az.state.Unlock() + currentTime := time.Now() + az.state.lastConnectionAttempt = ¤tTime + connected := !isOfflineError(err) + wasConnected := az.state.firstOffline == nil + stateChanged := connected != wasConnected + if stateChanged { + log.Warn("AzStorage::updateConnectionState : connected is now: %t", connected) + if connected { + az.state.firstOffline = nil + // reset the context to allow new requests + az.ctx, az.cancelFn = context.WithCancel(context.Background()) + // stop the retry ticker + az.state.retryTicker.Stop() + } else { + az.state.firstOffline = ¤tTime + // cancel all outstanding requests + az.cancelFn() + log.Warn("AzStorage::updateConnectionState : cancelled all outstanding requests") + // reset the ticker to retry the connection + az.state.retryTicker.Reset(time.Duration(az.stConfig.backoffTime) * time.Second) + } + } + return connected +} + +func isOfflineError(err error) bool { + // handle common error cases + switch { + case err == nil: + return false + case errors.Is(err, syscall.ENOENT): + return false + case errors.Is(err, context.DeadlineExceeded): + return true + case errors.Is(err, context.Canceled): + return true + case errors.Is(err, syscall.ECONNREFUSED): + return true + case errors.Is(err, &common.CloudUnreachableError{}): + return true + default: + var respErr *azcore.ResponseError + errors.As(err, &respErr) + if respErr != nil && storeBlobErrToErr(respErr) != ErrUnknown { + log.Debug("isOfflineError: errors.As(err, &respErr)") + return false + } + // log the error details + unwrappedErr := err + for unwrappedErr != nil { + log.Debug( + "isOfflineError: Uncaught AZ error is of type \"%T\" and value [%v].\n", + unwrappedErr, + unwrappedErr, + ) + unwrappedErr = errors.Unwrap(unwrappedErr) + } + return false + } +} + // ------------------------- Container listing ------------------------------------------- func (az *AzStorage) ListContainers() ([]string, error) { - return az.storage.ListContainers() + return az.storage.ListContainers(az.ctx) } // ------------------------- Core Operations ------------------------------------------- @@ -216,7 +338,8 @@ func (az *AzStorage) ListContainers() ([]string, error) { func (az *AzStorage) CreateDir(options internal.CreateDirOptions) error { log.Trace("AzStorage::CreateDir : %s", options.Name) - err := az.storage.CreateDirectory(internal.TruncateDirName(options.Name)) + err := az.storage.CreateDirectory(az.ctx, internal.TruncateDirName(options.Name)) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents( @@ -233,7 +356,8 @@ func (az *AzStorage) CreateDir(options internal.CreateDirOptions) error { func (az *AzStorage) DeleteDir(options internal.DeleteDirOptions) error { log.Trace("AzStorage::DeleteDir : %s", options.Name) - err := az.storage.DeleteDirectory(internal.TruncateDirName(options.Name)) + err := az.storage.DeleteDirectory(az.ctx, internal.TruncateDirName(options.Name)) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents(deleteDir, options.Name, nil) @@ -256,7 +380,8 @@ func formatListDirName(path string) string { func (az *AzStorage) IsDirEmpty(options internal.IsDirEmptyOptions) bool { log.Trace("AzStorage::IsDirEmpty : %s", options.Name) - list, _, err := az.storage.List(formatListDirName(options.Name), nil, 1) + list, _, err := az.storage.List(az.ctx, formatListDirName(options.Name), nil, 1) + az.updateConnectionState(err) if err != nil { log.Err("AzStorage::IsDirEmpty : error listing [%s]", err) return false @@ -293,7 +418,8 @@ func (az *AzStorage) StreamDir( options.Count = common.MaxDirListCount } - new_list, new_marker, err := az.storage.List(path, &options.Token, options.Count) + new_list, new_marker, err := az.storage.List(az.ctx, path, &options.Token, options.Count) + az.updateConnectionState(err) if err != nil { log.Err("AzStorage::StreamDir : Failed to read dir [%s]", err) return new_list, "", err @@ -340,7 +466,8 @@ func (az *AzStorage) RenameDir(options internal.RenameDirOptions) error { options.Src = internal.TruncateDirName(options.Src) options.Dst = internal.TruncateDirName(options.Dst) - err := az.storage.RenameDirectory(options.Src, options.Dst) + err := az.storage.RenameDirectory(az.ctx, options.Src, options.Dst) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents( @@ -365,7 +492,8 @@ func (az *AzStorage) CreateFile(options internal.CreateFileOptions) (*handlemap. return nil, syscall.EFAULT } - err := az.storage.CreateFile(options.Name, options.Mode) + err := az.storage.CreateFile(az.ctx, options.Name, options.Mode) + az.updateConnectionState(err) if err != nil { return nil, err } @@ -386,7 +514,8 @@ func (az *AzStorage) CreateFile(options internal.CreateFileOptions) (*handlemap. func (az *AzStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error) { log.Trace("AzStorage::OpenFile : %s", options.Name) - attr, err := az.storage.GetAttr(options.Name) + attr, err := az.storage.GetAttr(az.ctx, options.Name) + az.updateConnectionState(err) if err != nil { return nil, err } @@ -419,7 +548,8 @@ func (az *AzStorage) CloseFile(options internal.CloseFileOptions) error { func (az *AzStorage) DeleteFile(options internal.DeleteFileOptions) error { log.Trace("AzStorage::DeleteFile : %s", options.Name) - err := az.storage.DeleteFile(options.Name) + err := az.storage.DeleteFile(az.ctx, options.Name) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents(deleteFile, options.Name, nil) @@ -432,7 +562,8 @@ func (az *AzStorage) DeleteFile(options internal.DeleteFileOptions) error { func (az *AzStorage) RenameFile(options internal.RenameFileOptions) error { log.Trace("AzStorage::RenameFile : %s to %s", options.Src, options.Dst) - err := az.storage.RenameFile(options.Src, options.Dst, options.SrcAttr) + err := az.storage.RenameFile(az.ctx, options.Src, options.Dst, options.SrcAttr) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents( @@ -476,7 +607,7 @@ func (az *AzStorage) ReadInBuffer(options internal.ReadInBufferOptions) (length } length = int(dataLen) - err = az.storage.ReadInBuffer(path, options.Offset, dataLen, options.Data, options.Etag) + err = az.storage.ReadInBuffer(az.ctx, path, options.Offset, dataLen, options.Data, options.Etag) if err != nil { log.Err("AzStorage::ReadInBuffer : Failed to read %s [%s]", path, err.Error()) length = 0 @@ -486,20 +617,23 @@ func (az *AzStorage) ReadInBuffer(options internal.ReadInBufferOptions) (length } func (az *AzStorage) WriteFile(options internal.WriteFileOptions) (int, error) { - err := az.storage.Write(options) + err := az.storage.Write(az.ctx, options) + az.updateConnectionState(err) return len(options.Data), err } func (az *AzStorage) GetFileBlockOffsets( options internal.GetFileBlockOffsetsOptions, ) (*common.BlockOffsetList, error) { - return az.storage.GetFileBlockOffsets(options.Name) - + bol, err := az.storage.GetFileBlockOffsets(az.ctx, options.Name) + az.updateConnectionState(err) + return bol, err } func (az *AzStorage) TruncateFile(options internal.TruncateFileOptions) error { log.Trace("AzStorage::TruncateFile : %s to %d bytes", options.Name, options.Size) - err := az.storage.TruncateFile(options.Name, options.Size) + err := az.storage.TruncateFile(az.ctx, options.Name, options.Size) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents( @@ -514,12 +648,16 @@ func (az *AzStorage) TruncateFile(options internal.TruncateFileOptions) error { func (az *AzStorage) CopyToFile(options internal.CopyToFileOptions) error { log.Trace("AzStorage::CopyToFile : Read file %s", options.Name) - return az.storage.ReadToFile(options.Name, options.Offset, options.Count, options.File) + err := az.storage.ReadToFile(az.ctx, options.Name, options.Offset, options.Count, options.File) + az.updateConnectionState(err) + return err } func (az *AzStorage) CopyFromFile(options internal.CopyFromFileOptions) error { log.Trace("AzStorage::CopyFromFile : Upload file %s", options.Name) - return az.storage.WriteFromFile(options.Name, options.Metadata, options.File) + err := az.storage.WriteFromFile(az.ctx, options.Name, options.Metadata, options.File) + az.updateConnectionState(err) + return err } // Symlink operations @@ -533,7 +671,8 @@ func (az *AzStorage) CreateLink(options internal.CreateLinkOptions) error { return syscall.ENOTSUP } log.Trace("AzStorage::CreateLink : Create symlink %s -> %s", options.Name, options.Target) - err := az.storage.CreateLink(options.Name, options.Target) + err := az.storage.CreateLink(az.ctx, options.Name, options.Target) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents( @@ -553,7 +692,8 @@ func (az *AzStorage) ReadLink(options internal.ReadLinkOptions) (string, error) return "", syscall.ENOENT } log.Trace("AzStorage::ReadLink : Read symlink %s", options.Name) - data, err := az.storage.ReadBuffer(options.Name, 0, options.Size) + data, err := az.storage.ReadBuffer(az.ctx, options.Name, 0, options.Size) + az.updateConnectionState(err) if err != nil { azStatsCollector.PushEvents(readLink, options.Name, nil) @@ -566,12 +706,15 @@ func (az *AzStorage) ReadLink(options internal.ReadLinkOptions) (string, error) // Attribute operations func (az *AzStorage) GetAttr(options internal.GetAttrOptions) (attr *internal.ObjAttr, err error) { //log.Trace("AzStorage::GetAttr : Get attributes of file %s", name) - return az.storage.GetAttr(options.Name) + attr, err = az.storage.GetAttr(az.ctx, options.Name) + az.updateConnectionState(err) + return attr, err } func (az *AzStorage) Chmod(options internal.ChmodOptions) error { log.Trace("AzStorage::Chmod : Change mod of file %s", options.Name) - err := az.storage.ChangeMod(options.Name, options.Mode) + err := az.storage.ChangeMod(az.ctx, options.Name, options.Mode) + az.updateConnectionState(err) if err == nil { azStatsCollector.PushEvents( @@ -592,24 +735,38 @@ func (az *AzStorage) Chown(options internal.ChownOptions) error { options.Owner, options.Group, ) - return az.storage.ChangeOwner(options.Name, options.Owner, options.Group) + err := az.storage.ChangeOwner(az.ctx, options.Name, options.Owner, options.Group) + az.updateConnectionState(err) + return err } func (az *AzStorage) FlushFile(options internal.FlushFileOptions) error { log.Trace("AzStorage::FlushFile : Flush file %s", options.Handle.Path) - return az.storage.StageAndCommit(options.Handle.Path, options.Handle.CacheObj.BlockOffsetList) + err := az.storage.StageAndCommit( + az.ctx, + options.Handle.Path, + options.Handle.CacheObj.BlockOffsetList, + ) + az.updateConnectionState(err) + return err } func (az *AzStorage) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error) { - return az.storage.GetCommittedBlockList(name) + cbl, err := az.storage.GetCommittedBlockList(az.ctx, name) + az.updateConnectionState(err) + return cbl, err } func (az *AzStorage) StageData(opt internal.StageDataOptions) error { - return az.storage.StageBlock(opt.Name, opt.Data, opt.Id) + err := az.storage.StageBlock(az.ctx, opt.Name, opt.Data, opt.Id) + az.updateConnectionState(err) + return err } func (az *AzStorage) CommitData(opt internal.CommitDataOptions) error { - return az.storage.CommitBlocks(opt.Name, opt.List, opt.NewETag) + err := az.storage.CommitBlocks(az.ctx, opt.Name, opt.List, opt.NewETag) + az.updateConnectionState(err) + return err } // TODO : Below methods are pending to be implemented diff --git a/component/azstorage/block_blob.go b/component/azstorage/block_blob.go index b84f6cde1..c9228bfa3 100644 --- a/component/azstorage/block_blob.go +++ b/component/azstorage/block_blob.go @@ -255,13 +255,13 @@ func (bb *BlockBlob) IsAccountADLS() bool { return false } -func (bb *BlockBlob) ListContainers() ([]string, error) { +func (bb *BlockBlob) ListContainers(ctx context.Context) ([]string, error) { log.Trace("BlockBlob::ListContainers : Listing containers") cntList := make([]string, 0) pager := bb.Service.NewListContainersPager(nil) for pager.More() { - resp, err := pager.NextPage(context.Background()) + resp, err := pager.NextPage(ctx) if err != nil { log.Err("BlockBlob::ListContainers : Failed to get container list [%s]", err.Error()) return cntList, err @@ -274,6 +274,13 @@ func (bb *BlockBlob) ListContainers() ([]string, error) { return cntList, nil } +// check the connection to the service by calling GetProperties on the container +func (bb *BlockBlob) ConnectionOkay(ctx context.Context) error { + log.Trace("BlockBlob::ConnectionOkay : checking connection to cloud service") + _, err := bb.Container.GetProperties(ctx, nil) + return err +} + func (bb *BlockBlob) SetPrefixPath(path string) error { log.Trace("BlockBlob::SetPrefixPath : path %s", path) bb.Config.prefixPath = path @@ -281,38 +288,38 @@ func (bb *BlockBlob) SetPrefixPath(path string) error { } // CreateFile : Create a new file in the container/virtual directory -func (bb *BlockBlob) CreateFile(name string, mode os.FileMode) error { +func (bb *BlockBlob) CreateFile(ctx context.Context, name string, mode os.FileMode) error { log.Trace("BlockBlob::CreateFile : name %s", name) var data []byte - return bb.WriteFromBuffer(name, nil, data) + return bb.WriteFromBuffer(ctx, name, nil, data) } // CreateDirectory : Create a new directory in the container/virtual directory -func (bb *BlockBlob) CreateDirectory(name string) error { +func (bb *BlockBlob) CreateDirectory(ctx context.Context, name string) error { log.Trace("BlockBlob::CreateDirectory : name %s", name) var data []byte metadata := make(map[string]*string) metadata[folderKey] = to.Ptr("true") - return bb.WriteFromBuffer(name, metadata, data) + return bb.WriteFromBuffer(ctx, name, metadata, data) } // CreateLink : Create a symlink in the container/virtual directory -func (bb *BlockBlob) CreateLink(source string, target string) error { +func (bb *BlockBlob) CreateLink(ctx context.Context, source string, target string) error { log.Trace("BlockBlob::CreateLink : %s -> %s", source, target) data := []byte(target) metadata := make(map[string]*string) metadata[symlinkKey] = to.Ptr("true") - return bb.WriteFromBuffer(source, metadata, data) + return bb.WriteFromBuffer(ctx, source, metadata, data) } // DeleteFile : Delete a blob in the container/virtual directory -func (bb *BlockBlob) DeleteFile(name string) (err error) { +func (bb *BlockBlob) DeleteFile(ctx context.Context, name string) (err error) { log.Trace("BlockBlob::DeleteFile : name %s", name) blobClient := bb.getBlobClient(name) - _, err = blobClient.Delete(context.Background(), &blob.DeleteOptions{ + _, err = blobClient.Delete(ctx, &blob.DeleteOptions{ DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude), }) if err != nil { @@ -334,9 +341,9 @@ func (bb *BlockBlob) DeleteFile(name string) (err error) { } // DeleteDirectory : Delete a virtual directory in the container/virtual directory -func (bb *BlockBlob) DeleteDirectory(name string) (err error) { +func (bb *BlockBlob) DeleteDirectory(ctx context.Context, name string) (err error) { log.Trace("BlockBlob::DeleteDirectory : name %s", name) - err = bb.DeleteFile(name) + err = bb.DeleteFile(ctx, name) // libfuse deletes the files in the directory before this method is called. // If the marker blob for directory is not present, ignore the ENOENT error. if err == syscall.ENOENT { @@ -352,7 +359,12 @@ func (bb *BlockBlob) DeleteDirectory(name string) (err error) { // Etag of the destination blob changes. // Copy the LMT to the src attr if the copy is success. // https://learn.microsoft.com/en-us/rest/api/storageservices/copy-blob?tabs=microsoft-entra-id -func (bb *BlockBlob) RenameFile(source string, target string, srcAttr *internal.ObjAttr) error { +func (bb *BlockBlob) RenameFile( + ctx context.Context, + source string, + target string, + srcAttr *internal.ObjAttr, +) error { log.Trace("BlockBlob::RenameFile : %s -> %s", source, target) blobClient := bb.getBlobClient(source) @@ -361,7 +373,7 @@ func (bb *BlockBlob) RenameFile(source string, target string, srcAttr *internal. // not specifying source blob metadata, since passing empty metadata headers copies // the source blob metadata to destination blob copyResponse, err := newBlobClient.StartCopyFromURL( - context.Background(), + ctx, blobClient.URL(), &blob.StartCopyFromURLOptions{ Tier: bb.Config.defaultTier, @@ -389,7 +401,7 @@ func (bb *BlockBlob) RenameFile(source string, target string, srcAttr *internal. for copyStatus != nil && *copyStatus == blob.CopyStatusTypePending { time.Sleep(time.Second * 1) pollCnt++ - prop, err = newBlobClient.GetProperties(context.Background(), &blob.GetPropertiesOptions{ + prop, err = newBlobClient.GetProperties(ctx, &blob.GetPropertiesOptions{ CPKInfo: bb.blobCPKOpt, }) if err != nil { @@ -414,7 +426,7 @@ func (bb *BlockBlob) RenameFile(source string, target string, srcAttr *internal. log.Trace("BlockBlob::RenameFile : %s -> %s done", source, target) // Copy of the file is done so now delete the older file - err = bb.DeleteFile(source) + err = bb.DeleteFile(ctx, source) for retry := 0; retry < 3 && err == syscall.ENOENT; retry++ { // Sometimes backend is able to copy source file to destination but when we try to delete the // source files it returns back with ENOENT. If file was just created on backend it might happen @@ -426,7 +438,7 @@ func (bb *BlockBlob) RenameFile(source string, target string, srcAttr *internal. retry, ) time.Sleep(1 * time.Second) - err = bb.DeleteFile(source) + err = bb.DeleteFile(ctx, source) } if err == syscall.ENOENT { @@ -439,7 +451,7 @@ func (bb *BlockBlob) RenameFile(source string, target string, srcAttr *internal. } // RenameDirectory : Rename the directory -func (bb *BlockBlob) RenameDirectory(source string, target string) error { +func (bb *BlockBlob) RenameDirectory(ctx context.Context, source string, target string) error { log.Trace("BlockBlob::RenameDirectory : %s -> %s", source, target) srcDirPresent := false @@ -447,7 +459,7 @@ func (bb *BlockBlob) RenameDirectory(source string, target string) error { Prefix: to.Ptr(bb.getFormattedPath(source) + "/"), }) for pager.More() { - listBlobResp, err := pager.NextPage(context.Background()) + listBlobResp, err := pager.NextPage(ctx) if err != nil { log.Err("BlockBlob::RenameDirectory : Failed to get list of blobs %s", err.Error()) return err @@ -457,7 +469,7 @@ func (bb *BlockBlob) RenameDirectory(source string, target string) error { for _, blobInfo := range listBlobResp.Segment.BlobItems { srcDirPresent = true srcPath := removePrefixPath(bb.Config.prefixPath, *blobInfo.Name) - err = bb.RenameFile(srcPath, strings.Replace(srcPath, source, target, 1), nil) + err = bb.RenameFile(ctx, srcPath, strings.Replace(srcPath, source, target, 1), nil) if err != nil { log.Err( "BlockBlob::RenameDirectory : Failed to rename file %s [%s]", @@ -470,7 +482,7 @@ func (bb *BlockBlob) RenameDirectory(source string, target string) error { // To rename source marker blob check its properties before calling rename on it. blobClient := bb.Container.NewBlockBlobClient(filepath.Join(bb.Config.prefixPath, source)) - _, err := blobClient.GetProperties(context.Background(), &blob.GetPropertiesOptions{ + _, err := blobClient.GetProperties(ctx, &blob.GetPropertiesOptions{ CPKInfo: bb.blobCPKOpt, }) if err != nil { @@ -490,14 +502,17 @@ func (bb *BlockBlob) RenameDirectory(source string, target string) error { } } - return bb.RenameFile(source, target, nil) + return bb.RenameFile(ctx, source, target, nil) } -func (bb *BlockBlob) getAttrUsingRest(name string) (attr *internal.ObjAttr, err error) { +func (bb *BlockBlob) getAttrUsingRest( + ctx context.Context, + name string, +) (attr *internal.ObjAttr, err error) { log.Trace("BlockBlob::getAttrUsingRest : name %s", name) blobClient := bb.getBlockBlobClient(name) - prop, err := blobClient.GetProperties(context.Background(), &blob.GetPropertiesOptions{ + prop, err := blobClient.GetProperties(ctx, &blob.GetPropertiesOptions{ CPKInfo: bb.blobCPKOpt, }) @@ -546,7 +561,10 @@ func (bb *BlockBlob) getAttrUsingRest(name string) (attr *internal.ObjAttr, err return attr, nil } -func (bb *BlockBlob) getAttrUsingList(name string) (attr *internal.ObjAttr, err error) { +func (bb *BlockBlob) getAttrUsingList( + ctx context.Context, + name string, +) (attr *internal.ObjAttr, err error) { log.Trace("BlockBlob::getAttrUsingList : name %s", name) iteration := 0 @@ -555,7 +573,7 @@ func (bb *BlockBlob) getAttrUsingList(name string) (attr *internal.ObjAttr, err blobsRead := 0 for marker != nil || iteration == 0 { - blobs, new_marker, err = bb.List(name, marker, bb.Config.maxResultsForList) + blobs, new_marker, err = bb.List(ctx, name, marker, bb.Config.maxResultsForList) if err != nil { e := storeBlobErrToErr(err) switch e { @@ -612,14 +630,14 @@ func (bb *BlockBlob) getAttrUsingList(name string) (attr *internal.ObjAttr, err } // GetAttr : Retrieve attributes of the blob -func (bb *BlockBlob) GetAttr(name string) (attr *internal.ObjAttr, err error) { +func (bb *BlockBlob) GetAttr(ctx context.Context, name string) (attr *internal.ObjAttr, err error) { log.Trace("BlockBlob::GetAttr : name %s", name) // To support virtual directories with no marker blob, we call list instead of get properties since list will not return a 404 if bb.Config.virtualDirectory { - attr, err = bb.getAttrUsingList(name) + attr, err = bb.getAttrUsingList(ctx, name) } else { - attr, err = bb.getAttrUsingRest(name) + attr, err = bb.getAttrUsingRest(ctx, name) } if bb.Config.filter != nil && attr != nil { @@ -640,6 +658,7 @@ func (bb *BlockBlob) GetAttr(name string) (attr *internal.ObjAttr, err error) { // This fetches the list using a marker so the caller code should handle marker logic // If count=0 - fetch max entries func (bb *BlockBlob) List( + ctx context.Context, prefix string, marker *string, count int32, @@ -666,7 +685,7 @@ func (bb *BlockBlob) List( Include: bb.listDetails, }) - listBlob, err := pager.NextPage(context.Background()) + listBlob, err := pager.NextPage(ctx) // Note: Since we make a list call with a prefix, we will not fail here for a non-existent directory. // The blob service will not validate for us whether or not the path exists. @@ -681,7 +700,7 @@ func (bb *BlockBlob) List( // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) // Since block blob does not support acls, we set mode to 0 and FlagModeDefault to true so the fuse layer can return the default permission. - blobList, dirList, err := bb.processBlobItems(listBlob.Segment.BlobItems) + blobList, dirList, err := bb.processBlobItems(ctx, listBlob.Segment.BlobItems) if err != nil { return nil, nil, err } @@ -691,7 +710,7 @@ func (bb *BlockBlob) List( // dirList contains all dirs for which we got 0 byte meta file in this iteration, so exclude those and add rest to the list // Note: Since listing is paginated, sometimes the marker file may come in a different iteration from the BlobPrefix. For such // cases we manually call GetAttr to check the existence of the marker file. - err = bb.processBlobPrefixes(listBlob.Segment.BlobPrefixes, dirList, &blobList) + err = bb.processBlobPrefixes(ctx, listBlob.Segment.BlobPrefixes, dirList, &blobList) if err != nil { return nil, nil, err } @@ -709,6 +728,7 @@ func (bb *BlockBlob) getListPath(prefix string) string { } func (bb *BlockBlob) processBlobItems( + ctx context.Context, blobItems []*container.BlobItem, ) ([]*internal.ObjAttr, map[string]bool, error) { blobList := make([]*internal.ObjAttr, 0) @@ -718,7 +738,7 @@ func (bb *BlockBlob) processBlobItems( for _, blobInfo := range blobItems { blobInfo.Name = bb.getFileName(*blobInfo.Name) - blobAttr, err := bb.getBlobAttr(blobInfo) + blobAttr, err := bb.getBlobAttr(ctx, blobInfo) if err != nil { return nil, nil, err } @@ -747,13 +767,16 @@ func (bb *BlockBlob) processBlobItems( return blobList, dirList, nil } -func (bb *BlockBlob) getBlobAttr(blobInfo *container.BlobItem) (*internal.ObjAttr, error) { +func (bb *BlockBlob) getBlobAttr( + ctx context.Context, + blobInfo *container.BlobItem, +) (*internal.ObjAttr, error) { if blobInfo.Properties.CustomerProvidedKeySHA256 != nil && *blobInfo.Properties.CustomerProvidedKeySHA256 != "" { log.Trace( "BlockBlob::List : blob is encrypted with customer provided key so fetching metadata explicitly using REST", ) - return bb.getAttrUsingRest(*blobInfo.Name) + return bb.getAttrUsingRest(ctx, *blobInfo.Name) } mode, err := bb.getFileMode(blobInfo.Properties.Permissions) if err != nil { @@ -809,6 +832,7 @@ func (bb *BlockBlob) dereferenceTime(input *time.Time, defaultTime time.Time) ti } func (bb *BlockBlob) processBlobPrefixes( + ctx context.Context, blobPrefixes []*container.BlobPrefix, dirList map[string]bool, blobList *[]*internal.ObjAttr, @@ -828,7 +852,7 @@ func (bb *BlockBlob) processBlobPrefixes( *blobList = append(*blobList, attr) } else { // marker file not found in current iteration, so we need to manually check attributes via REST - _, err := bb.getAttrUsingRest(*blobInfo.Name) + _, err := bb.getAttrUsingRest(ctx, *blobInfo.Name) // marker file also not found via manual check, safe to add to list // For HNS accounts mounted as FNS we used to list directories and files in blobfusev1, // in blobfusev2 to replicate this behaviour the below check of blobInfo.Properties != nil is added. @@ -926,7 +950,13 @@ func trackDownload(name string, bytesTransferred int64, count int64, downloadPtr } // ReadToFile : Download a blob to a local file -func (bb *BlockBlob) ReadToFile(name string, offset int64, count int64, fi *os.File) (err error) { +func (bb *BlockBlob) ReadToFile( + ctx context.Context, + name string, + offset int64, + count int64, + fi *os.File, +) (err error) { log.Trace("BlockBlob::ReadToFile : name %s, offset : %d, count %d", name, offset, count) //defer exectime.StatTimeCurrentBlock("BlockBlob::ReadToFile")() @@ -948,7 +978,7 @@ func (bb *BlockBlob) ReadToFile(name string, offset int64, count int64, fi *os.F Count: count, } - _, err = blobClient.DownloadFile(context.Background(), fi, &dlOpts) + _, err = blobClient.DownloadFile(ctx, fi, &dlOpts) if err != nil { e := storeBlobErrToErr(err) @@ -972,7 +1002,7 @@ func (bb *BlockBlob) ReadToFile(name string, offset int64, count int64, fi *os.F log.Warn("BlockBlob::ReadToFile : Failed to generate MD5 Sum for %s", name) } else { // Get latest properties from container to get the md5 of blob - prop, err := blobClient.GetProperties(context.Background(), &blob.GetPropertiesOptions{ + prop, err := blobClient.GetProperties(ctx, &blob.GetPropertiesOptions{ CPKInfo: bb.blobCPKOpt, }) if err != nil { @@ -996,11 +1026,16 @@ func (bb *BlockBlob) ReadToFile(name string, offset int64, count int64, fi *os.F } // ReadBuffer : Download a specific range from a blob to a buffer -func (bb *BlockBlob) ReadBuffer(name string, offset int64, length int64) ([]byte, error) { +func (bb *BlockBlob) ReadBuffer( + ctx context.Context, + name string, + offset int64, + length int64, +) ([]byte, error) { log.Trace("BlockBlob::ReadBuffer : name %s, offset %v, len %v", name, offset, length) var buff []byte if length == 0 { - attr, err := bb.GetAttr(name) + attr, err := bb.GetAttr(ctx, name) if err != nil { return buff, err } @@ -1016,7 +1051,7 @@ func (bb *BlockBlob) ReadBuffer(name string, offset int64, length int64) ([]byte Count: length, } - _, err := blobClient.DownloadBuffer(context.Background(), buff, &dlOpts) + _, err := blobClient.DownloadBuffer(ctx, buff, &dlOpts) if err != nil { e := storeBlobErrToErr(err) @@ -1036,6 +1071,7 @@ func (bb *BlockBlob) ReadBuffer(name string, offset int64, length int64) ([]byte // ReadInBuffer : Download specific range from a file to a user provided buffer func (bb *BlockBlob) ReadInBuffer( + ctx context.Context, name string, offset int64, length int64, @@ -1048,8 +1084,7 @@ func (bb *BlockBlob) ReadInBuffer( } blobClient := bb.getBlobClient(name) - - ctx, cancel := context.WithTimeout(context.Background(), max_context_timeout*time.Minute) + ctx, cancel := context.WithTimeout(ctx, max_context_timeout*time.Minute) defer cancel() opt := &blob.DownloadStreamOptions{ @@ -1179,6 +1214,7 @@ func trackUpload(name string, bytesTransferred int64, count int64, uploadPtr *in // WriteFromFile : Upload local file to blob func (bb *BlockBlob) WriteFromFile( + ctx context.Context, name string, metadata map[string]*string, fi *os.File, @@ -1238,7 +1274,7 @@ func (bb *BlockBlob) WriteFromFile( } } - _, err = blobClient.UploadFile(context.Background(), fi, uploadOptions) + _, err = blobClient.UploadFile(ctx, fi, uploadOptions) if err != nil { serr := storeBlobErrToErr(err) @@ -1274,13 +1310,18 @@ func (bb *BlockBlob) WriteFromFile( } // WriteFromBuffer : Upload from a buffer to a blob -func (bb *BlockBlob) WriteFromBuffer(name string, metadata map[string]*string, data []byte) error { +func (bb *BlockBlob) WriteFromBuffer( + ctx context.Context, + name string, + metadata map[string]*string, + data []byte, +) error { log.Trace("BlockBlob::WriteFromBuffer : name %s", name) blobClient := bb.getBlockBlobClient(name) defer log.TimeTrack(time.Now(), "BlockBlob::WriteFromBuffer", name) - _, err := blobClient.UploadBuffer(context.Background(), data, &blockblob.UploadBufferOptions{ + _, err := blobClient.UploadBuffer(ctx, data, &blockblob.UploadBufferOptions{ BlockSize: bb.Config.blockSize, Concurrency: bb.Config.maxConcurrency, Metadata: metadata, @@ -1300,13 +1341,16 @@ func (bb *BlockBlob) WriteFromBuffer(name string, metadata map[string]*string, d } // GetFileBlockOffsets: store blocks ids and corresponding offsets -func (bb *BlockBlob) GetFileBlockOffsets(name string) (*common.BlockOffsetList, error) { +func (bb *BlockBlob) GetFileBlockOffsets( + ctx context.Context, + name string, +) (*common.BlockOffsetList, error) { var blockOffset int64 = 0 blockList := common.BlockOffsetList{} blobClient := bb.getBlockBlobClient(name) storageBlockList, err := blobClient.GetBlockList( - context.Background(), + ctx, blockblob.BlockListTypeCommitted, nil, ) @@ -1386,6 +1430,7 @@ func (bb *BlockBlob) createNewBlocks( } func (bb *BlockBlob) removeBlocks( + ctx context.Context, blockList *common.BlockOffsetList, size int64, name string, @@ -1402,7 +1447,14 @@ func (bb *BlockBlob) removeBlocks( blk.Data = make([]byte, blk.EndIndex-blk.StartIndex) blk.Flags.Set(common.DirtyBlock) - err := bb.ReadInBuffer(name, blk.StartIndex, blk.EndIndex-blk.StartIndex, blk.Data, nil) + err := bb.ReadInBuffer( + ctx, + name, + blk.StartIndex, + blk.EndIndex-blk.StartIndex, + blk.Data, + nil, + ) if err != nil { log.Err("BlockBlob::removeBlocks : Failed to remove blocks %s [%s]", name, err.Error()) } @@ -1415,9 +1467,9 @@ func (bb *BlockBlob) removeBlocks( return blockList } -func (bb *BlockBlob) TruncateFile(name string, size int64) error { +func (bb *BlockBlob) TruncateFile(ctx context.Context, name string, size int64) error { // log.Trace("BlockBlob::TruncateFile : name=%s, size=%d", name, size) - attr, err := bb.GetAttr(name) + attr, err := bb.GetAttr(ctx, name) if err != nil { log.Err( "BlockBlob::TruncateFile : Failed to get attributes of file %s [%s]", @@ -1447,7 +1499,7 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { } data := make([]byte, blkSize) - _, err = blobClient.StageBlock(context.Background(), + _, err = blobClient.StageBlock(ctx, id, streaming.NopCloser(bytes.NewReader(data)), &blockblob.StageBlockOptions{ @@ -1466,7 +1518,7 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { size -= blkSize } - err = bb.CommitBlocks(blobName, blkList, nil) + err = bb.CommitBlocks(ctx, blobName, blkList, nil) if err != nil { log.Err( "BlockBlob::TruncateFile : Failed to commit blocks for %s [%s]", @@ -1476,7 +1528,7 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { return err } } else { - err := bb.WriteFromBuffer(name, nil, make([]byte, size)) + err := bb.WriteFromBuffer(ctx, name, nil, make([]byte, size)) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to set the %s to 0 bytes [%s]", name, err.Error()) } @@ -1486,12 +1538,12 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { //If new size is less than 256MB if size < blockblob.MaxUploadBlobBytes { - data, err := bb.HandleSmallFile(name, size, attr.Size) + data, err := bb.HandleSmallFile(ctx, name, size, attr.Size) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to read small file %s", name, err.Error()) return err } - err = bb.WriteFromBuffer(name, nil, data) + err = bb.WriteFromBuffer(ctx, name, nil, data) if err != nil { log.Err( "BlockBlob::TruncateFile : Failed to write from buffer file %s", @@ -1501,25 +1553,25 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { return err } } else { - bol, err := bb.GetFileBlockOffsets(name) + bol, err := bb.GetFileBlockOffsets(ctx, name) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to get block list of file %s [%s]", name, err.Error()) return err } if bol.SmallFile() { - data, err := bb.HandleSmallFile(name, size, attr.Size) + data, err := bb.HandleSmallFile(ctx, name, size, attr.Size) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to read small file %s", name, err.Error()) return err } - err = bb.WriteFromBuffer(name, nil, data) + err = bb.WriteFromBuffer(ctx, name, nil, data) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to write from buffer file %s", name, err.Error()) return err } } else { if size < attr.Size { - bol = bb.removeBlocks(bol, size, name) + bol = bb.removeBlocks(ctx, bol, size, name) } else if size > attr.Size { _, err = bb.createNewBlocks(bol, bol.BlockList[len(bol.BlockList)-1].EndIndex, size-attr.Size) if err != nil { @@ -1527,7 +1579,7 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { return err } } - err = bb.StageAndCommit(name, bol) + err = bb.StageAndCommit(ctx, name, bol) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to stage and commit file %s", name, err.Error()) return err @@ -1538,16 +1590,21 @@ func (bb *BlockBlob) TruncateFile(name string, size int64) error { return nil } -func (bb *BlockBlob) HandleSmallFile(name string, size int64, originalSize int64) ([]byte, error) { +func (bb *BlockBlob) HandleSmallFile( + ctx context.Context, + name string, + size int64, + originalSize int64, +) ([]byte, error) { var data = make([]byte, size) var err error if size > originalSize { - err = bb.ReadInBuffer(name, 0, 0, data, nil) + err = bb.ReadInBuffer(ctx, name, 0, 0, data, nil) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to read small file %s", name, err.Error()) } } else { - err = bb.ReadInBuffer(name, 0, size, data, nil) + err = bb.ReadInBuffer(ctx, name, 0, size, data, nil) if err != nil { log.Err("BlockBlob::TruncateFile : Failed to read small file %s", name, err.Error()) } @@ -1556,7 +1613,7 @@ func (bb *BlockBlob) HandleSmallFile(name string, size int64, originalSize int64 } // Write : write data at given offset to a blob -func (bb *BlockBlob) Write(options internal.WriteFileOptions) error { +func (bb *BlockBlob) Write(ctx context.Context, options internal.WriteFileOptions) error { name := options.Handle.Path offset := options.Offset defer log.TimeTrack(time.Now(), "BlockBlob::Write", options.Handle.Path) @@ -1564,7 +1621,7 @@ func (bb *BlockBlob) Write(options internal.WriteFileOptions) error { // tracks the case where our offset is great than our current file size (appending only - not modifying pre-existing data) var dataBuffer *[]byte // when the file offset mapping is cached we don't need to make a get block list call - fileOffsets, err := bb.GetFileBlockOffsets(name) + fileOffsets, err := bb.GetFileBlockOffsets(ctx, name) if err != nil { return err } @@ -1573,7 +1630,7 @@ func (bb *BlockBlob) Write(options internal.WriteFileOptions) error { // case 1: file consists of no blocks (small file) if fileOffsets.SmallFile() { // get all the data - oldData, _ := bb.ReadBuffer(name, 0, 0) + oldData, _ := bb.ReadBuffer(ctx, name, 0, 0) // update the data with the new data // if we're only overwriting existing data if int64(len(oldData)) >= offset+length { @@ -1599,7 +1656,7 @@ func (bb *BlockBlob) Write(options internal.WriteFileOptions) error { } } // WriteFromBuffer should be able to handle the case where now the block is too big and gets split into multiple blocks - err := bb.WriteFromBuffer(name, options.Metadata, *dataBuffer) + err := bb.WriteFromBuffer(ctx, name, options.Metadata, *dataBuffer) if err != nil { log.Err("BlockBlob::Write : Failed to upload to blob %s ", name, err.Error()) return err @@ -1622,7 +1679,7 @@ func (bb *BlockBlob) Write(options internal.WriteFileOptions) error { oldDataBuffer := make([]byte, oldDataSize+newBufferSize) if !appendOnly { // fetch the blocks that will be impacted by the new changes so we can overwrite them - err = bb.ReadInBuffer(name, fileOffsets.BlockList[index].StartIndex, oldDataSize, oldDataBuffer, nil) + err = bb.ReadInBuffer(ctx, name, fileOffsets.BlockList[index].StartIndex, oldDataSize, oldDataBuffer, nil) if err != nil { log.Err("BlockBlob::Write : Failed to read data in buffer %s [%s]", name, err.Error()) } @@ -1630,7 +1687,7 @@ func (bb *BlockBlob) Write(options internal.WriteFileOptions) error { // this gives us where the offset with respect to the buffer that holds our old data - so we can start writing the new data blockOffset := offset - fileOffsets.BlockList[index].StartIndex copy(oldDataBuffer[blockOffset:], data) - err := bb.stageAndCommitModifiedBlocks(name, oldDataBuffer, fileOffsets) + err := bb.stageAndCommitModifiedBlocks(ctx, name, oldDataBuffer, fileOffsets) return err } return nil @@ -1638,6 +1695,7 @@ func (bb *BlockBlob) Write(options internal.WriteFileOptions) error { // TODO: make a similar method facing stream that would enable us to write to cached blocks then stage and commit func (bb *BlockBlob) stageAndCommitModifiedBlocks( + ctx context.Context, name string, data []byte, offsetList *common.BlockOffsetList, @@ -1649,7 +1707,7 @@ func (bb *BlockBlob) stageAndCommitModifiedBlocks( blockIDList = append(blockIDList, blk.Id) if blk.Dirty() { _, err := blobClient.StageBlock( - context.Background(), + ctx, blk.Id, streaming.NopCloser( bytes.NewReader(data[blockOffset:(blk.EndIndex-blk.StartIndex)+blockOffset]), @@ -1671,7 +1729,7 @@ func (bb *BlockBlob) stageAndCommitModifiedBlocks( blockOffset = (blk.EndIndex - blk.StartIndex) + blockOffset } } - _, err := blobClient.CommitBlockList(context.Background(), + _, err := blobClient.CommitBlockList(ctx, blockIDList, &blockblob.CommitBlockListOptions{ HTTPHeaders: &blob.HTTPHeaders{ @@ -1692,7 +1750,11 @@ func (bb *BlockBlob) stageAndCommitModifiedBlocks( return nil } -func (bb *BlockBlob) StageAndCommit(name string, bol *common.BlockOffsetList) error { +func (bb *BlockBlob) StageAndCommit( + ctx context.Context, + name string, + bol *common.BlockOffsetList, +) error { // lock on the blob name so that no stage and commit race condition occur causing failure blobMtx := bb.blockLocks.GetLock(name) blobMtx.Lock() @@ -1710,7 +1772,7 @@ func (bb *BlockBlob) StageAndCommit(name string, bol *common.BlockOffsetList) er data = blk.Data } if blk.Dirty() { - _, err := blobClient.StageBlock(context.Background(), + _, err := blobClient.StageBlock(ctx, blk.Id, streaming.NopCloser(bytes.NewReader(data)), &blockblob.StageBlockOptions{ @@ -1733,7 +1795,7 @@ func (bb *BlockBlob) StageAndCommit(name string, bol *common.BlockOffsetList) er } } if staged { - _, err := blobClient.CommitBlockList(context.Background(), + _, err := blobClient.CommitBlockList(ctx, blockIDList, &blockblob.CommitBlockListOptions{ HTTPHeaders: &blob.HTTPHeaders{ @@ -1758,7 +1820,7 @@ func (bb *BlockBlob) StageAndCommit(name string, bol *common.BlockOffsetList) er } // ChangeMod : Change mode of a blob -func (bb *BlockBlob) ChangeMod(name string, _ os.FileMode) error { +func (bb *BlockBlob) ChangeMod(ctx context.Context, name string, _ os.FileMode) error { log.Trace("BlockBlob::ChangeMod : name %s", name) if bb.Config.ignoreAccessModifiers { @@ -1772,7 +1834,7 @@ func (bb *BlockBlob) ChangeMod(name string, _ os.FileMode) error { } // ChangeOwner : Change owner of a blob -func (bb *BlockBlob) ChangeOwner(name string, _ int, _ int) error { +func (bb *BlockBlob) ChangeOwner(ctx context.Context, name string, _ int, _ int) error { log.Trace("BlockBlob::ChangeOwner : name %s", name) if bb.Config.ignoreAccessModifiers { @@ -1786,13 +1848,16 @@ func (bb *BlockBlob) ChangeOwner(name string, _ int, _ int) error { } // GetCommittedBlockList : Get the list of committed blocks -func (bb *BlockBlob) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error) { +func (bb *BlockBlob) GetCommittedBlockList( + ctx context.Context, + name string, +) (*internal.CommittedBlockList, error) { blobClient := bb.Container.NewBlockBlobClient( common.JoinUnixFilepath(bb.Config.prefixPath, name), ) storageBlockList, err := blobClient.GetBlockList( - context.Background(), + ctx, blockblob.BlockListTypeCommitted, nil, ) @@ -1823,10 +1888,10 @@ func (bb *BlockBlob) GetCommittedBlockList(name string) (*internal.CommittedBloc } // StageBlock : stages a block and returns its blockid -func (bb *BlockBlob) StageBlock(name string, data []byte, id string) error { +func (bb *BlockBlob) StageBlock(ctx context.Context, name string, data []byte, id string) error { log.Trace("BlockBlob::StageBlock : name %s, ID %v, length %v", name, id, len(data)) - ctx, cancel := context.WithTimeout(context.Background(), max_context_timeout*time.Minute) + ctx, cancel := context.WithTimeout(ctx, max_context_timeout*time.Minute) defer cancel() blobClient := bb.Container.NewBlockBlobClient( @@ -1853,10 +1918,15 @@ func (bb *BlockBlob) StageBlock(name string, data []byte, id string) error { } // CommitBlocks : persists the block list -func (bb *BlockBlob) CommitBlocks(name string, blockList []string, newEtag *string) error { +func (bb *BlockBlob) CommitBlocks( + ctx context.Context, + name string, + blockList []string, + newEtag *string, +) error { log.Trace("BlockBlob::CommitBlocks : name %s", name) - ctx, cancel := context.WithTimeout(context.Background(), max_context_timeout*time.Minute) + ctx, cancel := context.WithTimeout(ctx, max_context_timeout*time.Minute) defer cancel() blobClient := bb.Container.NewBlockBlobClient( diff --git a/component/azstorage/block_blob_test.go b/component/azstorage/block_blob_test.go index 657d14ac9..01364990b 100644 --- a/component/azstorage/block_blob_test.go +++ b/component/azstorage/block_blob_test.go @@ -453,6 +453,37 @@ func (s *blockBlobTestSuite) TestListContainers() { s.assert.Equal(num, count) } +func (s *blockBlobTestSuite) TestCloudConnected() { + defer s.cleanupTest() + s.assert.True(s.az.CloudConnected()) +} + +func (s *blockBlobTestSuite) TestUpdateConnectionState() { + defer s.cleanupTest() + connected := s.az.updateConnectionState(&common.CloudUnreachableError{}) + s.assert.False(connected) + s.assert.False(s.az.CloudConnected()) + connected = s.az.updateConnectionState(nil) + s.assert.True(connected) + s.assert.True(s.az.CloudConnected()) +} + +func (s *blockBlobTestSuite) TestCloudOfflineCached() { + defer s.cleanupTest() + s.az.updateConnectionState(&common.CloudUnreachableError{}) + s.assert.False(s.az.CloudConnected()) + s.az.updateConnectionState(nil) +} + +func (s *blockBlobTestSuite) TestCloudOfflineContext() { + defer s.cleanupTest() + s.az.updateConnectionState(&common.CloudUnreachableError{}) + h, err := s.az.CreateFile(internal.CreateFileOptions{Name: "file" + randomString(8)}) + s.assert.Nil(h) + s.assert.True(isOfflineError(err)) + s.az.updateConnectionState(nil) +} + // TODO : ListContainersHuge: Maybe this is overkill? func checkMetadata(metadata map[string]*string, key string, val string) bool { @@ -1053,6 +1084,16 @@ func (s *blockBlobTestSuite) TestCreateFile() { name := generateFileName() h, err := s.az.CreateFile(internal.CreateFileOptions{Name: name}) + // log error information to debug log + unwrappedErr := err + for unwrappedErr != nil { + fmt.Printf( + "Uncaught AZ error is of type \"%T\" and value %v.\n", + unwrappedErr, + unwrappedErr, + ) + unwrappedErr = errors.Unwrap(unwrappedErr) + } s.assert.NoError(err) s.assert.NotNil(h) @@ -2940,7 +2981,7 @@ func (s *blockBlobTestSuite) TestFlushFileUpdateChunkedFile() { updatedBlock := make([]byte, 2*MB) rand.Read(updatedBlock) h.CacheObj.BlockOffsetList.BlockList[1].Data = make([]byte, blockSize) - s.az.storage.ReadInBuffer( + s.az.storage.ReadInBuffer(ctx, name, int64(blockSize), int64(blockSize), @@ -2993,7 +3034,7 @@ func (s *blockBlobTestSuite) TestFlushFileTruncateUpdateChunkedFile() { // truncate block h.CacheObj.BlockOffsetList.BlockList[1].Data = make([]byte, blockSize/2) h.CacheObj.BlockOffsetList.BlockList[1].EndIndex = int64(blockSize + blockSize/2) - s.az.storage.ReadInBuffer( + s.az.storage.ReadInBuffer(ctx, name, int64(blockSize), int64(blockSize)/2, @@ -3489,10 +3530,10 @@ func (s *blockBlobTestSuite) TestMD5SetOnUpload() { s.assert.Equal(blockblob.MaxUploadBlobBytes+1, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotEmpty(prop.MD5) @@ -3501,7 +3542,7 @@ func (s *blockBlobTestSuite) TestMD5SetOnUpload() { s.assert.NoError(err) s.assert.Equal(localMD5, prop.MD5) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = f.Close() _ = os.Remove(name) }) @@ -3552,14 +3593,14 @@ func (s *blockBlobTestSuite) TestMD5NotSetOnUpload() { s.assert.Equal(blockblob.MaxUploadBlobBytes+1, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.Empty(prop.MD5) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = f.Close() _ = os.Remove(name) }) @@ -3610,10 +3651,10 @@ func (s *blockBlobTestSuite) TestMD5AutoSetOnUpload() { s.assert.Equal(100, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotEmpty(prop.MD5) @@ -3622,7 +3663,7 @@ func (s *blockBlobTestSuite) TestMD5AutoSetOnUpload() { s.assert.NoError(err) s.assert.Equal(localMD5, prop.MD5) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = f.Close() _ = os.Remove(name) }) @@ -3673,7 +3714,7 @@ func (s *blockBlobTestSuite) TestInvalidateMD5PostUpload() { s.assert.Equal(100, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) blobClient := s.containerClient.NewBlobClient(name) @@ -3683,7 +3724,7 @@ func (s *blockBlobTestSuite) TestInvalidateMD5PostUpload() { nil, ) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotEmpty(prop.MD5) @@ -3692,7 +3733,7 @@ func (s *blockBlobTestSuite) TestInvalidateMD5PostUpload() { s.assert.NoError(err) s.assert.NotEqual(localMD5, prop.MD5) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = f.Close() _ = os.Remove(name) }) @@ -3743,12 +3784,12 @@ func (s *blockBlobTestSuite) TestValidateAutoMD5OnRead() { s.assert.Equal(100, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) _ = f.Close() _ = os.Remove(name) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotEmpty(prop.MD5) @@ -3756,10 +3797,10 @@ func (s *blockBlobTestSuite) TestValidateAutoMD5OnRead() { s.assert.NoError(err) s.assert.NotNil(f) - err = s.az.storage.ReadToFile(name, 0, 100, f) + err = s.az.storage.ReadToFile(ctx, name, 0, 100, f) s.assert.NoError(err) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = os.Remove(name) }) } @@ -3809,12 +3850,12 @@ func (s *blockBlobTestSuite) TestValidateManualMD5OnRead() { s.assert.Equal(blockblob.MaxUploadBlobBytes+1, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) _ = f.Close() _ = os.Remove(name) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotEmpty(prop.MD5) @@ -3822,10 +3863,10 @@ func (s *blockBlobTestSuite) TestValidateManualMD5OnRead() { s.assert.NoError(err) s.assert.NotNil(f) - err = s.az.storage.ReadToFile(name, 0, blockblob.MaxUploadBlobBytes+1, f) + err = s.az.storage.ReadToFile(ctx, name, 0, blockblob.MaxUploadBlobBytes+1, f) s.assert.NoError(err) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = os.Remove(name) }) } @@ -3875,7 +3916,7 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnRead() { s.assert.Equal(100, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) _ = f.Close() _ = os.Remove(name) @@ -3887,7 +3928,7 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnRead() { nil, ) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotEmpty(prop.MD5) @@ -3895,11 +3936,11 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnRead() { s.assert.NoError(err) s.assert.NotNil(f) - err = s.az.storage.ReadToFile(name, 0, 100, f) + err = s.az.storage.ReadToFile(ctx, name, 0, 100, f) s.assert.Error(err) s.assert.Contains(err.Error(), "md5 sum mismatch on download") - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = os.Remove(name) }) } @@ -3949,7 +3990,7 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { s.assert.Equal(100, n) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name, nil, f) + err = s.az.storage.WriteFromFile(ctx, name, nil, f) s.assert.NoError(err) _ = f.Close() _ = os.Remove(name) @@ -3961,7 +4002,7 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { nil, ) - prop, err := s.az.storage.GetAttr(name) + prop, err := s.az.storage.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotEmpty(prop.MD5) @@ -3969,10 +4010,10 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { s.assert.NoError(err) s.assert.NotNil(f) - err = s.az.storage.ReadToFile(name, 0, 100, f) + err = s.az.storage.ReadToFile(ctx, name, 0, 100, f) s.assert.NoError(err) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = os.Remove(name) }) } @@ -4008,21 +4049,21 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { // s.assert.Nil(err) // s.assert.NotNil(f) -// err = s.az.storage.ReadToFile(name, 0, int64(len(data)), f) +// err = s.az.storage.ReadToFile(ctx, name, 0, int64(len(data)), f) // s.assert.Nil(err) // fileData, err := os.ReadFile(name) // s.assert.Nil(err) // s.assert.EqualValues(data, fileData) // buf := make([]byte, len(data)) -// err = s.az.storage.ReadInBuffer(name, 0, int64(len(data)), buf, nil) +// err = s.az.storage.ReadInBuffer(ctx, name, 0, int64(len(data)), buf, nil) // s.assert.Nil(err) // s.assert.EqualValues(data, buf) // rbuf, err := s.az.storage.ReadBuffer(name, 0, int64(len(data))) // s.assert.Nil(err) // s.assert.EqualValues(data, rbuf) -// _ = s.az.storage.DeleteFile(name) +// _ = s.az.storage.DeleteFile(ctx, name) // _ = os.Remove(name) // } @@ -4052,7 +4093,7 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { // s.assert.Nil(err) // _, _ = f.Seek(0, 0) -// err = s.az.storage.WriteFromFile(name1, nil, f) +// err = s.az.storage.WriteFromFile(ctx, name1, nil, f) // s.assert.Nil(err) // file := s.containerClient.NewBlobClient(name1) @@ -4091,8 +4132,8 @@ func (s *blockBlobTestSuite) TestInvalidMD5OnReadNoVaildate() { // s.assert.Nil(err) // s.assert.NotNil(resp.RequestID) -// _ = s.az.storage.DeleteFile(name1) -// _ = s.az.storage.DeleteFile(name2) +// _ = s.az.storage.DeleteFile(ctx, name1) +// _ = s.az.storage.DeleteFile(ctx, name2) // _ = os.Remove(name1) // } @@ -4347,7 +4388,7 @@ func (s *blockBlobTestSuite) TestList() { base := generateDirectoryName() s.setupHierarchy(base) - blobList, marker, err := s.az.storage.List("", nil, 0) + blobList, marker, err := s.az.storage.List(ctx, "", nil, 0) s.assert.NoError(err) emptyString := "" s.assert.Equal(&emptyString, marker) @@ -4355,7 +4396,7 @@ func (s *blockBlobTestSuite) TestList() { s.assert.Len(blobList, 3) // Test listing with prefix - blobList, marker, err = s.az.storage.List(base+"b/", nil, 0) + blobList, marker, err = s.az.storage.List(ctx, base+"b/", nil, 0) s.assert.NoError(err) s.assert.Equal(&emptyString, marker) s.assert.NotNil(blobList) @@ -4370,7 +4411,7 @@ func (s *blockBlobTestSuite) TestList() { // s.assert.Nil(marker) // Test listing with count - blobList, marker, err = s.az.storage.List("", nil, 1) + blobList, marker, err = s.az.storage.List(ctx, "", nil, 1) s.assert.NoError(err) s.assert.NotNil(blobList) s.assert.NotEmpty(marker) diff --git a/component/azstorage/connection.go b/component/azstorage/connection.go index 294c1531e..4d6738bd5 100644 --- a/component/azstorage/connection.go +++ b/component/azstorage/connection.go @@ -26,6 +26,7 @@ package azstorage import ( + "context" "os" "github.com/Seagate/cloudfuse/common" @@ -90,47 +91,65 @@ type AzConnection interface { Configure(cfg AzStorageConfig) error UpdateConfig(cfg AzStorageConfig) error + ConnectionOkay(ctx context.Context) error SetupPipeline() error TestPipeline() error IsAccountADLS() bool - ListContainers() ([]string, error) + ListContainers(ctx context.Context) ([]string, error) // This is just for test, shall not be used otherwise SetPrefixPath(string) error - CreateFile(name string, mode os.FileMode) error - CreateDirectory(name string) error - CreateLink(source string, target string) error + CreateFile(ctx context.Context, name string, mode os.FileMode) error + CreateDirectory(ctx context.Context, name string) error + CreateLink(ctx context.Context, source string, target string) error - DeleteFile(name string) error - DeleteDirectory(name string) error + DeleteFile(ctx context.Context, name string) error + DeleteDirectory(ctx context.Context, name string) error - RenameFile(string, string, *internal.ObjAttr) error - RenameDirectory(string, string) error + RenameFile(context.Context, string, string, *internal.ObjAttr) error + RenameDirectory(context.Context, string, string) error - GetAttr(name string) (attr *internal.ObjAttr, err error) + GetAttr(ctx context.Context, name string) (attr *internal.ObjAttr, err error) // Standard operations to be supported by any account type - List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error) - - ReadToFile(name string, offset int64, count int64, fi *os.File) error - ReadBuffer(name string, offset int64, length int64) ([]byte, error) - ReadInBuffer(name string, offset int64, length int64, data []byte, etag *string) error - - WriteFromFile(name string, metadata map[string]*string, fi *os.File) error - WriteFromBuffer(name string, metadata map[string]*string, data []byte) error - Write(options internal.WriteFileOptions) error - GetFileBlockOffsets(name string) (*common.BlockOffsetList, error) - - ChangeMod(string, os.FileMode) error - ChangeOwner(string, int, int) error - TruncateFile(string, int64) error - StageAndCommit(name string, bol *common.BlockOffsetList) error - - GetCommittedBlockList(string) (*internal.CommittedBlockList, error) - StageBlock(string, []byte, string) error - CommitBlocks(string, []string, *string) error + List( + ctx context.Context, + prefix string, + marker *string, + count int32, + ) ([]*internal.ObjAttr, *string, error) + + ReadToFile(ctx context.Context, name string, offset int64, count int64, fi *os.File) error + ReadBuffer(ctx context.Context, name string, offset int64, length int64) ([]byte, error) + ReadInBuffer( + ctx context.Context, + name string, + offset int64, + length int64, + data []byte, + etag *string, + ) error + + WriteFromFile(ctx context.Context, name string, metadata map[string]*string, fi *os.File) error + WriteFromBuffer( + ctx context.Context, + name string, + metadata map[string]*string, + data []byte, + ) error + Write(ctx context.Context, options internal.WriteFileOptions) error + GetFileBlockOffsets(ctx context.Context, name string) (*common.BlockOffsetList, error) + + ChangeMod(context.Context, string, os.FileMode) error + ChangeOwner(context.Context, string, int, int) error + TruncateFile(context.Context, string, int64) error + StageAndCommit(ctx context.Context, name string, bol *common.BlockOffsetList) error + + GetCommittedBlockList(context.Context, string) (*internal.CommittedBlockList, error) + StageBlock(context.Context, string, []byte, string) error + CommitBlocks(context.Context, string, []string, *string) error UpdateServiceClient(_, _ string) error diff --git a/component/azstorage/datalake.go b/component/azstorage/datalake.go index 4e0434b7e..4a72912e4 100644 --- a/component/azstorage/datalake.go +++ b/component/azstorage/datalake.go @@ -217,9 +217,15 @@ func (dl *Datalake) IsAccountADLS() bool { return dl.BlockBlob.IsAccountADLS() } -func (dl *Datalake) ListContainers() ([]string, error) { +// check the connection to the service by calling GetProperties on the container +func (dl *Datalake) ConnectionOkay(ctx context.Context) error { + log.Trace("BlockBlob::ConnectionOkay : checking connection to cloud service") + return dl.BlockBlob.ConnectionOkay(ctx) +} + +func (dl *Datalake) ListContainers(ctx context.Context) ([]string, error) { log.Trace("Datalake::ListContainers : Listing containers") - return dl.BlockBlob.ListContainers() + return dl.BlockBlob.ListContainers(ctx) } func (dl *Datalake) SetPrefixPath(path string) error { @@ -229,14 +235,14 @@ func (dl *Datalake) SetPrefixPath(path string) error { } // CreateFile : Create a new file in the filesystem/directory -func (dl *Datalake) CreateFile(name string, mode os.FileMode) error { +func (dl *Datalake) CreateFile(ctx context.Context, name string, mode os.FileMode) error { log.Trace("Datalake::CreateFile : name %s", name) - err := dl.BlockBlob.CreateFile(name, mode) + err := dl.BlockBlob.CreateFile(ctx, name, mode) if err != nil { log.Err("Datalake::CreateFile : Failed to create file %s [%s]", name, err.Error()) return err } - err = dl.ChangeMod(name, mode) + err = dl.ChangeMod(ctx, name, mode) if err != nil { log.Err( "Datalake::CreateFile : Failed to set permissions on file %s [%s]", @@ -250,11 +256,11 @@ func (dl *Datalake) CreateFile(name string, mode os.FileMode) error { } // CreateDirectory : Create a new directory in the filesystem/directory -func (dl *Datalake) CreateDirectory(name string) error { +func (dl *Datalake) CreateDirectory(ctx context.Context, name string) error { log.Trace("Datalake::CreateDirectory : name %s", name) directoryURL := dl.getDirectoryClient(name) - _, err := directoryURL.Create(context.Background(), &directory.CreateOptions{ + _, err := directoryURL.Create(ctx, &directory.CreateOptions{ CPKInfo: dl.datalakeCPKOpt, AccessConditions: &directory.AccessConditions{ ModifiedAccessConditions: &directory.ModifiedAccessConditions{ @@ -294,16 +300,16 @@ func (dl *Datalake) CreateDirectory(name string) error { } // CreateLink : Create a symlink in the filesystem/directory -func (dl *Datalake) CreateLink(source string, target string) error { +func (dl *Datalake) CreateLink(ctx context.Context, source string, target string) error { log.Trace("Datalake::CreateLink : %s -> %s", source, target) - return dl.BlockBlob.CreateLink(source, target) + return dl.BlockBlob.CreateLink(ctx, source, target) } // DeleteFile : Delete a file in the filesystem/directory -func (dl *Datalake) DeleteFile(name string) (err error) { +func (dl *Datalake) DeleteFile(ctx context.Context, name string) (err error) { log.Trace("Datalake::DeleteFile : name %s", name) fileClient := dl.getFileClient(name) - _, err = fileClient.Delete(context.Background(), nil) + _, err = fileClient.Delete(ctx, nil) if err != nil { serr := storeDatalakeErrToErr(err) switch serr { @@ -330,11 +336,11 @@ func (dl *Datalake) DeleteFile(name string) (err error) { } // DeleteDirectory : Delete a directory in the filesystem/directory -func (dl *Datalake) DeleteDirectory(name string) (err error) { +func (dl *Datalake) DeleteDirectory(ctx context.Context, name string) (err error) { log.Trace("Datalake::DeleteDirectory : name %s", name) directoryClient := dl.getDirectoryClient(name) - _, err = directoryClient.Delete(context.Background(), nil) + _, err = directoryClient.Delete(ctx, nil) // TODO : There is an ability to pass a continuation token here for recursive delete, should we implement this logic to follow continuation token? The SDK does not currently do this. if err != nil { serr := storeDatalakeErrToErr(err) @@ -353,13 +359,18 @@ func (dl *Datalake) DeleteDirectory(name string) (err error) { // RenameFile : Rename the file // While renaming the file, Creation time is preserved but LMT is changed for the destination blob. // and also Etag of the destination blob changes -func (dl *Datalake) RenameFile(source string, target string, srcAttr *internal.ObjAttr) error { +func (dl *Datalake) RenameFile( + ctx context.Context, + source string, + target string, + srcAttr *internal.ObjAttr, +) error { log.Trace("Datalake::RenameFile : %s -> %s", source, target) fileClient := dl.getFileClientPathEscape(source) renameResponse, err := fileClient.Rename( - context.Background(), + ctx, dl.getFormattedPath(target), &file.RenameOptions{ CPKInfo: dl.datalakeCPKOpt, @@ -380,12 +391,12 @@ func (dl *Datalake) RenameFile(source string, target string, srcAttr *internal.O } // RenameDirectory : Rename the directory -func (dl *Datalake) RenameDirectory(source string, target string) error { +func (dl *Datalake) RenameDirectory(ctx context.Context, source string, target string) error { log.Trace("Datalake::RenameDirectory : %s -> %s", source, target) directoryClient := dl.getDirectoryClientPathEscape(source) _, err := directoryClient.Rename( - context.Background(), + ctx, dl.getFormattedPath(target), &directory.RenameOptions{ CPKInfo: dl.datalakeCPKOpt, @@ -406,11 +417,14 @@ func (dl *Datalake) RenameDirectory(source string, target string) error { } // GetAttr : Retrieve attributes of the path -func (dl *Datalake) GetAttr(name string) (blobAttr *internal.ObjAttr, err error) { +func (dl *Datalake) GetAttr( + ctx context.Context, + name string, +) (blobAttr *internal.ObjAttr, err error) { log.Trace("Datalake::GetAttr : name %s", name) fileClient := dl.getFileClient(name) - prop, err := fileClient.GetProperties(context.Background(), &file.GetPropertiesOptions{ + prop, err := fileClient.GetProperties(ctx, &file.GetPropertiesOptions{ CPKInfo: dl.datalakeCPKOpt, }) if err != nil { @@ -457,7 +471,7 @@ func (dl *Datalake) GetAttr(name string) (blobAttr *internal.ObjAttr, err error) } if dl.Config.honourACL && dl.Config.authConfig.ObjectID != "" { - acl, err := fileClient.GetAccessControl(context.Background(), nil) + acl, err := fileClient.GetAccessControl(ctx, nil) if err != nil { // Just ignore the error here as rest of the attributes have been retrieved log.Err("Datalake::GetAttr : Failed to get ACL for %s [%s]", name, err.Error()) @@ -489,36 +503,50 @@ func (dl *Datalake) GetAttr(name string) (blobAttr *internal.ObjAttr, err error) // This fetches the list using a marker so the caller code should handle marker logic // If count=0 - fetch max entries func (dl *Datalake) List( + ctx context.Context, prefix string, marker *string, count int32, ) ([]*internal.ObjAttr, *string, error) { - return dl.BlockBlob.List(prefix, marker, count) + return dl.BlockBlob.List(ctx, prefix, marker, count) } // ReadToFile : Download a file to a local file -func (dl *Datalake) ReadToFile(name string, offset int64, count int64, fi *os.File) (err error) { - return dl.BlockBlob.ReadToFile(name, offset, count, fi) +func (dl *Datalake) ReadToFile( + ctx context.Context, + name string, + offset int64, + count int64, + fi *os.File, +) (err error) { + return dl.BlockBlob.ReadToFile(ctx, name, offset, count, fi) } // ReadBuffer : Download a specific range from a file to a buffer -func (dl *Datalake) ReadBuffer(name string, offset int64, length int64) ([]byte, error) { - return dl.BlockBlob.ReadBuffer(name, offset, length) +func (dl *Datalake) ReadBuffer( + ctx context.Context, + name string, + offset int64, + length int64, +) ([]byte, error) { + return dl.BlockBlob.ReadBuffer(ctx, name, offset, length) } // ReadInBuffer : Download specific range from a file to a user provided buffer func (dl *Datalake) ReadInBuffer( + ctx context.Context, name string, offset int64, length int64, data []byte, etag *string, ) error { - return dl.BlockBlob.ReadInBuffer(name, offset, length, data, etag) + return dl.BlockBlob.ReadInBuffer(ctx, name, offset, length, data, etag) } // WriteFromFile : Upload local file to file func (dl *Datalake) WriteFromFile( + ctx context.Context, name string, metadata map[string]*string, fi *os.File, @@ -531,7 +559,7 @@ func (dl *Datalake) WriteFromFile( if dl.Config.preserveACL { fileClient = dl.Filesystem.NewFileClient(filepath.Join(dl.Config.prefixPath, name)) - resp, err := fileClient.GetAccessControl(context.Background(), nil) + resp, err := fileClient.GetAccessControl(ctx, nil) if err != nil { log.Err("Datalake::getACL : Failed to get ACLs for file %s [%s]", name, err.Error()) } else if resp.ACL != nil { @@ -540,13 +568,13 @@ func (dl *Datalake) WriteFromFile( } // Upload the file, which will override the permissions and ACL - retCode := dl.BlockBlob.WriteFromFile(name, metadata, fi) + retCode := dl.BlockBlob.WriteFromFile(ctx, name, metadata, fi) if acl != "" { // Cannot set both permissions and ACL in one call. ACL includes permission as well so just setting those back // Just setting up the permissions will delete existing ACLs applied on the blob so do not convert this code to // just set the permissions. - _, err := fileClient.SetAccessControl(context.Background(), &file.SetAccessControlOptions{ + _, err := fileClient.SetAccessControl(ctx, &file.SetAccessControlOptions{ ACL: &acl, }) @@ -560,29 +588,41 @@ func (dl *Datalake) WriteFromFile( } // WriteFromBuffer : Upload from a buffer to a file -func (dl *Datalake) WriteFromBuffer(name string, metadata map[string]*string, data []byte) error { - return dl.BlockBlob.WriteFromBuffer(name, metadata, data) +func (dl *Datalake) WriteFromBuffer( + ctx context.Context, + name string, + metadata map[string]*string, + data []byte, +) error { + return dl.BlockBlob.WriteFromBuffer(ctx, name, metadata, data) } // Write : Write to a file at given offset -func (dl *Datalake) Write(options internal.WriteFileOptions) error { - return dl.BlockBlob.Write(options) +func (dl *Datalake) Write(ctx context.Context, options internal.WriteFileOptions) error { + return dl.BlockBlob.Write(ctx, options) } -func (dl *Datalake) StageAndCommit(name string, bol *common.BlockOffsetList) error { - return dl.BlockBlob.StageAndCommit(name, bol) +func (dl *Datalake) StageAndCommit( + ctx context.Context, + name string, + bol *common.BlockOffsetList, +) error { + return dl.BlockBlob.StageAndCommit(ctx, name, bol) } -func (dl *Datalake) GetFileBlockOffsets(name string) (*common.BlockOffsetList, error) { - return dl.BlockBlob.GetFileBlockOffsets(name) +func (dl *Datalake) GetFileBlockOffsets( + ctx context.Context, + name string, +) (*common.BlockOffsetList, error) { + return dl.BlockBlob.GetFileBlockOffsets(ctx, name) } -func (dl *Datalake) TruncateFile(name string, size int64) error { - return dl.BlockBlob.TruncateFile(name, size) +func (dl *Datalake) TruncateFile(ctx context.Context, name string, size int64) error { + return dl.BlockBlob.TruncateFile(ctx, name, size) } // ChangeMod : Change mode of a path -func (dl *Datalake) ChangeMod(name string, mode os.FileMode) error { +func (dl *Datalake) ChangeMod(ctx context.Context, name string, mode os.FileMode) error { log.Trace("Datalake::ChangeMod : Change mode of file %s to %s", name, mode) fileClient := dl.getFileClient(name) @@ -591,7 +631,7 @@ func (dl *Datalake) ChangeMod(name string, mode os.FileMode) error { // and create new string with the username included in the string // Keeping this code here so in future if its required we can get the string and manipulate - currPerm, err := fileURL.getACL(context.Background()) + currPerm, err := fileURL.getACL(ctx) e := storeDatalakeErrToErr(err) if e == ErrFileNotFound { return syscall.ENOENT @@ -602,7 +642,7 @@ func (dl *Datalake) ChangeMod(name string, mode os.FileMode) error { */ newPerm := getACLPermissions(mode) - _, err := fileClient.SetAccessControl(context.Background(), &file.SetAccessControlOptions{ + _, err := fileClient.SetAccessControl(ctx, &file.SetAccessControlOptions{ Permissions: &newPerm, }) if err != nil { @@ -627,7 +667,7 @@ func (dl *Datalake) ChangeMod(name string, mode os.FileMode) error { } // ChangeOwner : Change owner of a path -func (dl *Datalake) ChangeOwner(name string, _ int, _ int) error { +func (dl *Datalake) ChangeOwner(ctx context.Context, name string, _ int, _ int) error { log.Trace("Datalake::ChangeOwner : name %s", name) if dl.Config.ignoreAccessModifiers { @@ -640,7 +680,7 @@ func (dl *Datalake) ChangeOwner(name string, _ int, _ int) error { // fileURL := dl.Filesystem.NewRootDirectoryURL().NewFileURL(common.JoinUnixFilepath(dl.Config.prefixPath, name)) // group := strconv.Itoa(gid) // owner := strconv.Itoa(uid) - // _, err := fileURL.SetAccessControl(context.Background(), azbfs.BlobFSAccessControl{Group: group, Owner: owner}) + // _, err := fileURL.SetAccessControl(ctx, azbfs.BlobFSAccessControl{Group: group, Owner: owner}) // e := storeDatalakeErrToErr(err) // if e == ErrFileNotFound { // return syscall.ENOENT @@ -652,18 +692,26 @@ func (dl *Datalake) ChangeOwner(name string, _ int, _ int) error { } // GetCommittedBlockList : Get the list of committed blocks -func (dl *Datalake) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error) { - return dl.BlockBlob.GetCommittedBlockList(name) +func (dl *Datalake) GetCommittedBlockList( + ctx context.Context, + name string, +) (*internal.CommittedBlockList, error) { + return dl.BlockBlob.GetCommittedBlockList(ctx, name) } // StageBlock : stages a block and returns its blockid -func (dl *Datalake) StageBlock(name string, data []byte, id string) error { - return dl.BlockBlob.StageBlock(name, data, id) +func (dl *Datalake) StageBlock(ctx context.Context, name string, data []byte, id string) error { + return dl.BlockBlob.StageBlock(ctx, name, data, id) } // CommitBlocks : persists the block list -func (dl *Datalake) CommitBlocks(name string, blockList []string, newEtag *string) error { - return dl.BlockBlob.CommitBlocks(name, blockList, newEtag) +func (dl *Datalake) CommitBlocks( + ctx context.Context, + name string, + blockList []string, + newEtag *string, +) error { + return dl.BlockBlob.CommitBlocks(ctx, name, blockList, newEtag) } func (dl *Datalake) SetFilter(filter string) error { diff --git a/component/azstorage/datalake_test.go b/component/azstorage/datalake_test.go index 192149080..fb5b3af3c 100644 --- a/component/azstorage/datalake_test.go +++ b/component/azstorage/datalake_test.go @@ -2503,6 +2503,7 @@ func (s *datalakeTestSuite) TestFlushFileUpdateChunkedFile() { rand.Read(updatedBlock) h.CacheObj.BlockOffsetList.BlockList[1].Data = make([]byte, blockSize) s.az.storage.ReadInBuffer( + ctx, name, int64(blockSize), int64(blockSize), @@ -2558,6 +2559,7 @@ func (s *datalakeTestSuite) TestFlushFileTruncateUpdateChunkedFile() { h.CacheObj.BlockOffsetList.BlockList[1].Data = make([]byte, blockSize/2) h.CacheObj.BlockOffsetList.BlockList[1].EndIndex = int64(blockSize + blockSize/2) s.az.storage.ReadInBuffer( + ctx, name, int64(blockSize), int64(blockSize)/2, @@ -3060,21 +3062,21 @@ func (s *datalakeTestSuite) TestDownloadWithCPKEnabled() { s.assert.NoError(err) s.assert.NotNil(f) - err = s.az.storage.ReadToFile(name, 0, int64(len(data)), f) + err = s.az.storage.ReadToFile(ctx, name, 0, int64(len(data)), f) s.assert.NoError(err) fileData, err := os.ReadFile(name) s.assert.NoError(err) s.assert.Equal(data, fileData) buf := make([]byte, len(data)) - err = s.az.storage.ReadInBuffer(name, 0, int64(len(data)), buf, nil) + err = s.az.storage.ReadInBuffer(ctx, name, 0, int64(len(data)), buf, nil) s.assert.NoError(err) s.assert.Equal(data, buf) - rbuf, err := s.az.storage.ReadBuffer(name, 0, int64(len(data))) + rbuf, err := s.az.storage.ReadBuffer(ctx, name, 0, int64(len(data))) s.assert.NoError(err) s.assert.Equal(data, rbuf) - _ = s.az.storage.DeleteFile(name) + _ = s.az.storage.DeleteFile(ctx, name) _ = os.Remove(name) } @@ -3111,12 +3113,12 @@ func (s *datalakeTestSuite) TestUploadWithCPKEnabled() { s.assert.NoError(err) _, _ = f.Seek(0, 0) - err = s.az.storage.WriteFromFile(name1, nil, f) + err = s.az.storage.WriteFromFile(ctx, name1, nil, f) s.assert.NoError(err) // Blob should have updated data fileClient := s.containerClient.NewFileClient(name1) - attr, err := s.az.storage.(*Datalake).GetAttr(name1) + attr, err := s.az.storage.(*Datalake).GetAttr(ctx, name1) s.assert.NoError(err) s.assert.NotNil(attr) @@ -3134,7 +3136,7 @@ func (s *datalakeTestSuite) TestUploadWithCPKEnabled() { s.assert.NotNil(resp.RequestID) name2 := generateFileName() - err = s.az.storage.WriteFromBuffer(name2, nil, data) + err = s.az.storage.WriteFromBuffer(ctx, name2, nil, data) s.assert.NoError(err) fileClient = s.containerClient.NewFileClient(name2) @@ -3151,8 +3153,8 @@ func (s *datalakeTestSuite) TestUploadWithCPKEnabled() { s.assert.NoError(err) s.assert.NotNil(resp.RequestID) - _ = s.az.storage.DeleteFile(name1) - _ = s.az.storage.DeleteFile(name2) + _ = s.az.storage.DeleteFile(ctx, name1) + _ = s.az.storage.DeleteFile(ctx, name2) _ = os.Remove(name1) } @@ -3398,7 +3400,7 @@ func (s *datalakeTestSuite) TestList() { base := generateDirectoryName() s.setupHierarchy(base) - blobList, marker, err := s.az.storage.List(base, nil, 0) + blobList, marker, err := s.az.storage.List(ctx, base, nil, 0) s.assert.NoError(err) emptyString := "" s.assert.Equal(&emptyString, marker) @@ -3407,7 +3409,7 @@ func (s *datalakeTestSuite) TestList() { s.assert.NotEqual(0, blobList[0].Mode) // Test listing with prefix - blobList, marker, err = s.az.storage.List(base+"b/", nil, 0) + blobList, marker, err = s.az.storage.List(ctx, base+"b/", nil, 0) s.assert.NoError(err) s.assert.Equal(&emptyString, marker) s.assert.NotNil(blobList) @@ -3416,13 +3418,13 @@ func (s *datalakeTestSuite) TestList() { s.assert.NotEqual(0, blobList[0].Mode) // Test listing with marker - blobList, marker, err = s.az.storage.List(base, to.Ptr("invalid-marker"), 0) + blobList, marker, err = s.az.storage.List(ctx, base, to.Ptr("invalid-marker"), 0) s.assert.Error(err) s.assert.Empty(blobList) s.assert.Nil(marker) // Test listing with count - blobList, marker, err = s.az.storage.List("", nil, 1) + blobList, marker, err = s.az.storage.List(ctx, "", nil, 1) s.assert.NoError(err) s.assert.NotNil(blobList) s.assert.NotEmpty(marker) diff --git a/component/file_cache/async.go b/component/file_cache/async.go new file mode 100644 index 000000000..ada6eafb2 --- /dev/null +++ b/component/file_cache/async.go @@ -0,0 +1,355 @@ +/* + Licensed under the MIT License . + + Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +package file_cache + +import ( + "context" + "errors" + "os" + "path/filepath" + "time" + + "github.com/Seagate/cloudfuse/common" + "github.com/Seagate/cloudfuse/common/config" + "github.com/Seagate/cloudfuse/common/log" + "github.com/Seagate/cloudfuse/internal" + "github.com/Seagate/cloudfuse/internal/handlemap" + "github.com/robfig/cron/v3" +) + +type UploadWindow struct { + name string `yaml:"name"` + cronExpr string `yaml:"cron"` + duration time.Duration + cronEntryID int +} + +type Config struct { + Schedule WeeklySchedule `yaml:"schedule"` +} + +type WeeklySchedule []UploadWindow + +func (fc *FileCache) configureScheduler() error { + // load from config + var rawSchedule []map[string]interface{} + err := config.UnmarshalKey(compName+".schedule", &rawSchedule) + if err != nil { + return err + } + // initialize the scheduler + fc.cronScheduler = cron.New(cron.WithSeconds()) + // Convert raw schedule to WeeklySchedule + fc.schedule = make(WeeklySchedule, 0, len(rawSchedule)) + for _, rawWindow := range rawSchedule { + window := UploadWindow{} + if name, ok := rawWindow["name"].(string); ok { + window.name = name + } + if cronExpr, ok := rawWindow["cron"].(string); ok { + window.cronExpr = cronExpr + } + if duration, ok := rawWindow["duration"].(string); ok { + window.duration, err = time.ParseDuration(duration) + if err != nil { + log.Err( + "FileCache::Configure : %s invalid window duration %s (%v)", + window.name, + duration, + err, + ) + return err + } + } + // load schedules into cron + var initialWindowEndTime time.Time + entryId, err := fc.cronScheduler.AddFunc(window.cronExpr, func() { + // Is this a transition from inactive? + windowCount := fc.activeWindows.Add(1) + if windowCount == 1 { + // transition to active - open the window + fc.schedulerActiveCh = make(chan struct{}) + log.Info( + "FileCache::SchedulerCronFunc : %s - enabled scheduled uploads", + window.name, + ) + } + log.Info( + "FileCache::SchedulerCronFunc : %s (%s) started (numActive=%d)", + window.name, + window.cronExpr, + windowCount, + ) + // When should the window close? + remainingDuration := window.duration + currentTime := time.Now() + if initialWindowEndTime.After(currentTime) { + remainingDuration = initialWindowEndTime.Sub(currentTime) + } + // Create a context to end the window + ctx, cancel := context.WithTimeout(context.Background(), remainingDuration) + defer cancel() + for { + select { + case <-fc.componentStopping: + log.Info("FileCache::SchedulerCronFunc : %s - stopping cron job", window.name) + return + case <-ctx.Done(): + // Window has completed, update active window count + windowCount = fc.activeWindows.Add(-1) + log.Info( + "FileCache::SchedulerCronFunc : %s (%s) ended (numActive=%d)", + window.name, + window.duration, + windowCount, + ) + // Only close resources when the last window ends + if windowCount == 0 { + close(fc.schedulerActiveCh) + log.Info( + "FileCache::SchedulerCronFunc : %s - disabled scheduled uploads", + window.name, + ) + } + return + } + } + }) + if err != nil { + log.Err( + "FileCache::Configure : Schedule %s invalid cron expression (%v)", + window.name, + err, + ) + return err + } + // calculate end time for windows that will start open + now := time.Now() + entry := fc.cronScheduler.Entry(entryId) + for t := entry.Schedule.Next(now.Add(-window.duration)); now.After(t); t = entry.Schedule.Next(t) { + initialWindowEndTime = t.Add(window.duration) + } + // save window to fc.schedule + window.cronEntryID = int(entryId) + fc.schedule = append(fc.schedule, window) + log.Info( + "FileCache::Configure : Added schedule %s ('%s', %s)", + window.name, + window.cronExpr, + window.duration, + ) + } + + return nil +} + +func (fc *FileCache) startScheduler() { + // check if any schedules should already be active + for _, window := range fc.schedule { + entry := fc.cronScheduler.Entry(cron.EntryID(window.cronEntryID)) + // check if this entry should already be active + // did this entry have a start time within the last duration? + now := time.Now() + var initialWindowEndTime time.Time + for t := entry.Schedule.Next(now.Add(-window.duration)); now.After(t); t = entry.Schedule.Next(t) { + initialWindowEndTime = t.Add(window.duration) + } + if !initialWindowEndTime.IsZero() { + go entry.Job.Run() + } + } + fc.cronScheduler.Start() +} + +func (fc *FileCache) addPendingOp(name string, flock *common.LockMapItem) { + log.Trace("FileCache::addPendingOp : %s", name) + fc.pendingOps.Store(name, struct{}{}) + select { + case fc.pendingOpAdded <- struct{}{}: + default: // do not block + } +} + +func (fc *FileCache) servicePendingOps() { + for { + select { + case <-fc.componentStopping: + log.Crit("FileCache::servicePendingOps : Stopping") + // TODO: Persist pending ops + return + case <-fc.schedulerActiveCh: + // upload schedule is not active, wait before checking again + select { + case <-time.After(time.Second): + case <-fc.componentStopping: + } + default: + // check if we're connected + if !fc.NextComponent().CloudConnected() { + // we are offline, wait for a while before checking again + select { + case <-time.After(time.Second): + case <-fc.componentStopping: + } + break + } + numFilesProcessed := 0 + // Iterate over pending ops + fc.pendingOps.Range(func(key, value interface{}) bool { + numFilesProcessed++ + select { + case <-fc.componentStopping: + return false + case <-fc.schedulerActiveCh: + return false // upload window ended + default: + path := key.(string) + err := fc.uploadPendingFile(path) + if isOffline(err) { + return false // connection lost - abort iteration + } + if err != nil { + log.Err("FileCache::servicePendingOps : %s upload failed: %v", path, err) + } + } + return true // Continue the iteration + }) + log.Info( + "FileCache::servicePendingOps : Completed upload cycle, processed %d files", + numFilesProcessed, + ) + if numFilesProcessed == 0 { + // we're online but there's nothing to do + // wait for a task to be added + select { + case <-fc.pendingOpAdded: + case <-fc.componentStopping: + } + } + } + } +} + +func (fc *FileCache) uploadPendingFile(name string) error { + log.Trace("FileCache::uploadPendingFile : %s", name) + + // lock the file + flock := fc.fileLocks.Get(name) + flock.Lock() + defer flock.Unlock() + + // don't double upload + _, stillPending := fc.pendingOps.Load(name) + if !stillPending { + return nil + } + + // look up file (or folder!) + localPath := filepath.Join(fc.tmpPath, name) + info, err := os.Stat(localPath) + if err != nil { + log.Err("FileCache::uploadPendingFile : %s failed to stat file. Here's why: %v", name, err) + return err + } + if info.IsDir() { + // upload folder + options := internal.CreateDirOptions{Name: name, Mode: info.Mode()} + err = fc.NextComponent().CreateDir(options) + if err != nil && !os.IsExist(err) { + return err + } + } else { + // this is a file + // prepare a handle + handle := handlemap.NewHandle(name) + // open the cached file + f, err := common.OpenFile(localPath, os.O_RDONLY, fc.defaultPermission) + if err != nil { + log.Err("FileCache::uploadPendingFile : %s failed to open file. Here's why: %v", name, err) + return err + } + // write handle attributes + inf, err := f.Stat() + if err == nil { + handle.Size = inf.Size() + } + handle.UnixFD = uint64(f.Fd()) + handle.SetFileObject(f) + handle.Flags.Set(handlemap.HandleFlagDirty) + + // upload the file + err = fc.flushFileInternal(internal.FlushFileOptions{Handle: handle, AsyncUpload: true}) + f.Close() + if err != nil { + log.Err("FileCache::uploadPendingFile : %s Upload failed. Here's why: %v", name, err) + return err + } + } + // update state + log.Info("FileCache::uploadPendingFile : File uploaded: %s", name) + fc.pendingOps.Delete(name) + + return nil +} + +func (fc *FileCache) IsScheduled(objName string) bool { + _, inSchedule := fc.pendingOps.Load(objName) + return inSchedule +} + +// this returns true when offline access is enabled, and it's safe to access this object offline +func (fc *FileCache) offlineOperationAllowed(name string) bool { + return fc.offlineAccess && fc.notInCloud(name) +} + +// returns true if we *know* that this entity does not exist in cloud storage +// otherwise returns false (including ambiguous cases) +func (fc *FileCache) notInCloud(name string) bool { + notInCloud, _ := fc.checkCloud(name) + return notInCloud +} + +// notInCloud is true if we *know* that this entity does not exist in cloud storage +// and getAttrErr is the error returned from GetAttr +func (fc *FileCache) checkCloud(name string) (notInCloud bool, getAttrErr error) { + _, getAttrErr = fc.NextComponent().GetAttr(internal.GetAttrOptions{Name: name}) + notInCloud = errors.Is(getAttrErr, os.ErrNotExist) + return notInCloud, getAttrErr +} + +// checks if the error returned from cloud storage means we're offline +func isOffline(err error) bool { + return errors.Is(err, &common.CloudUnreachableError{}) +} + +// checks whether we have usable metadata, despite being offline +func offlineDataAvailable(err error) bool { + return isOffline(err) && cachedData(err) +} + +// checks whether we have usable metadata, despite being offline +func cachedData(err error) bool { + return !errors.Is(err, &common.NoCachedDataError{}) || !isOffline(err) +} diff --git a/component/file_cache/cache_policy.go b/component/file_cache/cache_policy.go index 97870deec..9a57101e1 100644 --- a/component/file_cache/cache_policy.go +++ b/component/file_cache/cache_policy.go @@ -28,6 +28,7 @@ package file_cache import ( "fmt" "os" + "sync" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/log" @@ -45,7 +46,8 @@ type cachePolicyConfig struct { highThreshold float64 lowThreshold float64 - fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) + fileLocks *common.LockMap // uses object name (common.JoinUnixFilepath) + pendingOps *sync.Map policyTrace bool } diff --git a/component/file_cache/file_cache.go b/component/file_cache/file_cache.go index 640554b74..e2385ccc4 100644 --- a/component/file_cache/file_cache.go +++ b/component/file_cache/file_cache.go @@ -27,6 +27,7 @@ package file_cache import ( "context" + "errors" "fmt" "io" "io/fs" @@ -36,6 +37,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -45,6 +47,7 @@ import ( "github.com/Seagate/cloudfuse/internal" "github.com/Seagate/cloudfuse/internal/handlemap" "github.com/Seagate/cloudfuse/internal/stats_manager" + "github.com/robfig/cron/v3" ) // Common structure for Component @@ -59,11 +62,13 @@ type FileCache struct { allowNonEmpty bool cacheTimeout float64 policyTrace bool - missedChmodList sync.Map // uses object name (common.JoinUnixFilepath) - mountPath string // uses os.Separator (filepath.Join) - scheduleOps sync.Map // uses object name (common.JoinUnixFilepath) + missedChmodList sync.Map // uses object name (common.JoinUnixFilepath) + pendingOps sync.Map // uses object name (common.JoinUnixFilepath) + pendingOpAdded chan struct{} // signals when an offline operation is queued + mountPath string // uses os.Separator (filepath.Join) allowOther bool offloadIO bool + offlineAccess bool syncToFlush bool syncToDelete bool maxCacheSize float64 @@ -77,13 +82,11 @@ type FileCache struct { lazyWrite bool fileCloseOpt sync.WaitGroup - stopAsyncUpload chan struct{} - schedule WeeklySchedule - uploadNotifyCh chan struct{} - alwaysOn bool - activeWindows int - activeWindowsMutex *sync.Mutex - closeWindowCh chan struct{} + componentStopping chan struct{} + schedule WeeklySchedule + cronScheduler *cron.Cron + activeWindows atomic.Int32 + schedulerActiveCh chan struct{} } // Structure defining your config parameters @@ -103,8 +106,9 @@ type FileCacheOptions struct { AllowNonEmpty bool `config:"allow-non-empty-temp" yaml:"allow-non-empty-temp,omitempty"` CleanupOnStart bool `config:"cleanup-on-start" yaml:"cleanup-on-start,omitempty"` - EnablePolicyTrace bool `config:"policy-trace" yaml:"policy-trace,omitempty"` - OffloadIO bool `config:"offload-io" yaml:"offload-io,omitempty"` + BlockOfflineAccess bool `config:"block-offline-access" yaml:"block-offline-access,omitempty"` + EnablePolicyTrace bool `config:"policy-trace" yaml:"policy-trace,omitempty"` + OffloadIO bool `config:"offload-io" yaml:"offload-io,omitempty"` SyncToFlush bool `config:"sync-to-flush" yaml:"sync-to-flush"` SyncNoOp bool `config:"ignore-sync" yaml:"ignore-sync,omitempty"` @@ -180,11 +184,16 @@ func (fc *FileCache) Start(ctx context.Context) error { fileCacheStatsCollector = stats_manager.NewStatsCollector(fc.Name()) log.Debug("Starting file cache stats collector") - fc.uploadNotifyCh = make(chan struct{}, 1) - err = fc.SetupScheduler() - if err != nil { - log.Warn("FileCache::Start : Failed to setup scheduler [%s]", err.Error()) + // setup async uploads + fc.componentStopping = make(chan struct{}) + fc.schedulerActiveCh = make(chan struct{}) + fc.pendingOpAdded = make(chan struct{}, 1) + if len(fc.schedule) > 0 { + log.Info("FileCache::Start : Scheduler enabled") + close(fc.schedulerActiveCh) + fc.startScheduler() } + go fc.servicePendingOps() return nil } @@ -193,6 +202,9 @@ func (fc *FileCache) Start(ctx context.Context) error { func (fc *FileCache) Stop() error { log.Trace("Stopping component : %s", fc.Name()) + // stop async uploads + close(fc.componentStopping) + // Wait for all async upload to complete if any if fc.lazyWrite { log.Info("FileCache::Stop : Waiting for async close to complete") @@ -265,6 +277,7 @@ func (fc *FileCache) Configure(_ bool) error { fc.allowNonEmpty = conf.AllowNonEmpty fc.policyTrace = conf.EnablePolicyTrace fc.offloadIO = conf.OffloadIO + fc.offlineAccess = !conf.BlockOfflineAccess fc.syncToFlush = conf.SyncToFlush fc.syncToDelete = !conf.SyncNoOp fc.refreshSec = conf.RefreshSec @@ -359,50 +372,18 @@ func (fc *FileCache) Configure(_ bool) error { } if config.IsSet(compName + ".schedule") { - var rawSchedule []map[string]interface{} - err := config.UnmarshalKey(compName+".schedule", &rawSchedule) + err = fc.configureScheduler() if err != nil { log.Err( "FileCache::Configure : Failed to parse schedule configuration [%s]", err.Error(), ) - } else { - // Convert raw schedule to WeeklySchedule - fc.schedule = make(WeeklySchedule, 0, len(rawSchedule)) - for _, rawWindow := range rawSchedule { - window := UploadWindow{} - if name, ok := rawWindow["name"].(string); ok { - window.Name = name - } - if cronStr, ok := rawWindow["cron"].(string); ok { - window.CronExpr = cronStr - } - if durStr, ok := rawWindow["duration"].(string); ok { - window.Duration = durStr - } - if !isValidCronExpression(window.CronExpr) { - log.Err("FileCache::Configure : Invalid cron expression '%s' for schedule window '%s', skipping", - window.CronExpr, window.Name) - continue - } - - // Validate duration - _, err := time.ParseDuration(window.Duration) - if err != nil { - log.Err("FileCache::Configure : Invalid duration '%s' for schedule window '%s': %v, skipping", - window.Duration, window.Name, err) - continue - } - - fc.schedule = append(fc.schedule, window) - log.Info("FileCache::Configure : Parsed schedule %s: cron=%s, duration=%s", - window.Name, window.CronExpr, window.Duration) - } + return fmt.Errorf("config error in %s [invalid schedule format: %w]", fc.Name(), err) } } log.Crit( - "FileCache::Configure : create-empty %t, cache-timeout %d, tmp-path %s, max-size-mb %d, high-mark %d, low-mark %d, refresh-sec %v, max-eviction %v, hard-limit %v, policy %s, allow-non-empty-temp %t, cleanup-on-start %t, policy-trace %t, offload-io %t, sync-to-flush %t, ignore-sync %t, defaultPermission %v, diskHighWaterMark %v, maxCacheSize %v, mountPath %v", + "FileCache::Configure : create-empty %t, cache-timeout %d, tmp-path %s, max-size-mb %d, high-mark %d, low-mark %d, refresh-sec %v, max-eviction %v, hard-limit %v, policy %s, allow-non-empty-temp %t, cleanup-on-start %t, policy-trace %t, offload-io %t, !block-offline-access %t, sync-to-flush %t, !ignore-sync %t, defaultPermission %v, diskHighWaterMark %v, maxCacheSize %v, mountPath %v, scheduleWindows: %d", fc.createEmptyFile, int(fc.cacheTimeout), fc.tmpPath, @@ -417,6 +398,7 @@ func (fc *FileCache) Configure(_ bool) error { conf.CleanupOnStart, fc.policyTrace, fc.offloadIO, + fc.offlineAccess, fc.syncToFlush, fc.syncToDelete, fc.defaultPermission, @@ -471,6 +453,7 @@ func (fc *FileCache) GetPolicyConfig(conf FileCacheOptions) cachePolicyConfig { maxSizeMB: conf.MaxSizeMB, fileLocks: fc.fileLocks, policyTrace: conf.EnablePolicyTrace, + pendingOps: &fc.pendingOps, } return cacheConfig @@ -479,7 +462,10 @@ func (fc *FileCache) GetPolicyConfig(conf FileCacheOptions) cachePolicyConfig { func (fc *FileCache) StatFs() (*common.Statfs_t, bool, error) { statfs, populated, err := fc.NextComponent().StatFs() + // TODO: handle offline errors if populated { + // if we are offline, this will return EIO to the system + // TODO: Is this the desired behavior? return statfs, populated, err } @@ -539,23 +525,106 @@ func isLocalDirEmpty(path string) bool { return err == io.EOF } -// Note: The primary purpose of the file cache is to keep track of files that are opened by the user. -// So we do not need to support some APIs like Create Directory since the file cache will manage -// creating local directories as needed. +func (fc *FileCache) CreateDir(options internal.CreateDirOptions) error { + log.Trace("FileCache::CreateDir : %s", options.Name) + + // if offline access is disabled, just pass this call on to the attribute cache + if !fc.offlineAccess { + return fc.NextComponent().CreateDir(options) + } + + localPath := filepath.Join(fc.tmpPath, options.Name) -// DeleteDir: Recursively invalidate the directory and its children + // Do not call nextComponent.CreateDir when we are offline. + // Otherwise the attribute cache could go out of sync with the cloud. + if fc.NextComponent().CloudConnected() { + // we have a cloud connection, so it's safe to call the next component + err := fc.NextComponent().CreateDir(options) + if err == nil || errors.Is(err, os.ErrExist) { + // creating the directory in cloud either worked, or it already exists + // make sure the directory exists in local cache + mkdirErr := os.MkdirAll(localPath, options.Mode.Perm()) + if mkdirErr != nil { + log.Err( + "FileCache::CreateDir : %s failed to create local directory. Here's why: %v", + localPath, + mkdirErr, + ) + } + } + return err + } + + // we are offline + // check if the directory exists in cloud storage + notInCloud, err := fc.checkCloud(options.Name) + switch { + case notInCloud: + // the directory does not exist in the cloud, so we can create it locally + err = os.Mkdir(localPath, options.Mode.Perm()) + if err != nil { + // report and return the error, since it will rightly return EEXIST when needed, etc + log.Err("FileCache::CreateDir : %s os.Mkdir failed. Here's why: %v", err) + } else { + // record this directory to sync to cloud later + // Note: the s3storage component can return success on CreateDir, even without a cloud connection. + // The thread that pushes local changes to the cloud will have to account for this + // to avoid creating an entry for this directory in the attribute cache, + // which would give us the false impression that the directory is in the cloud. + flock := fc.fileLocks.Get(options.Name) + flock.Lock() + defer flock.Unlock() + fc.addPendingOp(options.Name, flock) + log.Info("FileCache::CreateDir : %s created offline and queued for cloud sync", options.Name) + } + case err != nil && !isOffline(err): + // we seem to have regained our cloud connection, but GetAttr failed for some reason + // log this and return the error from GetAttr as is + log.Err("FileCache::CreateDir : %s GetAttr failed. Here's why: %v", options.Name, err) + case errors.Is(err, &common.NoCachedDataError{}): + // we are offline and we don't know whether the directory exists in cloud storage + // block directory creation (to protect data consistency) + log.Warn( + "FileCache::CreateDir : %s might exist in cloud storage. Creation is blocked.", + options.Name, + ) + default: + // the directory already exists in cloud storage + err = os.ErrExist + // use distinct log messages for when the attribute cache entry is valid or expired + if err == nil { // valid + log.Warn("FileCache::CreateDir : %s already exists in cloud storage", options.Name) + } else { // expired + log.Warn("FileCache::CreateDir : %s already exists in cloud storage (and we are offline)", options.Name) + } + } + return err +} + +// DeleteDir: Delete empty directory func (fc *FileCache) DeleteDir(options internal.DeleteDirOptions) error { log.Trace("FileCache::DeleteDir : %s", options.Name) // The libfuse component only calls DeleteDir on empty directories, so this directory must be empty err := fc.NextComponent().DeleteDir(options) if err != nil { - log.Err("FileCache::DeleteDir : %s failed", options.Name) + log.Err("FileCache::DeleteDir : %s failed. Here's why: %v", options.Name, err) // There is a chance that meta file for directory was not created in which case // rest api delete will fail while we still need to cleanup the local cache for the same } else { fc.policy.CachePurge(filepath.Join(fc.tmpPath, options.Name)) } + // is the cloud connection down? Is offline access enabled? + if isOffline(err) && fc.offlineOperationAllowed(options.Name) { + // this is a local directory + // remove it from the deferred cloud operations + // TODO: protect this with a semaphore (probably flock) + fc.pendingOps.Delete(options.Name) + // delete it locally + fc.policy.CachePurge(filepath.Join(fc.tmpPath, options.Name)) + // clear the error + err = nil + } return err } @@ -571,6 +640,13 @@ func (fc *FileCache) StreamDir( // To cover case 1, grab all entries from storage attrs, token, err := fc.NextComponent().StreamDir(options) + if isOffline(err) && fc.offlineAccess { + // we're offline and offline access is allowed, so let's check if we have valid a listing + if !errors.Is(err, &common.NoCachedDataError{}) { + // drop the error message + err = nil + } + } if err != nil { return attrs, token, err } @@ -579,6 +655,13 @@ func (fc *FileCache) StreamDir( localPath := filepath.Join(fc.tmpPath, options.Name) dirents, err := os.ReadDir(localPath) if err != nil { + if !os.IsNotExist(err) { + log.Err( + "FileCache::StreamDir : %s os.ReadDir failed. Here's why: %v", + options.Name, + err, + ) + } return attrs, token, nil } @@ -627,7 +710,7 @@ func (fc *FileCache) StreamDir( // and container or not. So we rely on getAttr to tell if entry was cached then it exists in cloud storage too // If entry does not exists on storage then only return a local item here. _, err := fc.NextComponent().GetAttr(internal.GetAttrOptions{Name: entryPath}) - if err != nil && (err == syscall.ENOENT || os.IsNotExist(err)) { + if err != nil && errors.Is(err, os.ErrNotExist) { // get the lock on the file, to allow any pending operation to complete flock := fc.fileLocks.Get(entryPath) flock.RLock() @@ -636,7 +719,6 @@ func (fc *FileCache) StreamDir( filepath.Join(localPath, entry.Name()), ) // Grab local cache attributes flock.RUnlock() - // If local file is not locked then only use its attributes otherwise rely on container attributes if err == nil { // Case 2 (file only in local cache) so create a new attributes and add them to the storage attributes log.Debug("FileCache::StreamDir : serving %s from local cache", entryPath) @@ -734,6 +816,7 @@ func (fc *FileCache) deleteEmptyDirs(options internal.DeleteDirOptions) (bool, e func (fc *FileCache) RenameDir(options internal.RenameDirOptions) error { log.Trace("FileCache::RenameDir : src=%s, dst=%s", options.Src, options.Dst) + // first we need to lock all the files involved // get a list of source objects form both cloud and cache // cloud var cloudObjects []string @@ -758,14 +841,15 @@ func (fc *FileCache) RenameDir(options internal.RenameDirOptions) error { return err } // combine the lists - objectNames := combineLists(cloudObjects, localObjects) - - // add object destinations, and sort the result - for _, srcName := range objectNames { + srcObjects := combineLists(cloudObjects, localObjects) + // add destinations + var dstObjects []string + for _, srcName := range srcObjects { dstName := strings.Replace(srcName, options.Src, options.Dst, 1) - objectNames = append(objectNames, dstName) + dstObjects = append(dstObjects, dstName) } - sort.Strings(objectNames) + // combine sources and destinations + objectNames := combineLists(srcObjects, dstObjects) // acquire a file lock on each entry (and defer unlock) flocks := make([]*common.LockMapItem, 0, len(objectNames)) @@ -778,8 +862,15 @@ func (fc *FileCache) RenameDir(options internal.RenameDirOptions) error { // rename the directory in the cloud err = fc.NextComponent().RenameDir(options) - if err != nil { - log.Err("FileCache::RenameDir : error %s [%s]", options.Src, err.Error()) + // if we are offline, and offline access is enabled, allow local directories to be renamed + if isOffline(err) && fc.offlineOperationAllowed(options.Src) && fc.notInCloud(options.Dst) { + log.Warn( + "FileCache::RenameDir : %s -> %s Cloud is unreachable but neither directory is in cloud storage. Proceeding with offline rename.", + options.Src, + options.Dst, + ) + } else if err != nil { + log.Err("FileCache::RenameDir : %s -> %s Cloud rename failed. Here's why: %v", options.Src, options.Dst, err) return err } @@ -809,6 +900,8 @@ func (fc *FileCache) RenameDir(options internal.RenameDirOptions) error { } // remember to delete the src directory later (after its contents are deleted) directoriesToPurge = append(directoriesToPurge, path) + // update pending cloud ops + fc.renamePendingOp(fc.getObjectName(path), fc.getObjectName(newPath), fc.fileLocks.Get(fc.getObjectName(newPath))) } } else { // stat(localPath) failed. err is the one returned by stat @@ -850,9 +943,12 @@ func (fc *FileCache) listCloudObjects(prefix string) (objectNames []string, err var attrSlice []*internal.ObjAttr attrSlice, token, err = fc.NextComponent(). StreamDir(internal.StreamDirOptions{Name: prefix, Token: token}) - if err != nil { + if offlineDataAvailable(err) && fc.offlineAccess { + err = nil + } else if err != nil { return } + // collect the object names for i := len(attrSlice) - 1; i >= 0; i-- { attr := attrSlice[i] if !attr.IsDir() { @@ -903,26 +999,27 @@ func (fc *FileCache) listCachedObjects(directory string) (objectNames []string, func combineLists(listA, listB []string) []string { // since both lists are sorted, we can combine the two lists using a double-indexed for loop - combinedList := listA + var combinedList []string i := 0 // Index for listA j := 0 // Index for listB - // Iterate through both lists, adding entries from B that are missing from A + // Iterate through both lists, adding entries in order for i < len(listA) && j < len(listB) { itemA := listA[i] itemB := listB[j] if itemA < itemB { + combinedList = append(combinedList, itemA) i++ } else if itemA > itemB { - // we could insert here, but it's probably better to just sort later combinedList = append(combinedList, itemB) j++ } else { + // the items are the same - just add one + combinedList = append(combinedList, itemA) i++ j++ } } - // sort and return - sort.Strings(combinedList) + return combinedList } @@ -950,6 +1047,7 @@ func unlockAll(flocks []*common.LockMapItem) { func (fc *FileCache) CreateFile(options internal.CreateFileOptions) (*handlemap.Handle, error) { //defer exectime.StatTimeCurrentBlock("FileCache::CreateFile")() log.Trace("FileCache::CreateFile : name=%s, mode=%d", options.Name, options.Mode) + var offline bool flock := fc.fileLocks.Get(options.Name) flock.Lock() @@ -958,11 +1056,20 @@ func (fc *FileCache) CreateFile(options internal.CreateFileOptions) (*handlemap. // createEmptyFile was added to optionally support immutable containers. If customers do not care about immutability they can set this to true. if fc.createEmptyFile { newF, err := fc.NextComponent().CreateFile(options) + if err == nil { + newF.GetFileObject().Close() + } + // are we offline? + if isOffline(err) && fc.offlineOperationAllowed(options.Name) { + // remember that we're offline + offline = true + // clear the error + err = nil + } if err != nil { log.Err("FileCache::CreateFile : Failed to create file %s", options.Name) return nil, err } - newF.GetFileObject().Close() } // Create the file in local cache @@ -1015,6 +1122,11 @@ func (fc *FileCache) CreateFile(options internal.CreateFileOptions) (*handlemap. // update state flock.LazyOpen = false + // if we're offline, record this operation as pending + if offline { + fc.addPendingOp(options.Name, flock) + } + return handle, nil } @@ -1031,7 +1143,7 @@ func (fc *FileCache) validateStorageError( ) error { // For methods that take in file name, the goal is to update the path in cloud storage and the local cache. // See comments in GetAttr for the different situations we can run into. This specifically handles case 2. - if os.IsNotExist(err) { + if !isOffline(err) && errors.Is(err, os.ErrNotExist) { log.Debug("FileCache::%s : %s does not exist in cloud storage", method, path) if !fc.createEmptyFile { // Check if the file exists in the local cache @@ -1057,6 +1169,7 @@ func (fc *FileCache) validateStorageError( func (fc *FileCache) DeleteFile(options internal.DeleteFileOptions) error { log.Trace("FileCache::DeleteFile : name=%s", options.Name) + localPath := filepath.Join(fc.tmpPath, options.Name) flock := fc.fileLocks.Get(options.Name) flock.Lock() @@ -1064,19 +1177,23 @@ func (fc *FileCache) DeleteFile(options internal.DeleteFileOptions) error { err := fc.NextComponent().DeleteFile(options) err = fc.validateStorageError(options.Name, err, "DeleteFile", true) + if isOffline(err) && fc.offlineOperationAllowed(options.Name) { + // we are offline and the file is not in cloud, so handle deletion locally + // reset err to whether the local file exists + _, err = os.Stat(localPath) + } if err != nil { - log.Err("FileCache::DeleteFile : error %s [%s]", options.Name, err.Error()) + log.Err("FileCache::DeleteFile : %s deletion failed. Here's why: %v", options.Name, err) return err } - localPath := filepath.Join(fc.tmpPath, options.Name) + // delete file from cache fc.policy.CachePurge(localPath) - // Delete from scheduleOps if it exists - fc.scheduleOps.Delete(options.Name) - // update file state flock.LazyOpen = false + // remove deleted file from async upload map + fc.pendingOps.Delete(options.Name) return nil } @@ -1111,7 +1228,40 @@ func (fc *FileCache) openFileInternal(handle *handlemap.Handle, flock *common.Lo fc.policy.CacheValid(localPath) downloadRequired, fileExists, attr, err := fc.isDownloadRequired(localPath, handle.Path, flock) - if err != nil && !os.IsNotExist(err) { + + // handle offline cases + if isOffline(err) || !fc.NextComponent().CloudConnected() { + if !fc.offlineAccess { + // offline access is not allowed + if downloadRequired || !cachedData(err) { + // data is unavailable - do not open the file + log.Err("FileCache::OpenFile : %s can't download data (offline)", handle.Path) + return &common.CloudUnreachableError{} + } else { + // download is not required, but we can't write while we're offline + // TODO: should we just allow writes, in case the connection is re-established soon? + log.Err("FileCache::OpenFile : %s Read-only enabled (offline)", handle.Path) + flags = os.O_RDONLY + } + } else if !errors.Is(err, os.ErrNotExist) { + // offline access is allowed, but this object might exist in cloud storage + if fileExists { + // data is cached but (might be) in cloud, so only allow read-only access + log.Err("FileCache::OpenFile : %s Read-only access, for consistency offline", handle.Path) + flags = os.O_RDONLY + if downloadRequired { + log.Warn("FileCache::OpenFile : %s ignoring refresh timer (offline)", handle.Path) + downloadRequired = false + } + } else { + // data is unavailable - do not open the file + log.Err("FileCache::OpenFile : %s data unavailable (offline)", handle.Path) + return &common.CloudUnreachableError{} + } + } + } + + if err != nil && !errors.Is(err, os.ErrNotExist) { log.Err( "FileCache::openFileInternal : Failed to check if download is required for %s [%s]", handle.Path, @@ -1184,8 +1334,6 @@ func (fc *FileCache) openFileInternal(handle *handlemap.Handle, flock *common.Lo // Update the last download time of this file flock.SetDownloadTime() - // update file state - flock.LazyOpen = false log.Debug("FileCache::openFileInternal : Download of %s is complete", handle.Path) f.Close() @@ -1359,7 +1507,7 @@ func (fc *FileCache) isDownloadRequired( // get cloud attributes cloudAttr, err := fc.NextComponent().GetAttr(internal.GetAttrOptions{Name: objectPath}) - if err != nil && !os.IsNotExist(err) { + if err != nil && !errors.Is(err, os.ErrNotExist) { log.Err( "FileCache::isDownloadRequired : Failed to get attr of %s [%s]", objectPath, @@ -1665,7 +1813,7 @@ func (fc *FileCache) flushFileInternal(options internal.FlushFileOptions) error fc.policy.CacheValid(localPath) // if our handle is dirty then that means we wrote to the file if options.Handle.Dirty() { - if fc.lazyWrite && !options.CloseInProgress { + if fc.lazyWrite && !options.CloseInProgress && !options.AsyncUpload { // As lazy-write is enable, upload will be scheduled when file is closed. log.Info( "FileCache::FlushFile : %s will be flushed when handle %d is closed", @@ -1702,11 +1850,15 @@ func (fc *FileCache) flushFileInternal(options internal.FlushFileOptions) error // The local handle can still be used for read and write. var orgMode fs.FileMode modeChanged := false - notInCloud := fc.notInCloud( - options.Handle.Path, - ) - // Figure out if we should upload immediately or append to pending OPS - if options.AsyncUpload || !notInCloud || fc.alwaysOn { + // decide whether to schedule the upload instead + var scheduleUpload bool + select { + case <-fc.schedulerActiveCh: + // schedule is inactive - defer new object writes + scheduleUpload = fc.notInCloud(options.Handle.Path) + default: + } + if !scheduleUpload { uploadHandle, err := common.Open(localPath) if err != nil { if os.IsPermission(err) { @@ -1740,19 +1892,6 @@ func (fc *FileCache) flushFileInternal(options internal.FlushFileOptions) error }) uploadHandle.Close() - if err == nil { - // Clear dirty flag since file was successfully uploaded - options.Handle.Flags.Clear(handlemap.HandleFlagDirty) - } - - if err != nil { - log.Err( - "FileCache::FlushFile : %s upload failed [%s]", - options.Handle.Path, - err.Error(), - ) - return err - } if modeChanged { err1 := os.Chmod(localPath, orgMode) @@ -1764,20 +1903,33 @@ func (fc *FileCache) flushFileInternal(options internal.FlushFileOptions) error ) } } + + switch { + case err == nil: + options.Handle.Flags.Clear(handlemap.HandleFlagDirty) + case isOffline(err) && fc.offlineAccess: + log.Warn("FileCache::FlushFile : %s upload delayed (offline)", options.Handle.Path) + // add file to upload queue + _, err := os.Stat(localPath) + if err == nil { + flock := fc.fileLocks.Get(options.Handle.Path) + fc.addPendingOp(options.Handle.Path, flock) + } + default: + log.Err("FileCache::FlushFile : %s upload failed [%v]", options.Handle.Path, err) + return err + } } else { - //push to scheduleOps as default since we don't want to upload to the cloud + //push to pendingOps as default since we don't want to upload to the cloud log.Info( "FileCache::FlushFile : %s upload deferred (Scheduled for upload)", options.Handle.Path, ) _, statErr := os.Stat(localPath) if statErr == nil { - fc.markFileForUpload(options.Handle.Path) - flock := fc.fileLocks.Get(options.Handle.Path) - flock.SyncPending = true + fc.addPendingOp(options.Handle.Path, fc.fileLocks.Get(options.Handle.Path)) } options.Handle.Flags.Clear(handlemap.HandleFlagDirty) - } // If chmod was done on the file before it was uploaded to container then setting up mode would have been missed @@ -1830,25 +1982,30 @@ func (fc *FileCache) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr // To cover case 1, get attributes from storage var exists bool attrs, err := fc.NextComponent().GetAttr(options) - if err != nil { - if err == syscall.ENOENT || os.IsNotExist(err) { - log.Debug("FileCache::GetAttr : %s does not exist in cloud storage", options.Name) - exists = false - } else { - log.Err("FileCache::GetAttr : Failed to get attr of %s [%s]", options.Name, err.Error()) - return nil, err - } - } else { + switch { + case !isOffline(err) && os.IsNotExist(err): + log.Debug("FileCache::GetAttr : %s does not exist in cloud storage", options.Name) + case err == nil: exists = true + case offlineDataAvailable(err) && fc.offlineAccess: + // we are offline, but we can respond from the attribute cache + exists = !errors.Is(err, os.ErrNotExist) + log.Debug("FileCache::GetAttr : %s exists=%t from cache (offline)", options.Name, exists) + default: + log.Err("FileCache::GetAttr : %s GetAttr failed. Here's why: %v", options.Name, err) + return nil, err } // To cover cases 2 and 3, grab the attributes from the local cache localPath := filepath.Join(fc.tmpPath, options.Name) info, err := os.Stat(localPath) flock.RUnlock() - // All directory operations are guaranteed to be synced with storage so they cannot be in a case 2 or 3 state. - if err == nil && !info.IsDir() { - if exists { // Case 3 (file in cloud storage and in local cache) so update the relevant attributes + if err == nil { + if !exists { // Case 2 (only in local cache) + log.Debug("FileCache::GetAttr : serving %s attr from local cache", options.Name) + exists = true + attrs = newObjAttr(options.Name, info) + } else if !info.IsDir() { // Case 3 (file in cloud storage and in local cache) so update the relevant attributes // attrs is a pointer returned by NextComponent // modifying attrs could corrupt cached directory listings // to update properties, we need to make a deep copy first @@ -1856,10 +2013,6 @@ func (fc *FileCache) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr newAttr.Mtime = info.ModTime() newAttr.Size = info.Size() attrs = &newAttr - } else { // Case 2 (file only in local cache) so create a new attributes and add them to the storage attributes - log.Debug("FileCache::GetAttr : serving %s attr from local cache", options.Name) - exists = true - attrs = newObjAttr(options.Name, info) } } @@ -1889,8 +2042,12 @@ func (fc *FileCache) RenameFile(options internal.RenameFileOptions) error { defer dflock.Unlock() err := fc.NextComponent().RenameFile(options) - localOnly := os.IsNotExist(err) + localOnly := errors.Is(err, os.ErrNotExist) err = fc.validateStorageError(options.Src, err, "RenameFile", true) + if isOffline(err) && fc.offlineOperationAllowed(options.Src) { + log.Debug("FileCache::RenameFile : %s Offline rename allowed", options.Src) + err = nil + } if err != nil { log.Err("FileCache::RenameFile : %s failed to rename file [%s]", options.Src, err.Error()) return err @@ -1918,14 +2075,6 @@ func (fc *FileCache) renameLocalFile( ) fc.policy.CacheValid(localDstPath) - // Transfer entry from scheduleOps if it exists - if _, found := fc.scheduleOps.Load(srcName); found { - fc.scheduleOps.Store(dstName, struct{}{}) - fc.scheduleOps.Delete(srcName) - - // Ensure SyncPending flag is set on destination - dflock.SyncPending = true - } case os.IsNotExist(err): if localOnly { // neither cloud nor file cache has this file, so return ENOENT @@ -1959,10 +2108,19 @@ func (fc *FileCache) renameLocalFile( // rename open handles fc.renameOpenHandles(srcName, dstName, sflock, dflock) + // update pending cloud ops + fc.renamePendingOp(fc.getObjectName(localSrcPath), fc.getObjectName(localDstPath), dflock) return nil } +func (fc *FileCache) renamePendingOp(srcName, dstName string, dflock *common.LockMapItem) { + _, operationPending := fc.pendingOps.LoadAndDelete(srcName) + if operationPending { + fc.pendingOps.Store(dstName, struct{}{}) + } +} + // files should already be locked before calling this function func (fc *FileCache) renameOpenHandles( srcName, dstName string, @@ -1983,6 +2141,8 @@ func (fc *FileCache) renameOpenHandles( sflock.Dec() dflock.Inc() } + // copy flags + dflock.LazyOpen = sflock.LazyOpen } } @@ -2005,12 +2165,18 @@ func (fc *FileCache) TruncateFile(options internal.TruncateFileOptions) error { } } + var offlineOkay bool flock := fc.fileLocks.Get(options.Name) flock.Lock() defer flock.Unlock() err := fc.NextComponent().TruncateFile(options) err = fc.validateStorageError(options.Name, err, "TruncateFile", true) + if isOffline(err) && fc.offlineOperationAllowed(options.Name) { + log.Debug("FileCache::TruncateFile : %s Offline truncate allowed", options.Name) + offlineOkay = true + err = nil + } if err != nil { log.Err("FileCache::TruncateFile : %s failed to truncate [%s]", options.Name, err.Error()) return err @@ -2019,9 +2185,8 @@ func (fc *FileCache) TruncateFile(options internal.TruncateFileOptions) error { // Update the size of the file in the local cache localPath := filepath.Join(fc.tmpPath, options.Name) info, err := os.Stat(localPath) - if err == nil || os.IsExist(err) { + if err == nil { fc.policy.CacheValid(localPath) - if info.Size() != options.Size { err = os.Truncate(localPath, options.Size) if err != nil { @@ -2031,6 +2196,9 @@ func (fc *FileCache) TruncateFile(options internal.TruncateFileOptions) error { err.Error(), ) return err + } else if offlineOkay { + fc.addPendingOp(options.Name, flock) + log.Warn("FileCache::TruncateFile : %s operation queued (offline)", options.Name) } } } @@ -2052,16 +2220,17 @@ func (fc *FileCache) Chmod(options internal.ChmodOptions) error { // file must be locked before calling this function func (fc *FileCache) chmodInternal(options internal.ChmodOptions) error { log.Trace("FileCache::Chmod : Change mode of path %s", options.Name) + var offlineOkay bool // Update the file in cloud storage err := fc.NextComponent().Chmod(options) err = fc.validateStorageError(options.Name, err, "Chmod", false) if err != nil { - if err != syscall.EIO { + case2okay := err == syscall.EIO + offlineOkay = isOffline(err) && fc.offlineOperationAllowed(options.Name) + if !case2okay && !offlineOkay { log.Err("FileCache::Chmod : %s failed to change mode [%s]", options.Name, err.Error()) return err - } else { - fc.missedChmodList.LoadOrStore(options.Name, true) } } @@ -2080,6 +2249,11 @@ func (fc *FileCache) chmodInternal(options internal.ChmodOptions) error { err.Error(), ) return err + } else if offlineOkay { + log.Warn("FileCache::Chmod : %s operation queued (offline)", options.Name) + fc.missedChmodList.LoadOrStore(options.Name, true) + flock := fc.fileLocks.Get(options.Name) + fc.addPendingOp(options.Name, flock) } } } @@ -2091,6 +2265,7 @@ func (fc *FileCache) chmodInternal(options internal.ChmodOptions) error { func (fc *FileCache) Chown(options internal.ChownOptions) error { log.Trace("FileCache::Chown : Change owner of path %s", options.Name) + var offlineOkay bool flock := fc.fileLocks.Get(options.Name) flock.Lock() defer flock.Unlock() @@ -2098,6 +2273,11 @@ func (fc *FileCache) Chown(options internal.ChownOptions) error { // Update the file in cloud storage err := fc.NextComponent().Chown(options) err = fc.validateStorageError(options.Name, err, "Chown", false) + if isOffline(err) && fc.offlineOperationAllowed(options.Name) { + log.Debug("FileCache::Chown : %s Offline chown allowed", options.Name) + offlineOkay = true + err = nil + } if err != nil { log.Err("FileCache::Chown : %s failed to change owner [%s]", options.Name, err.Error()) return err @@ -2105,8 +2285,7 @@ func (fc *FileCache) Chown(options internal.ChownOptions) error { // Update the owner and group of the file in the local cache localPath := filepath.Join(fc.tmpPath, options.Name) - _, err = os.Stat(localPath) - if err == nil { + if _, err = os.Stat(localPath); err == nil { fc.policy.CacheValid(localPath) if runtime.GOOS != "windows" { @@ -2118,6 +2297,10 @@ func (fc *FileCache) Chown(options internal.ChownOptions) error { err.Error(), ) return err + } else if offlineOkay { + // TODO: we have no missedChownList to track this... should we make one? Or should we just ignore this call? + log.Warn("FileCache::Chown : %s operation queued (offline)", options.Name) + fc.addPendingOp(options.Name, flock) } } } @@ -2138,8 +2321,7 @@ func (fc *FileCache) FileUsed(name string) error { // << DO NOT DELETE ANY AUTO GENERATED CODE HERE >> func NewFileCacheComponent() internal.Component { comp := &FileCache{ - fileLocks: common.NewLockMap(), - activeWindowsMutex: &sync.Mutex{}, + fileLocks: common.NewLockMap(), } comp.SetName(compName) config.AddConfigChangeEventListener(comp) diff --git a/component/file_cache/file_cache_test.go b/component/file_cache/file_cache_test.go index 3e839fe1d..663a24d58 100644 --- a/component/file_cache/file_cache_test.go +++ b/component/file_cache/file_cache_test.go @@ -135,13 +135,21 @@ func (suite *fileCacheTestSuite) setupTestHelper(configuration string) { suite.assert = assert.New(suite.T()) config.ReadConfigFromReader(strings.NewReader(configuration)) - suite.loopback = newLoopbackFS() - suite.fileCache = newTestFileCache(suite.loopback) - err := suite.loopback.Start(context.Background()) - if err != nil { - panic(fmt.Sprintf("Unable to start loopback [%s]", err.Error())) + if suite.useMock { + suite.mockCtrl = gomock.NewController(suite.T()) + suite.mock = internal.NewMockComponent(suite.mockCtrl) + suite.fileCache = newTestFileCache(suite.mock) + // always simulate being offline + suite.mock.EXPECT().CloudConnected().AnyTimes().Return(false) + } else { + suite.loopback = newLoopbackFS() + suite.fileCache = newTestFileCache(suite.loopback) + err := suite.loopback.Start(context.Background()) + if err != nil { + panic(fmt.Sprintf("Unable to start next component [%s]", err.Error())) + } } - err = suite.fileCache.Start(context.Background()) + err := suite.fileCache.Start(context.Background()) if err != nil { panic(fmt.Sprintf("Unable to start file cache [%s]", err.Error())) } @@ -149,11 +157,15 @@ func (suite *fileCacheTestSuite) setupTestHelper(configuration string) { } func (suite *fileCacheTestSuite) cleanupTest() { - suite.loopback.Stop() err := suite.fileCache.Stop() if err != nil { panic(fmt.Sprintf("Unable to stop file cache [%s]", err.Error())) } + if suite.useMock { + suite.mockCtrl.Finish() + } else { + suite.loopback.Stop() + } // Delete the temp directories created err = os.RemoveAll(suite.cache_path) @@ -426,12 +438,46 @@ func (suite *fileCacheTestSuite) TestCreateDir() { err := suite.fileCache.CreateDir(options) suite.assert.NoError(err) - // Path should not be added to the file cache - suite.assert.NoDirExists(filepath.Join(suite.cache_path, path)) + // Path should be added to the file cache + suite.assert.DirExists(filepath.Join(suite.cache_path, path)) // Path should be in fake storage suite.assert.DirExists(filepath.Join(suite.fake_storage_path, path)) } +// Tests CreateDir +func (suite *fileCacheTestSuite) TestCreateDirErrExist() { + defer suite.cleanupTest() + path := "a" + options := internal.CreateDirOptions{Name: path} + err := suite.fileCache.CreateDir(options) + suite.assert.NoError(err) + // test + err = suite.fileCache.CreateDir(options) + suite.assert.ErrorIs(err, os.ErrExist) +} + +// Tests CreateDir +func (suite *fileCacheTestSuite) TestCreateDirOffline() { + // enable mock component + suite.cleanupTest() + defaultConfig := fmt.Sprintf( + "file_cache:\n path: %s\n offload-io: true", + suite.cache_path, + ) + suite.useMock = true + suite.setupTestHelper(defaultConfig) + defer suite.cleanupTest() + // setup + path := "a" + options := internal.CreateDirOptions{Name: path} + suite.mock.EXPECT().GetAttr(internal.GetAttrOptions{Name: path}).Return(nil, os.ErrNotExist) + err := suite.fileCache.CreateDir(options) + suite.assert.NoError(err) + + // Path should be added to the file cache + suite.assert.DirExists(filepath.Join(suite.cache_path, path)) +} + func (suite *fileCacheTestSuite) TestDeleteDir() { defer suite.cleanupTest() // Setup @@ -1403,10 +1449,10 @@ func (suite *fileCacheTestSuite) TestWriteFileErrorBadFd() { // Setup file := "file20" handle := handlemap.NewHandle(file) - len, err := suite.fileCache.WriteFile(internal.WriteFileOptions{Handle: handle}) + length, err := suite.fileCache.WriteFile(internal.WriteFileOptions{Handle: handle}) suite.assert.Error(err) suite.assert.EqualValues(syscall.EBADF, err) - suite.assert.Equal(0, len) + suite.assert.Equal(0, length) } func (suite *fileCacheTestSuite) TestFlushFileEmpty() { @@ -1502,8 +1548,8 @@ loopbackfs: suite.assert.FileExists(filepath.Join(suite.cache_path, file)) suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, file)) - _, exists := suite.fileCache.scheduleOps.Load(file) - suite.assert.True(exists, "File should be in scheduleOps after creation") + _, exists := suite.fileCache.pendingOps.Load(file) + suite.assert.True(exists, "File should be in pendingOps after creation") // wait for uploads to start time.Sleep(time.Until(testStartTime.Add(2 * time.Second).Truncate(time.Second))) @@ -1513,12 +1559,8 @@ loopbackfs: _, err = os.Stat(filepath.Join(suite.fake_storage_path, file)) } suite.assert.FileExists(filepath.Join(suite.fake_storage_path, file)) - _, exists = suite.fileCache.scheduleOps.Load(file) - suite.assert.False(exists, "File should have been removed from scheduleOps after upload") - suite.assert.False( - suite.fileCache.fileLocks.Get(file).SyncPending, - "SyncPending flag should be cleared after upload", - ) + _, exists = suite.fileCache.pendingOps.Load(file) + suite.assert.False(exists, "File should have been removed from pendingOps after upload") } func (suite *fileCacheTestSuite) TestCronOnToOFFUpload() { @@ -1570,7 +1612,7 @@ loopbackfs: ) // wait until the window closes - time.Sleep(time.Since(testStartTime.Add(duration + 10*time.Millisecond))) + time.Sleep(time.Until(testStartTime.Add(duration + 10*time.Millisecond))) file2 := "scheduled_off_window.txt" handle2, err := suite.fileCache.CreateFile(internal.CreateFileOptions{Name: file2, Mode: 0777}) @@ -1582,10 +1624,8 @@ loopbackfs: suite.assert.NoError(err) suite.assert.FileExists(filepath.Join(suite.cache_path, file2)) suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, file2)) - _, scheduled := suite.fileCache.scheduleOps.Load(file2) + _, scheduled := suite.fileCache.pendingOps.Load(file2) suite.assert.True(scheduled, "File should be scheduled when scheduler is OFF") - flock := suite.fileCache.fileLocks.Get(file2) - suite.assert.True(flock.SyncPending, "SyncPending flag should be set") } func (suite *fileCacheTestSuite) TestNoScheduleAlwaysOn() { @@ -1615,17 +1655,12 @@ loopbackfs: suite.assert.NoError(err) suite.assert.FileExists(filepath.Join(suite.fake_storage_path, file), "File should be uploaded immediately with no schedule (always-on mode)") - _, exists := suite.fileCache.scheduleOps.Load(file) - suite.assert.False(exists, "File should not be in scheduleOps map") + _, exists := suite.fileCache.pendingOps.Load(file) + suite.assert.False(exists, "File should not be in pendingOps map") uploadedData, err := os.ReadFile(filepath.Join(suite.fake_storage_path, file)) suite.assert.NoError(err) suite.assert.Equal(data, uploadedData, "Uploaded file content should match original") - - flock := suite.fileCache.fileLocks.Get(file) - if flock != nil { - suite.assert.False(flock.SyncPending, "SyncPending flag should be clear") - } } func (suite *fileCacheTestSuite) TestExistingCloudFileImmediateUpload() { @@ -1660,8 +1695,8 @@ loopbackfs: originalContent := []byte("original cloud content") // Create the file in the cloud storage directly - err := os.MkdirAll(suite.fake_storage_path, 0777) - err = os.WriteFile(filepath.Join(suite.fake_storage_path, originalFile), originalContent, 0777) + _ = os.MkdirAll(suite.fake_storage_path, 0777) + _ = os.WriteFile(filepath.Join(suite.fake_storage_path, originalFile), originalContent, 0777) suite.assert.FileExists(filepath.Join(suite.fake_storage_path, originalFile)) suite.assert.NoFileExists(filepath.Join(suite.cache_path, originalFile)) @@ -1674,7 +1709,7 @@ loopbackfs: suite.assert.NoError(err) // Write new content to the file modifiedContent := []byte("modified cloud file content") - _, err = suite.fileCache.WriteFile(internal.WriteFileOptions{ + _, _ = suite.fileCache.WriteFile(internal.WriteFileOptions{ Handle: handle, Data: modifiedContent, Offset: 0, @@ -1730,9 +1765,9 @@ loopbackfs: suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, srcFile), "File should not exist in cloud storage when scheduler is OFF") - // Check if file is in scheduleOps with original name - _, existsInSchedule := suite.fileCache.scheduleOps.Load(srcFile) - suite.assert.True(existsInSchedule, "File should be in scheduleOps before rename") + // Check if file is in pendingOps with original name + _, existsInSchedule := suite.fileCache.pendingOps.Load(srcFile) + suite.assert.True(existsInSchedule, "File should be in pendingOps before rename") // Rename the file err = suite.fileCache.RenameFile(internal.RenameFileOptions{Src: srcFile, Dst: dstFile}) @@ -1744,24 +1779,18 @@ loopbackfs: suite.assert.FileExists(filepath.Join(suite.cache_path, dstFile), "Destination file should exist in local cache after rename") - // Check if the file has been renamed in scheduleOps - _, existsInScheduleOld := suite.fileCache.scheduleOps.Load(srcFile) + // Check if the file has been renamed in pendingOps + _, existsInScheduleOld := suite.fileCache.pendingOps.Load(srcFile) suite.assert.False( existsInScheduleOld, - "Old file name should not be in scheduleOps after rename", + "Old file name should not be in pendingOps after rename", ) - _, existsInScheduleNew := suite.fileCache.scheduleOps.Load(dstFile) - suite.assert.True(existsInScheduleNew, "New file name should be in scheduleOps after rename") - - // Check that file lock status was properly transferred - flock := suite.fileCache.fileLocks.Get(dstFile) - if flock != nil { - suite.assert.True(flock.SyncPending, "SyncPending flag should be set on renamed file") - } + _, existsInScheduleNew := suite.fileCache.pendingOps.Load(dstFile) + suite.assert.True(existsInScheduleNew, "New file name should be in pendingOps after rename") } -func (suite *fileCacheTestSuite) TestDeleteFileAndScheduleOps() { +func (suite *fileCacheTestSuite) TestDeleteFileAndPendingOps() { defer suite.cleanupTest() now := time.Now() @@ -1806,9 +1835,9 @@ loopbackfs: suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, testFile), "File should not exist in cloud storage when scheduler is OFF") - // Check if file is in scheduleOps before deletion - _, existsInSchedule := suite.fileCache.scheduleOps.Load(testFile) - suite.assert.True(existsInSchedule, "File should be in scheduleOps before deletion") + // Check if file is in pendingOps before deletion + _, existsInSchedule := suite.fileCache.pendingOps.Load(testFile) + suite.assert.True(existsInSchedule, "File should be in pendingOps before deletion") err = suite.fileCache.DeleteFile(internal.DeleteFileOptions{Name: testFile}) suite.assert.NoError(err) @@ -1817,10 +1846,10 @@ loopbackfs: suite.assert.NoFileExists(filepath.Join(suite.cache_path, testFile), "File should not exist in local cache after deletion") - // Check if the file has been deleted in scheduleOps - _, existsInScheduleAfterDelete := suite.fileCache.scheduleOps.Load(testFile) + // Check if the file has been deleted in pendingOps + _, existsInScheduleAfterDelete := suite.fileCache.pendingOps.Load(testFile) suite.assert.False(existsInScheduleAfterDelete, - "File should not be in scheduleOps after deletion") + "File should not be in pendingOps after deletion") } func (suite *fileCacheTestSuite) TestCreateEmptyFileEqualTrue() { @@ -1864,10 +1893,10 @@ loopbackfs: "Handle should not be marked as dirty when create-empty-file is true", ) - // The file shouldn't be in scheduleOps because it's already in cloud storage - _, existsInSchedule := suite.fileCache.scheduleOps.Load(testFile) + // The file shouldn't be in pendingOps because it's already in cloud storage + _, existsInSchedule := suite.fileCache.pendingOps.Load(testFile) suite.assert.False(existsInSchedule, - "File should not be in scheduleOps because it's already in cloud storage") + "File should not be in pendingOps because it's already in cloud storage") err = suite.fileCache.CloseFile(internal.CloseFileOptions{Handle: handle}) suite.assert.NoError(err) @@ -1919,9 +1948,9 @@ loopbackfs: suite.assert.NoFileExists(filepath.Join(suite.fake_storage_path, testFile), "File should not exist in cloud storage when scheduler is OFF") - // Check if file is in scheduleOps initially - _, existsInSchedule := suite.fileCache.scheduleOps.Load(testFile) - suite.assert.True(existsInSchedule, "File should be in scheduleOps after creation") + // Check if file is in pendingOps initially + _, existsInSchedule := suite.fileCache.pendingOps.Load(testFile) + suite.assert.True(existsInSchedule, "File should be in pendingOps after creation") // Write to file again with updated content newContent := []byte("updated file content") @@ -1937,9 +1966,9 @@ loopbackfs: err = suite.fileCache.CloseFile(internal.CloseFileOptions{Handle: handle}) suite.assert.NoError(err) - // Check scheduleOps to verify changes - _, stillInSchedule := suite.fileCache.scheduleOps.Load(testFile) - suite.assert.True(stillInSchedule, "File should remain in scheduleOps after modification") + // Check pendingOps to verify changes + _, stillInSchedule := suite.fileCache.pendingOps.Load(testFile) + suite.assert.True(stillInSchedule, "File should remain in pendingOps after modification") // Verify the local content was updated localData, err := os.ReadFile(filepath.Join(suite.cache_path, testFile)) @@ -1972,30 +2001,9 @@ loopbackfs: suite.cache_path, suite.fake_storage_path, ) - suite.setupTestHelper(configContent) - - // The invalid schedule should be skipped but valid one should be there - hasValidSchedule := false - for _, sched := range suite.fileCache.schedule { - if sched.Name == "InvalidTest" { - suite.assert.Fail("Invalid schedule should not be added") - } - if sched.Name == "ValidTest" { - hasValidSchedule = true - } - } - suite.assert.True(hasValidSchedule, "Valid schedule entry should be processed") - - // Test that operations still work with the valid schedule - file := "test_after_invalid_cron.txt" - handle, err := suite.fileCache.CreateFile(internal.CreateFileOptions{Name: file, Mode: 0777}) - suite.assert.NoError(err) - - err = suite.fileCache.CloseFile(internal.CloseFileOptions{Handle: handle}) - suite.assert.NoError(err) - suite.assert.FileExists(filepath.Join(suite.cache_path, file), - "File should be created successfully despite invalid cron expression") + // The invalid schedule should creash the configuration + suite.assert.Panics(func() { suite.setupTestHelper(configContent) }) } func (suite *fileCacheTestSuite) TestOverlappingSchedules() { diff --git a/component/file_cache/lru_policy.go b/component/file_cache/lru_policy.go index 5e026d865..36c1a041d 100644 --- a/component/file_cache/lru_policy.go +++ b/component/file_cache/lru_policy.go @@ -73,15 +73,13 @@ type lruPolicy struct { // DU utility was found on the path or not duPresent bool - - // Tracks scheduled files to skip during eviction - schedule *FileCache } // LRUPolicySnapshot represents the *persisted state* of lruPolicy. // It contains only the fields that need to be saved, and they are exported. type LRUPolicySnapshot struct { NodeList []string // Just node names, *without their fc.tmp prefix*, in linked list order + SyncPendingFlags []bool // whether each file in NodeList belongs in the pendingOps map CurrMarkerPosition uint64 // Node index of currMarker LastMarkerPosition uint64 // Node index of lastMarker } @@ -160,11 +158,6 @@ func (p *lruPolicy) ShutdownPolicy() error { return p.createSnapshot().writeToFile(p.tmpPath) } -func (fc *FileCache) IsScheduled(objName string) bool { - _, inSchedule := fc.scheduleOps.Load(objName) - return inSchedule -} - func (p *lruPolicy) createSnapshot() *LRUPolicySnapshot { log.Trace("lruPolicy::saveSnapshot") var snapshot LRUPolicySnapshot @@ -181,7 +174,11 @@ func (p *lruPolicy) createSnapshot() *LRUPolicySnapshot { case current == p.lastMarker: snapshot.LastMarkerPosition = index case strings.HasPrefix(current.name, p.tmpPath): - snapshot.NodeList = append(snapshot.NodeList, current.name[len(p.tmpPath):]) + relName := current.name[len(p.tmpPath):] + snapshot.NodeList = append(snapshot.NodeList, relName) + objName := common.NormalizeObjectName(relName[1:]) + _, isPending := p.pendingOps.Load(objName) + snapshot.SyncPendingFlags = append(snapshot.SyncPendingFlags, isPending) default: log.Err("lruPolicy::saveSnapshot : %s Ignoring unrecognized cache path", current.name) } @@ -194,6 +191,8 @@ func (p *lruPolicy) loadSnapshot(snapshot *LRUPolicySnapshot) { if snapshot == nil { return } + // maintain backward compatibility + loadPendingOps := len(snapshot.NodeList) == len(snapshot.SyncPendingFlags) p.Lock() defer p.Unlock() // walk the slice and write the entries into the policy @@ -201,7 +200,12 @@ func (p *lruPolicy) loadSnapshot(snapshot *LRUPolicySnapshot) { nodeIndex := 0 nextNode := p.head tail := p.lastMarker - for _, v := range snapshot.NodeList { + for i, v := range snapshot.NodeList { + // populate pendingOps + if loadPendingOps && snapshot.SyncPendingFlags[i] { + objName := v[1:] + p.pendingOps.Store(objName, struct{}{}) + } // recreate the node fullPath := filepath.Join(p.tmpPath, v) newNode := &lruNode{ @@ -510,7 +514,7 @@ func (p *lruPolicy) deleteExpiredNodes() { if objName[0] == '/' { objName = objName[1:] } - if p.schedule != nil && p.schedule.IsScheduled(objName) { + if _, syncPending := p.pendingOps.Load(objName); syncPending { continue } @@ -579,6 +583,13 @@ func (p *lruPolicy) deleteItem(name string) { return } + // check if the file is pending upload (it was modified offline) + if _, syncPending := p.pendingOps.Load(objName); syncPending { + log.Warn("lruPolicy::DeleteItem : %s File is not synchronized to cloud storage", name) + p.CacheValid(name) + return + } + // There are no open handles for this file so it's safe to remove this // Check if the file exists first, since this is often the second time we're calling deleteFile _, err := os.Stat(name) diff --git a/component/file_cache/lru_policy_test.go b/component/file_cache/lru_policy_test.go index e8e384504..848578dee 100644 --- a/component/file_cache/lru_policy_test.go +++ b/component/file_cache/lru_policy_test.go @@ -315,6 +315,10 @@ func (suite *lruPolicyTestSuite) verifyPolicy(expectedPolicy, actualPolicy *lruP suite.assert.Same(actualPolicy.lastMarker, actual) default: suite.assert.Equal(expected.name, actual.name) + objName := expected.name[len(suite.policy.tmpPath)+1:] + _, expectedPending := expectedPolicy.pendingOps.Load(objName) + _, actualPending := actualPolicy.pendingOps.Load(objName) + suite.assert.Equal(expectedPending, actualPending) } suite.assert.NotNil(actual, "actual list is shorter than expected") suite.assert.NotNil(expected, "actual list is longer than expected") @@ -337,6 +341,32 @@ func (suite *lruPolicyTestSuite) TestCreateSnapshotEmpty() { suite.verifyPolicy(originalPolicy, suite.policy) } +func (suite *lruPolicyTestSuite) TestCreateSnapshot() { + defer suite.cleanupTest() + // setup + numFiles := 5 + pathPrefix := filepath.Join(cache_path, "temp") + for i := 1; i <= numFiles; i++ { + suite.policy.CacheValid(pathPrefix + fmt.Sprint(i)) + if i > 3 { + suite.policy.pendingOps.Store("temp"+fmt.Sprint(i), struct{}{}) + } + } + originalPolicy := suite.policy + // test + snapshot := suite.policy.createSnapshot() + suite.cleanupTest() + suite.setupTestHelper(originalPolicy.cachePolicyConfig) + suite.policy.loadSnapshot(snapshot) + // assert + suite.assert.NotNil(snapshot) + suite.assert.Len(snapshot.NodeList, numFiles) + for i, v := range snapshot.NodeList { + suite.assert.Equal(pathPrefix+fmt.Sprint(numFiles-i), filepath.Join(cache_path, v)) + } + suite.verifyPolicy(originalPolicy, suite.policy) +} + func (suite *lruPolicyTestSuite) TestCreateSnapshotWithTrailingMarkers() { defer suite.cleanupTest() // setup @@ -481,6 +511,7 @@ func (suite *lruPolicyTestSuite) TestSnapshotSerialization() { NodeList: []string{"a", "b", "c"}, CurrMarkerPosition: 1, LastMarkerPosition: 2, + SyncPendingFlags: []bool{true, false, false}, } // test err := snapshot.writeToFile(cache_path) @@ -491,50 +522,48 @@ func (suite *lruPolicyTestSuite) TestSnapshotSerialization() { suite.assert.Equal(snapshot, snapshotFromFile) // this checks deep equality } -func (suite *lruPolicyTestSuite) TestNoEvictionIfInScheduleOps() { +func (suite *lruPolicyTestSuite) TestNoEvictionIfInPendingOps() { defer suite.cleanupTest() - fileName := filepath.Join(cache_path, "scheduled_file") + name := "pending_file" + fileName := filepath.Join(cache_path, name) suite.policy.CacheValid(fileName) - fakeSchedule := &FileCache{} - fakeSchedule.scheduleOps.Store(common.NormalizeObjectName("scheduled_file"), struct{}{}) - suite.policy.schedule = fakeSchedule + suite.policy.pendingOps.Store(name, struct{}{}) time.Sleep(2 * time.Second) - suite.assert.True(suite.policy.IsCached(fileName), "File in scheduleOps should not be evicted") + suite.assert.True(suite.policy.IsCached(fileName), "File in pendingOps should not be evicted") } -func (suite *lruPolicyTestSuite) TestEvictionRespectsScheduleOps() { +func (suite *lruPolicyTestSuite) TestEvictionRespectsPendingOps() { defer suite.cleanupTest() + objNames := []string{"File1", "file2", "file3", "file4"} fileNames := []string{ - filepath.Join(cache_path, "file1"), - filepath.Join(cache_path, "file2"), - filepath.Join(cache_path, "file3"), - filepath.Join(cache_path, "file4"), + filepath.Join(cache_path, objNames[0]), + filepath.Join(cache_path, objNames[1]), + filepath.Join(cache_path, objNames[2]), + filepath.Join(cache_path, objNames[3]), } for _, name := range fileNames { suite.policy.CacheValid(name) } - fakeSchedule := &FileCache{} - fakeSchedule.scheduleOps.Store(common.NormalizeObjectName("file2"), struct{}{}) - fakeSchedule.scheduleOps.Store(common.NormalizeObjectName("file4"), struct{}{}) - suite.policy.schedule = fakeSchedule + suite.policy.pendingOps.Store(objNames[1], struct{}{}) + suite.policy.pendingOps.Store(objNames[3], struct{}{}) time.Sleep(3 * time.Second) suite.assert.False(suite.policy.IsCached(fileNames[0]), "file1 should be evicted") suite.assert.True( suite.policy.IsCached(fileNames[1]), - "file2 should NOT be evicted (in scheduleOps)", + "file2 should NOT be evicted (in pendingOps)", ) suite.assert.False(suite.policy.IsCached(fileNames[2]), "file3 should be evicted") suite.assert.True( suite.policy.IsCached(fileNames[3]), - "file4 should NOT be evicted (in scheduleOps)", + "file4 should NOT be evicted (in pendingOps)", ) } diff --git a/component/file_cache/scheduler.go b/component/file_cache/scheduler.go deleted file mode 100644 index 964bd2385..000000000 --- a/component/file_cache/scheduler.go +++ /dev/null @@ -1,301 +0,0 @@ -/* - Licensed under the MIT License . - - Copyright © 2023-2025 Seagate Technology LLC and/or its Affiliates - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -package file_cache - -import ( - "context" - "errors" - "os" - "path/filepath" - "time" - - "github.com/Seagate/cloudfuse/common" - "github.com/Seagate/cloudfuse/common/log" - "github.com/Seagate/cloudfuse/internal" - "github.com/Seagate/cloudfuse/internal/handlemap" - "github.com/robfig/cron/v3" -) - -type UploadWindow struct { - Name string `yaml:"name"` - CronExpr string `yaml:"cron"` - Duration string `yaml:"duration"` -} - -type Config struct { - Schedule WeeklySchedule `yaml:"schedule"` -} - -type WeeklySchedule []UploadWindow - -func (fc *FileCache) SetupScheduler() error { - if len(fc.schedule) == 0 { - log.Info( - "FileCache::SetupScheduler : Empty schedule configuration, defaulting to always-on mode", - ) - fc.alwaysOn = true - return nil - } - - // Setup the cron scheduler - cronScheduler := cron.New(cron.WithSeconds()) - fc.scheduleUploads(cronScheduler, fc.schedule) - cronScheduler.Start() - - log.Info("FileCache::SetupScheduler : Scheduler started successfully") - return nil -} - -func isValidCronExpression(expr string) bool { - parser := cron.NewParser( - cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor, - ) - _, err := parser.Parse(expr) - return err == nil -} - -func (fc *FileCache) scheduleUploads(c *cron.Cron, sched WeeklySchedule) { - // define callbacks to activate and disable uploads - startFunc := func() { - log.Info("FileCache::SetupScheduler : Starting scheduled upload window") - fc.closeWindowCh = make(chan struct{}) - } - endFunc := func() { - log.Info("FileCache::SetupScheduler : Upload window ended") - close(fc.closeWindowCh) - } - // start up the schedules - for _, config := range sched { - windowName := config.Name - duration, err := time.ParseDuration(config.Duration) - if err != nil { - log.Info("[%s] Invalid duration '%s': %v\n", windowName, config.Duration, err) - continue - } - var initialWindowEndTime time.Time - - cronEntryId, err := c.AddFunc(config.CronExpr, func() { - // Start a new window and track it - fc.activeWindowsMutex.Lock() - isFirstWindow := fc.activeWindows == 0 - fc.activeWindows++ - windowCount := fc.activeWindows - fc.activeWindowsMutex.Unlock() - - // activate uploads - if isFirstWindow { - // open the window - startFunc() - } - - log.Info( - "schedule [%s] (%s) starting (active windows=%d)", - windowName, - config.CronExpr, - windowCount, - ) - fc.servicePendingOps() - - // When should the window close? - remainingDuration := duration - currentTime := time.Now() - if initialWindowEndTime.After(currentTime) { - remainingDuration = initialWindowEndTime.Sub(currentTime) - } - // Create a context to end the window - window, cancel := context.WithTimeout(context.Background(), remainingDuration) - defer cancel() - - for { - select { - case <-fc.stopAsyncUpload: - log.Info("Shutting down upload scheduler") - return - case <-window.Done(): - // Window has completed, update active window count - fc.activeWindowsMutex.Lock() - fc.activeWindows-- - isLastWindow := fc.activeWindows == 0 - windowCount := fc.activeWindows - fc.activeWindowsMutex.Unlock() - - log.Info("[%s] Upload window ended at %s (remaining windows: %d)\n", - windowName, time.Now().Format(time.Kitchen), windowCount) - - // Only close resources when the last window ends - if isLastWindow { - endFunc() - } - return - case <-fc.uploadNotifyCh: - log.Debug("[%s] File change detected, processing pending uploads at %s\n", - windowName, time.Now().Format(time.Kitchen)) - fc.servicePendingOps() - } - } - }) - if err != nil { - log.Err("[%s] Failed to schedule cron job with expression '%s': %v\n", - windowName, config.CronExpr, err) - continue - } - - // check if this schedule should already be active - // did this schedule have a start time within the last duration? - schedule := c.Entry(cronEntryId) - now := time.Now() - for t := schedule.Schedule.Next(now.Add(-duration)); now.After(t); t = schedule.Schedule.Next(t) { - initialWindowEndTime = t.Add(duration) - } - if !initialWindowEndTime.IsZero() { - go schedule.Job.Run() - } - } -} - -func (fc *FileCache) markFileForUpload(path string) { - fc.scheduleOps.Store(path, struct{}{}) - select { - case fc.uploadNotifyCh <- struct{}{}: - // Successfully notified - log.Info( - "FileCache::markFileForUpload : Notified upload window about new file: %s", - path, - ) - default: - // Channel buffer is full, which means notifications are already pending - // No need to block here as uploads will be processed soon - log.Info( - "FileCache::markFileForUpload : Upload window notification channel full, skipping notify for: %s", - path, - ) - } -} - -func (fc *FileCache) servicePendingOps() { - log.Info("FileCache::servicePendingOps : Servicing pending uploads") - - // Process pending operations - numFilesProcessed := 0 - fc.scheduleOps.Range(func(key, value interface{}) bool { - numFilesProcessed++ - select { - case <-fc.stopAsyncUpload: - log.Info("FileCache::servicePendingOps : Upload processing interrupted") - return false - case <-fc.closeWindowCh: - return false - default: - path := key.(string) - err := fc.uploadPendingFile(path) - if err != nil { - log.Err( - "FileCache::servicePendingOps : %s upload failed: %v", - path, - err, - ) - } - } - return true - }) - - log.Info( - "FileCache::servicePendingOps : Completed upload cycle, processed %d files", - numFilesProcessed, - ) -} - -func (fc *FileCache) uploadPendingFile(name string) error { - log.Trace("FileCache::uploadPendingFile : %s", name) - - // lock the file - flock := fc.fileLocks.Get(name) - flock.Lock() - defer flock.Unlock() - - // don't double upload - if !flock.SyncPending { - return nil - } - - // look up file (or folder!) - localPath := filepath.Join(fc.tmpPath, name) - info, err := os.Stat(localPath) - if err != nil { - log.Err("FileCache::uploadPendingFile : %s failed to stat file. Here's why: %v", name, err) - return err - } - if info.IsDir() { - // upload folder - options := internal.CreateDirOptions{Name: name, Mode: info.Mode()} - err = fc.NextComponent().CreateDir(options) - if err != nil && !os.IsExist(err) { - return err - } - } else { - // this is a file - // prepare a handle - handle := handlemap.NewHandle(name) - // open the cached file - f, err := common.OpenFile(localPath, os.O_RDONLY, fc.defaultPermission) - if err != nil { - log.Err("FileCache::uploadPendingFile : %s failed to open file. Here's why: %v", name, err) - return err - } - // write handle attributes - inf, err := f.Stat() - if err == nil { - handle.Size = inf.Size() - } - handle.UnixFD = uint64(f.Fd()) - handle.SetFileObject(f) - handle.Flags.Set(handlemap.HandleFlagDirty) - - // upload the file - err = fc.flushFileInternal(internal.FlushFileOptions{Handle: handle, CloseInProgress: true, AsyncUpload: true}) - f.Close() - if err != nil { - log.Err("FileCache::uploadPendingFile : %s Upload failed. Cause: %v", name, err) - return err - } - } - // update state - flock.SyncPending = false - fc.scheduleOps.Delete(name) - log.Info("FileCache::uploadPendingFile : File uploaded: %s", name) - - return nil -} - -func (fc *FileCache) notInCloud(name string) bool { - notInCloud, _ := fc.checkCloud(name) - return notInCloud -} - -func (fc *FileCache) checkCloud(name string) (notInCloud bool, getAttrErr error) { - _, getAttrErr = fc.NextComponent().GetAttr(internal.GetAttrOptions{Name: name}) - notInCloud = errors.Is(getAttrErr, os.ErrNotExist) - return notInCloud, getAttrErr -} diff --git a/component/loopback/loopback_fs.go b/component/loopback/loopback_fs.go index e9d8c36c9..6206a2366 100644 --- a/component/loopback/loopback_fs.go +++ b/component/loopback/loopback_fs.go @@ -100,6 +100,10 @@ func (lfs *LoopbackFS) Priority() internal.ComponentPriority { return internal.EComponentPriority.Consumer() } +func (lfs *LoopbackFS) CloudConnected() bool { + return true +} + func (lfs *LoopbackFS) CreateDir(options internal.CreateDirOptions) error { log.Trace("LoopbackFS::CreateDir : name=%s", options.Name) dirPath := filepath.Join(lfs.path, options.Name) diff --git a/component/s3storage/client.go b/component/s3storage/client.go index eccb64f68..1bfdbfa4d 100644 --- a/component/s3storage/client.go +++ b/component/s3storage/client.go @@ -156,8 +156,10 @@ func (cl *Client) Configure(cfg Config) error { ) } + ctx := context.Background() + defaultConfig, err := config.LoadDefaultConfig( - context.Background(), + ctx, config.WithSharedConfigProfile(cl.Config.AuthConfig.Profile), config.WithCredentialsProvider(credentialsProvider), config.WithAppID(UserAgent()), @@ -170,7 +172,7 @@ func (cl *Client) Configure(cfg Config) error { // If a config profile is provided the sdk checks that it exists, otherwise it fails and // does not try other credentials. So try the other ones here if the profile does not exist defaultConfig, err = config.LoadDefaultConfig( - context.Background(), + ctx, config.WithCredentialsProvider(credentialsProvider), config.WithAppID(UserAgent()), config.WithRegion(cl.Config.AuthConfig.Region), @@ -197,7 +199,7 @@ func (cl *Client) Configure(cfg Config) error { } // ListBuckets here to test connection to S3 backend - bucketList, err := cl.ListBuckets() + bucketList, err := cl.ListBuckets(ctx) if err != nil { log.Err("Client::Configure : listing buckets failed. Here's why: %v", err) @@ -234,7 +236,7 @@ func (cl *Client) Configure(cfg Config) error { // if no bucket-name was set, default to the first authorized bucket in the list if cl.Config.AuthConfig.BucketName == "" { // which buckets does the user have access to? - authorizedBucketList := cl.filterAuthorizedBuckets(bucketList) + authorizedBucketList := cl.filterAuthorizedBuckets(ctx, bucketList) switch len(authorizedBucketList) { case 0: // if there are none, return an error @@ -259,7 +261,7 @@ func (cl *Client) Configure(cfg Config) error { } // Check that the provided bucket exists and that user has access to bucket - _, err = cl.headBucket(cl.Config.AuthConfig.BucketName) + _, err = cl.headBucket(ctx, cl.Config.AuthConfig.BucketName) if err != nil { // From the aws-sdk-go-v2 documentation // If the bucket does not exist or you do not have permission to access it, @@ -270,7 +272,7 @@ func (cl *Client) Configure(cfg Config) error { } // Use list objects validate user can list objects - _, _, err = cl.List("/", nil, 1) + _, _, err = cl.List(ctx, "/", nil, 1) if err != nil { log.Err("Client::Configure : listing objects failed. Here's why: %v", err) return err @@ -291,19 +293,22 @@ func (cl *Client) Configure(cfg Config) error { } // Use ListBuckets and filterAuthorizedBuckets to get a list of buckets that the user has access to -func (cl *Client) ListAuthorizedBuckets() ([]string, error) { +func (cl *Client) ListAuthorizedBuckets(ctx context.Context) ([]string, error) { log.Trace("Client::ListAuthorizedBuckets") - allBuckets, err := cl.ListBuckets() + allBuckets, err := cl.ListBuckets(ctx) if err != nil { log.Err("Client::ListAuthorizedBuckets : Failed to list buckets. Here's why: %v", err) return allBuckets, err } - authorizedBuckets := cl.filterAuthorizedBuckets(allBuckets) + authorizedBuckets := cl.filterAuthorizedBuckets(ctx, allBuckets) return authorizedBuckets, nil } // filter out buckets for which we do not have permissions -func (cl *Client) filterAuthorizedBuckets(bucketList []string) (authorizedBucketList []string) { +func (cl *Client) filterAuthorizedBuckets( + ctx context.Context, + bucketList []string, +) (authorizedBucketList []string) { if len(bucketList) == 0 { return bucketList } @@ -314,7 +319,7 @@ func (cl *Client) filterAuthorizedBuckets(bucketList []string) (authorizedBucket wg.Add(1) go func(bucketName string) { defer wg.Done() - if _, err := cl.headBucket(bucketName); err == nil { + if _, err := cl.headBucket(ctx, bucketName); err == nil { authorizedBuckets <- bucketName } }(bucketName) @@ -386,21 +391,21 @@ func (cl *Client) SetPrefixPath(path string) error { } // CreateFile : Create a new file in the bucket/virtual directory -func (cl *Client) CreateFile(name string, mode os.FileMode) error { +func (cl *Client) CreateFile(ctx context.Context, name string, mode os.FileMode) error { log.Trace("Client::CreateFile : name %s", name) var data []byte - return cl.WriteFromBuffer(name, nil, data) + return cl.WriteFromBuffer(ctx, name, nil, data) } // CreateDirectory : Create a new directory in the bucket/virtual directory -func (cl *Client) CreateDirectory(name string) error { +func (cl *Client) CreateDirectory(ctx context.Context, name string) error { log.Trace("Client::CreateDirectory : name %s", name) // If the S3 endpoint does not support directory markers then we can do nothing here. // So, let's make it clear: we expect the OS to call GetAttr() on the directory // to make sure it doesn't exist before trying to create it. if cl.Config.enableDirMarker { - err := cl.putObject(putObjectOptions{name: name, isDir: true}) + err := cl.putObject(ctx, putObjectOptions{name: name, isDir: true}) if err != nil { log.Err("Client::CreateDirectory : putObject(%s) failed. Here's why: %v", name, err) return err @@ -411,7 +416,12 @@ func (cl *Client) CreateDirectory(name string) error { } // CreateLink : Create a symlink in the bucket/virtual directory -func (cl *Client) CreateLink(source string, target string, isSymlink bool) error { +func (cl *Client) CreateLink( + ctx context.Context, + source string, + target string, + isSymlink bool, +) error { log.Trace("Client::CreateLink : %s -> %s", source, target) data := []byte(target) @@ -419,15 +429,15 @@ func (cl *Client) CreateLink(source string, target string, isSymlink bool) error if isSymlink { symlinkMap[symlinkKey] = to.Ptr("true") } - return cl.WriteFromBuffer(source, symlinkMap, data) + return cl.WriteFromBuffer(ctx, source, symlinkMap, data) } // DeleteFile : Delete an object. // if the file does not exist, this returns an error (ENOENT). -func (cl *Client) DeleteFile(name string) error { +func (cl *Client) DeleteFile(ctx context.Context, name string) error { log.Trace("Client::DeleteFile : name %s", name) // first check if the object exists - attr, err := cl.getFileAttr(name) + attr, err := cl.getFileAttr(ctx, name) if err == syscall.ENOENT { log.Err("Client::DeleteFile : %s does not exist", name) return syscall.ENOENT @@ -439,7 +449,7 @@ func (cl *Client) DeleteFile(name string) error { isSymLink := attr.IsSymlink() // delete the object - err = cl.deleteObject(name, isSymLink, attr.IsDir()) + err = cl.deleteObject(ctx, name, isSymLink, attr.IsDir()) if err != nil { log.Err("Client::DeleteFile : Failed to delete object %s. Here's why: %v", name, err) return err @@ -451,7 +461,7 @@ func (cl *Client) DeleteFile(name string) error { // DeleteDirectory : Recursively delete all objects with the given prefix. // If name is given without a trailing slash, a slash will be added. // If the directory does not exist, no error will be returned. -func (cl *Client) DeleteDirectory(name string) error { +func (cl *Client) DeleteDirectory(ctx context.Context, name string) error { log.Trace("Client::DeleteDirectory : name %s", name) // make sure name has a trailing slash @@ -462,7 +472,7 @@ func (cl *Client) DeleteDirectory(name string) error { var err error for !done { // list all objects with the prefix - objects, marker, err := cl.List(name, marker, 0) + objects, marker, err := cl.List(ctx, name, marker, 0) if err != nil { log.Warn( "Client::DeleteDirectory : Failed to list object with prefix %s. Here's why: %v", @@ -487,7 +497,7 @@ func (cl *Client) DeleteDirectory(name string) error { var objectsToDelete []*internal.ObjAttr for _, object := range objects { if object.IsDir() { - err = cl.DeleteDirectory(object.Path) + err = cl.DeleteDirectory(ctx, object.Path) if err != nil { log.Err( "Client::DeleteDirectory : Failed to delete directory %s. Here's why: %v", @@ -500,7 +510,7 @@ func (cl *Client) DeleteDirectory(name string) error { } } // Delete the collected files - err = cl.deleteObjects(objectsToDelete) + err = cl.deleteObjects(ctx, objectsToDelete) if err != nil { log.Err( "Client::DeleteDirectory : deleteObjects() failed when called with %d objects. Here's why: %v", @@ -516,7 +526,7 @@ func (cl *Client) DeleteDirectory(name string) error { // Delete the current directory if cl.Config.enableDirMarker { - err = cl.deleteObject(name, false, true) + err = cl.deleteObject(ctx, name, false, true) if err != nil { log.Err( "Client::DeleteDirectory : Failed to delete directory %s. Here's why: %v", @@ -530,10 +540,16 @@ func (cl *Client) DeleteDirectory(name string) error { } // RenameFile : Rename the object (copy then delete). -func (cl *Client) RenameFile(source string, target string, isSymLink bool) error { +func (cl *Client) RenameFile( + ctx context.Context, + source string, + target string, + isSymLink bool, +) error { log.Trace("Client::RenameFile : %s -> %s", source, target) err := cl.renameObject( + ctx, renameObjectOptions{source: source, target: target, isSymLink: isSymLink}, ) if err != nil { @@ -549,7 +565,7 @@ func (cl *Client) RenameFile(source string, target string, isSymLink bool) error } // RenameDirectory : Rename the directory -func (cl *Client) RenameDirectory(source string, target string) error { +func (cl *Client) RenameDirectory(ctx context.Context, source string, target string) error { log.Trace("Client::RenameDirectory : %s -> %s", source, target) // TODO: should this fail when the target directory exists? @@ -563,7 +579,7 @@ func (cl *Client) RenameDirectory(source string, target string) error { var marker *string for !done { - sourceObjects, marker, err := cl.List(internal.ExtendDirName(source), marker, 0) + sourceObjects, marker, err := cl.List(ctx, internal.ExtendDirName(source), marker, 0) if err != nil { log.Err( "Client::RenameDirectory : Failed to list objects with prefix %s. Here's why: %v", @@ -577,9 +593,9 @@ func (cl *Client) RenameDirectory(source string, target string) error { srcPath := srcObject.Path dstPath := strings.Replace(srcPath, source, target, 1) if srcObject.IsDir() { - err = cl.RenameDirectory(srcPath, dstPath) + err = cl.RenameDirectory(ctx, srcPath, dstPath) } else { - err = cl.RenameFile(srcPath, dstPath, srcObject.IsSymlink()) //use sourceObjects to pass along symLink bool + err = cl.RenameFile(ctx, srcPath, dstPath, srcObject.IsSymlink()) //use sourceObjects to pass along symLink bool } if err != nil { log.Err( @@ -596,6 +612,7 @@ func (cl *Client) RenameDirectory(source string, target string) error { // Rename the current directory if cl.Config.enableDirMarker { err := cl.renameObject( + ctx, renameObjectOptions{source: source, target: target, isDir: true}, ) if err != nil { @@ -615,13 +632,13 @@ func (cl *Client) RenameDirectory(source string, target string) error { // GetAttr : Get attributes for a given file or folder. // If name is a file, it should not have a trailing slash. // If name is a directory, the trailing slash is optional. -func (cl *Client) GetAttr(name string) (*internal.ObjAttr, error) { +func (cl *Client) GetAttr(ctx context.Context, name string) (*internal.ObjAttr, error) { log.Trace("Client::GetAttr : name %s", name) // first let's suppose the caller is looking for a file // so if this was called with a trailing slash, don't look for an object if len(name) > 0 && name[len(name)-1] != '/' { - attr, err := cl.getFileAttr(name) + attr, err := cl.getFileAttr(ctx, name) if err == nil { return attr, err } @@ -633,29 +650,29 @@ func (cl *Client) GetAttr(name string) (*internal.ObjAttr, error) { // ensure a trailing slash dirName := internal.ExtendDirName(name) // now search for that as a directory - return cl.getDirectoryAttr(dirName) + return cl.getDirectoryAttr(ctx, dirName) } // Get attributes for the given file path. // Return ENOENT if there is no corresponding object in the bucket. // name should not have a trailing slash (or nothing will be found!). -func (cl *Client) getFileAttr(name string) (*internal.ObjAttr, error) { +func (cl *Client) getFileAttr(ctx context.Context, name string) (*internal.ObjAttr, error) { log.Trace("Client::getFileAttr : name %s", name) isSymlink := false - object, err := cl.headObject(name, isSymlink, false) + object, err := cl.headObject(ctx, name, isSymlink, false) if err == syscall.ENOENT && !cl.Config.disableSymlink { isSymlink = true - return cl.headObject(name, isSymlink, false) + return cl.headObject(ctx, name, isSymlink, false) } return object, err } -func (cl *Client) getDirectoryAttr(dirName string) (*internal.ObjAttr, error) { +func (cl *Client) getDirectoryAttr(ctx context.Context, dirName string) (*internal.ObjAttr, error) { log.Trace("Client::getDirectoryAttr : name %s", dirName) // Try seartching for the object directly if supported if cl.Config.enableDirMarker { - attr, err := cl.headObject(dirName, false, true) + attr, err := cl.headObject(ctx, dirName, false, true) if err == nil { return attr, err } @@ -664,7 +681,7 @@ func (cl *Client) getDirectoryAttr(dirName string) (*internal.ObjAttr, error) { // Otherwise, the cloud does not support directory markers, so use list // or the directory does exist but there is no marker for it, so look for an object // in the directory - objects, _, err := cl.List(dirName, nil, 1) + objects, _, err := cl.List(ctx, dirName, nil, 1) if err != nil { log.Err("Client::getDirectoryAttr : List(%s) failed. Here's why: %v", dirName, err) return nil, err @@ -682,7 +699,13 @@ func (cl *Client) getDirectoryAttr(dirName string) (*internal.ObjAttr, error) { // Download object data to a file handle. // Read starting at a byte offset from the start of the object, with length in bytes = count. // count = 0 reads to the end of the object. -func (cl *Client) ReadToFile(name string, offset int64, count int64, fi *os.File) error { +func (cl *Client) ReadToFile( + ctx context.Context, + name string, + offset int64, + count int64, + fi *os.File, +) error { log.Trace( "Client::ReadToFile : name %s, offset : %d, count %d -> file %s", name, @@ -693,7 +716,7 @@ func (cl *Client) ReadToFile(name string, offset int64, count int64, fi *os.File // If we are reading the entire object, then we can use a multipart download if !cl.Config.disableConcurrentDownload && offset == 0 && count == 0 { - err := cl.getObjectMultipartDownload(name, fi) + err := cl.getObjectMultipartDownload(ctx, name, fi) if err != nil { log.Err( "Client::ReadToFile : getObjectMultipartDownload(%s) failed. Here's why: %v", @@ -707,6 +730,7 @@ func (cl *Client) ReadToFile(name string, offset int64, count int64, fi *os.File // get object data objectDataReader, err := cl.getObject( + ctx, getObjectOptions{name: name, offset: offset, count: count}, ) if err != nil { @@ -750,6 +774,7 @@ func (cl *Client) ReadToFile(name string, offset int64, count int64, fi *os.File // len = 0 reads to the end of the object. // name is the file path func (cl *Client) ReadBuffer( + ctx context.Context, name string, offset int64, length int64, @@ -758,6 +783,7 @@ func (cl *Client) ReadBuffer( log.Trace("Client::ReadBuffer : name %s (%d+%d)", name, offset, length) // get object data objectDataReader, err := cl.getObject( + ctx, getObjectOptions{name: name, offset: offset, count: length, isSymLink: isSymlink}, ) if err != nil { @@ -782,10 +808,17 @@ func (cl *Client) ReadBuffer( // Reads starting at a byte offset from the start of the object, with length in bytes = len. // len = 0 reads to the end of the object. // name is the file path. -func (cl *Client) ReadInBuffer(name string, offset int64, length int64, data []byte) error { +func (cl *Client) ReadInBuffer( + ctx context.Context, + name string, + offset int64, + length int64, + data []byte, +) error { log.Trace("Client::ReadInBuffer : name %s offset %d len %d", name, offset, length) // get object data objectDataReader, err := cl.getObject( + ctx, getObjectOptions{name: name, offset: offset, count: length}, ) if err != nil { @@ -805,7 +838,12 @@ func (cl *Client) ReadInBuffer(name string, offset int64, length int64, data []b // Upload from a file handle to an object. // The metadata parameter is not used. -func (cl *Client) WriteFromFile(name string, metadata map[string]*string, fi *os.File) error { +func (cl *Client) WriteFromFile( + ctx context.Context, + name string, + metadata map[string]*string, + fi *os.File, +) error { isSymlink := getSymlinkBool(metadata) log.Trace("Client::WriteFromFile : file %s -> name %s", fi.Name(), name) @@ -828,6 +866,7 @@ func (cl *Client) WriteFromFile(name string, metadata map[string]*string, fi *os // upload file data err = cl.putObject( + ctx, putObjectOptions{name: name, objectData: fi, size: stat.Size(), isSymLink: isSymlink}, ) if err != nil { @@ -853,7 +892,12 @@ func (cl *Client) WriteFromFile(name string, metadata map[string]*string, fi *os // WriteFromBuffer : Upload from a buffer to an object. // name is the file path. -func (cl *Client) WriteFromBuffer(name string, metadata map[string]*string, data []byte) error { +func (cl *Client) WriteFromBuffer( + ctx context.Context, + name string, + metadata map[string]*string, + data []byte, +) error { log.Trace("Client::WriteFromBuffer : name %s", name) isSymlink := getSymlinkBool(metadata) @@ -862,6 +906,7 @@ func (cl *Client) WriteFromBuffer(name string, metadata map[string]*string, data // upload data to object // TODO: handle metadata with S3 err := cl.putObject( + ctx, putObjectOptions{ name: name, objectData: dataReader, @@ -876,10 +921,13 @@ func (cl *Client) WriteFromBuffer(name string, metadata map[string]*string, data } // GetFileBlockOffsets: store blocks ids and corresponding offsets. -func (cl *Client) GetFileBlockOffsets(name string) (*common.BlockOffsetList, error) { +func (cl *Client) GetFileBlockOffsets( + ctx context.Context, + name string, +) (*common.BlockOffsetList, error) { log.Trace("Client::GetFileBlockOffsets : name %s", name) blockList := common.BlockOffsetList{} - result, err := cl.headObject(name, false, false) + result, err := cl.headObject(ctx, name, false, false) if err != nil { log.Err("Client::GetFileBlockOffsets : Unable to headObject with name %v", name) return &blockList, err @@ -925,11 +973,11 @@ func (cl *Client) GetFileBlockOffsets(name string) (*common.BlockOffsetList, err // Truncate object to size in bytes. // name is the file path. -func (cl *Client) TruncateFile(name string, size int64) error { +func (cl *Client) TruncateFile(ctx context.Context, name string, size int64) error { log.Trace("Client::TruncateFile : Truncating %s to %dB.", name, size) // get object data - objectDataReader, err := cl.getObject(getObjectOptions{name: name}) + objectDataReader, err := cl.getObject(ctx, getObjectOptions{name: name}) if err != nil { log.Err("Client::TruncateFile : getObject(%s) failed. Here's why: %v", name, err) return err @@ -960,6 +1008,7 @@ func (cl *Client) TruncateFile(name string, size int64) error { // overwrite the object with the truncated data truncatedDataReader := bytes.NewReader(objectData) err = cl.putObject( + ctx, putObjectOptions{name: name, objectData: truncatedDataReader, size: int64(len(objectData))}, ) if err != nil { @@ -970,7 +1019,7 @@ func (cl *Client) TruncateFile(name string, size int64) error { } // Write : write data at given offset to an object -func (cl *Client) Write(options internal.WriteFileOptions) error { +func (cl *Client) Write(ctx context.Context, options internal.WriteFileOptions) error { name := options.Handle.Path offset := options.Offset data := options.Data @@ -981,7 +1030,7 @@ func (cl *Client) Write(options internal.WriteFileOptions) error { // tracks the case where our offset is great than our current file size (appending only - not modifying pre-existing data) var dataBuffer *[]byte - fileOffsets, err := cl.GetFileBlockOffsets(name) + fileOffsets, err := cl.GetFileBlockOffsets(ctx, name) if err != nil { return err } @@ -992,7 +1041,7 @@ func (cl *Client) Write(options internal.WriteFileOptions) error { // get the existing object data isSymlink := getSymlinkBool(options.Metadata) - oldData, _ := cl.ReadBuffer(name, 0, 0, isSymlink) + oldData, _ := cl.ReadBuffer(ctx, name, 0, 0, isSymlink) // update the data with the new data // if we're only overwriting existing data if int64(len(oldData)) >= offset+length { @@ -1019,7 +1068,7 @@ func (cl *Client) Write(options internal.WriteFileOptions) error { } // WriteFromBuffer should be able to handle the case where now the block is too big and gets split into multiple parts - err := cl.WriteFromBuffer(name, options.Metadata, *dataBuffer) + err := cl.WriteFromBuffer(ctx, name, options.Metadata, *dataBuffer) if err != nil { log.Err("Client::Write : Failed to upload to object. Here's why: %v ", name, err) return err @@ -1039,7 +1088,7 @@ func (cl *Client) Write(options internal.WriteFileOptions) error { oldDataBuffer := make([]byte, oldDataSize+newBufferSize) if !appendOnly { // fetch the parts that will be impacted by the new changes so we can overwrite them - err = cl.ReadInBuffer(name, fileOffsets.BlockList[index].StartIndex, oldDataSize, oldDataBuffer) + err = cl.ReadInBuffer(ctx, name, fileOffsets.BlockList[index].StartIndex, oldDataSize, oldDataBuffer) if err != nil { log.Err("BlockBlob::Write : Failed to read data in buffer %s [%s]", name, err.Error()) } @@ -1047,7 +1096,7 @@ func (cl *Client) Write(options internal.WriteFileOptions) error { // this gives us where the offset with respect to the buffer that holds our old data - so we can start writing the new data blockOffset := offset - fileOffsets.BlockList[index].StartIndex copy(oldDataBuffer[blockOffset:], data) - err := cl.stageAndCommitModifiedBlocks(name, oldDataBuffer, fileOffsets) + err := cl.stageAndCommitModifiedBlocks(ctx, name, oldDataBuffer, fileOffsets) return err } @@ -1086,6 +1135,7 @@ func (cl *Client) createNewBlocks(blockList *common.BlockOffsetList, offset, len } func (cl *Client) stageAndCommitModifiedBlocks( + ctx context.Context, name string, data []byte, offsetList *common.BlockOffsetList, @@ -1102,10 +1152,14 @@ func (cl *Client) stageAndCommitModifiedBlocks( } } - return cl.StageAndCommit(name, offsetList) + return cl.StageAndCommit(ctx, name, offsetList) } -func (cl *Client) StageAndCommit(name string, bol *common.BlockOffsetList) error { +func (cl *Client) StageAndCommit( + ctx context.Context, + name string, + bol *common.BlockOffsetList, +) error { // lock on the object name so that no stage and commit race condition occur causing failure objectMtx := cl.blockLocks.GetLock(name) objectMtx.Lock() @@ -1142,7 +1196,7 @@ func (cl *Client) StageAndCommit(name string, bol *common.BlockOffsetList) error var err error if combineBlocks { - bol.BlockList, err = cl.combineSmallBlocks(name, bol.BlockList) + bol.BlockList, err = cl.combineSmallBlocks(ctx, name, bol.BlockList) if err != nil { log.Err("Client::StageAndCommit : Failed to combine small blocks: %v ", name, err) return err @@ -1150,7 +1204,6 @@ func (cl *Client) StageAndCommit(name string, bol *common.BlockOffsetList) error } //struct for starting a multipart upload - ctx := context.Background() key := cl.getKey(name, false, false) //send command to start copy and get the upload id as it is needed later @@ -1261,7 +1314,7 @@ func (cl *Client) StageAndCommit(name string, bol *common.BlockOffsetList) error "Client::StageAndCommit : Attempting to abort upload due to error: ", err.Error(), ) - abortErr := cl.abortMultipartUpload(key, uploadID) + abortErr := cl.abortMultipartUpload(ctx, key, uploadID) return errors.Join(err, abortErr) } @@ -1296,7 +1349,7 @@ func (cl *Client) StageAndCommit(name string, bol *common.BlockOffsetList) error }) if err != nil { log.Info("Client::StageAndCommit : Attempting to abort upload due to error: ", err.Error()) - abortErr := cl.abortMultipartUpload(key, uploadID) + abortErr := cl.abortMultipartUpload(ctx, key, uploadID) return errors.Join(err, abortErr) } @@ -1307,6 +1360,7 @@ func (cl *Client) StageAndCommit(name string, bol *common.BlockOffsetList) error // than the smallest size for a part in AWS, which is 5 MB. Blocks smaller than 5MB will be combined with the // next block in the list. func (cl *Client) combineSmallBlocks( + ctx context.Context, name string, blockList []*common.Block, ) ([]*common.Block, error) { @@ -1330,7 +1384,7 @@ func (cl *Client) combineSmallBlocks( var addData []byte // If there is no data in the block and it is not truncated, we need to get it from the cloud. Otherwise we can just copy it. if len(blk.Data) == 0 && !blk.Truncated() { - result, err := cl.getObject(getObjectOptions{name: name, offset: blk.StartIndex, count: blk.EndIndex - blk.StartIndex}) + result, err := cl.getObject(ctx, getObjectOptions{name: name, offset: blk.StartIndex, count: blk.EndIndex - blk.StartIndex}) if err != nil { log.Err("Client::combineSmallBlocks : Unable to get object with error: ", err.Error()) return nil, err @@ -1362,8 +1416,8 @@ func (cl *Client) combineSmallBlocks( return newBlockList, nil } -func (cl *Client) GetUsedSize() (uint64, error) { - headBucketOutput, err := cl.headBucket(cl.Config.AuthConfig.BucketName) +func (cl *Client) GetUsedSize(ctx context.Context) (uint64, error) { + headBucketOutput, err := cl.headBucket(ctx, cl.Config.AuthConfig.BucketName) if err != nil { return 0, err } @@ -1391,10 +1445,13 @@ func (cl *Client) GetUsedSize() (uint64, error) { return bucketSizeBytes, nil } -func (cl *Client) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error) { +func (cl *Client) GetCommittedBlockList( + ctx context.Context, + name string, +) (*internal.CommittedBlockList, error) { log.Trace("Client::GetCommittedBlockList : name %s", name) blockList := make(internal.CommittedBlockList, 0) - result, err := cl.headObject(name, false, false) + result, err := cl.headObject(ctx, name, false, false) if err != nil { log.Err("Client::GetCommittedBlockList : Unable to headObject with name %v", name) return nil, err @@ -1437,11 +1494,10 @@ func (cl *Client) GetCommittedBlockList(name string) (*internal.CommittedBlockLi } // CommitBlocks : Initiates and completes an S3 multipart upload using locally cached blocks. -func (cl *Client) CommitBlocks(name string, blockList []string) error { +func (cl *Client) CommitBlocks(ctx context.Context, name string, blockList []string) error { log.Trace("Client::CommitBlocks: name %s, %d blocks", name, len(blockList)) //struct for starting a multipart upload - ctx := context.Background() key := cl.getKey(name, false, false) // Retrieve cached blocks for this file @@ -1455,6 +1511,7 @@ func (cl *Client) CommitBlocks(name string, blockList []string) error { name, ) return cl.putObject( + ctx, putObjectOptions{name: name, objectData: bytes.NewReader([]byte{}), size: 0}, ) } @@ -1576,7 +1633,7 @@ func (cl *Client) CommitBlocks(name string, blockList []string) error { name, uploadErr, ) - _ = cl.abortMultipartUpload(key, uploadID) // Attempt to clean up S3 + _ = cl.abortMultipartUpload(ctx, key, uploadID) // Attempt to clean up S3 cl.cleanupStagedBlocks(name) return uploadErr } @@ -1598,7 +1655,7 @@ func (cl *Client) CommitBlocks(name string, blockList []string) error { name, err, ) - _ = cl.abortMultipartUpload(key, uploadID) + _ = cl.abortMultipartUpload(ctx, key, uploadID) cl.cleanupStagedBlocks(name) return parseS3Err(err, fmt.Sprintf("CompleteMultipartUpload(%s)", name)) } diff --git a/component/s3storage/client_test.go b/component/s3storage/client_test.go index 18e6527be..d25be31fa 100644 --- a/component/s3storage/client_test.go +++ b/component/s3storage/client_test.go @@ -563,7 +563,7 @@ func (s *clientTestSuite) TestGetRegionEndpoint() { func (s *clientTestSuite) TestListBuckets() { defer s.cleanupTest() // TODO: generalize this test by creating, listing, then destroying a bucket - buckets, err := s.client.ListBuckets() + buckets, err := s.client.ListBuckets(context.TODO()) s.assert.NoError(err) s.assert.Contains(buckets, storageTestConfigurationParameters.BucketName) } @@ -581,7 +581,7 @@ func (s *clientTestSuite) TestDefaultBucketName() { ) err := s.setupTestHelper(config, false) s.assert.NoError(err) - buckets, _ := s.client.ListBuckets() + buckets, _ := s.client.ListBuckets(ctx) s.assert.Contains(buckets, s.client.Config.AuthConfig.BucketName) } @@ -592,8 +592,8 @@ func (s *clientTestSuite) TestSetPrefixPath() { fileName := generateFileName() err := s.client.SetPrefixPath(prefix) - s.assert.NoError(err) //stub - err = s.client.CreateFile(fileName, os.FileMode(0)) // create file uses prefix + s.assert.NoError(err) //stub + err = s.client.CreateFile(ctx, fileName, os.FileMode(0)) // create file uses prefix s.assert.NoError(err) // object should be at prefix @@ -609,7 +609,7 @@ func (s *clientTestSuite) TestCreateFile() { // setup name := generateFileName() - err := s.client.CreateFile(name, os.FileMode(0)) + err := s.client.CreateFile(ctx, name, os.FileMode(0)) s.assert.NoError(err) // file should be in bucket @@ -625,7 +625,7 @@ func (s *clientTestSuite) TestCreateDirectory() { // setup name := generateDirectoryName() - err := s.client.CreateDirectory(name) + err := s.client.CreateDirectory(ctx, name) s.assert.NoError(err) } func (s *clientTestSuite) TestCreateLink() { @@ -642,7 +642,7 @@ func (s *clientTestSuite) TestCreateLink() { s.assert.NoError(err) source := generateFileName() - err = s.client.CreateLink(source, target, true) + err = s.client.CreateLink(ctx, source, target, true) s.assert.NoError(err) source = s.client.getKey(source, true, false) @@ -668,7 +668,7 @@ func (s *clientTestSuite) TestReadLink() { source := generateFileName() - err := s.client.CreateLink(source, target, true) + err := s.client.CreateLink(ctx, source, target, true) s.assert.NoError(err) source = s.client.getKey(source, true, false) @@ -696,7 +696,7 @@ func (s *clientTestSuite) TestDeleteLink() { source := generateFileName() - err := s.client.CreateLink(source, target, true) + err := s.client.CreateLink(ctx, source, target, true) s.assert.NoError(err) source = s.client.getKey(source, true, false) @@ -733,7 +733,7 @@ func (s *clientTestSuite) TestDeleteLinks() { sources[i] = generateFileName() targets[i] = generateFileName() - err := s.client.CreateLink(folder+sources[i], targets[i], true) + err := s.client.CreateLink(ctx, folder+sources[i], targets[i], true) s.assert.NoError(err) sources[i] = s.client.getKey(sources[i], true, false) @@ -795,7 +795,7 @@ func (s *clientTestSuite) TestDeleteFile() { }) s.assert.NoError(err) - err = s.client.DeleteFile(name) + err = s.client.DeleteFile(ctx, name) s.assert.NoError(err) // This is similar to the s3 bucket command, use getobject for now @@ -821,7 +821,7 @@ func (s *clientTestSuite) TestDeleteDirectory() { }) s.assert.NoError(err) - err = s.client.DeleteDirectory(dirName) + err = s.client.DeleteDirectory(ctx, dirName) s.assert.NoError(err) // file in directory should no longer be there @@ -845,7 +845,7 @@ func (s *clientTestSuite) TestRenameFile() { s.assert.NoError(err) dst := generateFileName() - err = s.client.RenameFile(src, dst, false) + err = s.client.RenameFile(ctx, src, dst, false) s.assert.NoError(err) // Src should not be in the account @@ -870,7 +870,7 @@ func (s *clientTestSuite) TestRenameFileError() { src := generateFileName() dst := generateFileName() - err := s.client.RenameFile(src, dst, false) + err := s.client.RenameFile(ctx, src, dst, false) s.assert.EqualError(err, syscall.ENOENT.Error()) // Src should not be in the account @@ -901,7 +901,7 @@ func (s *clientTestSuite) TestRenameDirectory() { s.assert.NoError(err) dstDir := generateDirectoryName() - err = s.client.RenameDirectory(srcDir, dstDir) + err = s.client.RenameDirectory(ctx, srcDir, dstDir) s.assert.NoError(err) // file in srcDir should no longer be there @@ -931,7 +931,7 @@ func (s *clientTestSuite) TestGetAttrDir() { }) s.assert.NoError(err) - attr, err := s.client.GetAttr(dirName) + attr, err := s.client.GetAttr(ctx, dirName) s.assert.NoError(err) s.assert.NotNil(attr) s.assert.True(attr.IsDir()) @@ -949,7 +949,7 @@ func (s *clientTestSuite) TestGetAttrDirWithOnlyFile() { }) s.assert.NoError(err) - attr, err := s.client.GetAttr(dirName) + attr, err := s.client.GetAttr(ctx, dirName) s.assert.NoError(err) s.assert.NotNil(attr) s.assert.True(attr.IsDir()) @@ -970,7 +970,7 @@ func (s *clientTestSuite) TestGetAttrFile() { }) s.assert.NoError(err) - before, err := s.client.GetAttr(name) + before, err := s.client.GetAttr(ctx, name) // file info s.assert.NoError(err) @@ -994,7 +994,7 @@ func (s *clientTestSuite) TestGetAttrFile() { }) s.assert.NoError(err) - after, err := s.client.GetAttr(name) + after, err := s.client.GetAttr(ctx, name) s.assert.NoError(err) s.assert.NotNil(after.Mtime) @@ -1006,7 +1006,7 @@ func (s *clientTestSuite) TestGetAttrError() { name := generateFileName() // non existent file should throw error - _, err := s.client.GetAttr(name) + _, err := s.client.GetAttr(ctx, name) s.assert.Error(err) s.assert.EqualValues(syscall.ENOENT, err) } @@ -1023,9 +1023,10 @@ func (s *clientTestSuite) TestList() { ChecksumAlgorithm: s.client.Config.checksumAlgorithm, }) s.assert.NoError(err) + ctx := context.Background() // a/c2 c2 := base + "/c2" - _, err = s.awsS3Client.PutObject(context.Background(), &s3.PutObjectInput{ + _, err = s.awsS3Client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.client.Config.AuthConfig.BucketName), Key: aws.String(c2), ChecksumAlgorithm: s.client.Config.checksumAlgorithm, @@ -1033,7 +1034,7 @@ func (s *clientTestSuite) TestList() { s.assert.NoError(err) // ab/c1 abc1 := base + "b/c1" - _, err = s.awsS3Client.PutObject(context.Background(), &s3.PutObjectInput{ + _, err = s.awsS3Client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.client.Config.AuthConfig.BucketName), Key: aws.String(abc1), ChecksumAlgorithm: s.client.Config.checksumAlgorithm, @@ -1041,7 +1042,7 @@ func (s *clientTestSuite) TestList() { s.assert.NoError(err) // ac ac := base + "c" - _, err = s.awsS3Client.PutObject(context.Background(), &s3.PutObjectInput{ + _, err = s.awsS3Client.PutObject(ctx, &s3.PutObjectInput{ Bucket: aws.String(s.client.Config.AuthConfig.BucketName), Key: aws.String(ac), ChecksumAlgorithm: s.client.Config.checksumAlgorithm, @@ -1050,7 +1051,7 @@ func (s *clientTestSuite) TestList() { // with trailing "/" should return only the directory c1 and the file c2 baseTrail := base + "/" - objects, _, err := s.client.List(baseTrail, nil, 0) + objects, _, err := s.client.List(ctx, baseTrail, nil, 0) s.assert.NoError(err) s.assert.NotNil(objects) s.assert.Len(objects, 2) @@ -1062,7 +1063,7 @@ func (s *clientTestSuite) TestList() { // without trailing "/" only get file ac // if not including the trailing "/", List will return any files with the given prefix // but no directories - objects, _, err = s.client.List(base, nil, 0) + objects, _, err = s.client.List(ctx, base, nil, 0) s.assert.NoError(err) s.assert.NotNil(objects) s.assert.Len(objects, 1) @@ -1070,7 +1071,7 @@ func (s *clientTestSuite) TestList() { s.assert.False(objects[0].IsDir()) // When listing the root, List should not include the root - objects, _, err = s.client.List("", nil, 0) + objects, _, err = s.client.List(ctx, "", nil, 0) s.assert.NoError(err) s.assert.NotNil(objects) s.assert.NotEmpty(objects) @@ -1098,7 +1099,7 @@ func (s *clientTestSuite) TestReadToFile() { s.assert.NoError(err) defer os.Remove(f.Name()) - err = s.client.ReadToFile(name, 0, 0, f) + err = s.client.ReadToFile(ctx, name, 0, 0, f) s.assert.NoError(err) // file content should match generated body @@ -1132,7 +1133,7 @@ func (s *clientTestSuite) TestReadToFileRanged() { s.assert.NoError(err) defer os.Remove(f.Name()) - err = s.client.ReadToFile(name, 0, int64(bodyLen), f) + err = s.client.ReadToFile(ctx, name, 0, int64(bodyLen), f) s.assert.NoError(err) // file content should match generated body @@ -1169,7 +1170,7 @@ func (s *clientTestSuite) TestReadToFileNoMultipart() { s.assert.NoError(err) defer os.Remove(f.Name()) - err = s.client.ReadToFile(name, 0, 0, f) + err = s.client.ReadToFile(ctx, name, 0, 0, f) s.assert.NoError(err) // file content should match generated body @@ -1199,7 +1200,7 @@ func (s *clientTestSuite) TestReadBuffer() { }) s.assert.NoError(err) - result, err := s.client.ReadBuffer(name, 0, int64(bodyLen), false) + result, err := s.client.ReadBuffer(ctx, name, 0, int64(bodyLen), false) // result should match generated body s.assert.NoError(err) @@ -1223,7 +1224,7 @@ func (s *clientTestSuite) TestReadInBuffer() { outputLen := rand.IntN(bodyLen-1) + 1 // minimum buffer length of 1 output := make([]byte, outputLen) - err = s.client.ReadInBuffer(name, 0, int64(outputLen), output) + err = s.client.ReadInBuffer(ctx, name, 0, int64(outputLen), output) // read in buffer should match first outputLen characters of generated body s.assert.NoError(err) @@ -1245,7 +1246,7 @@ func (s *clientTestSuite) TestWriteFromFile() { s.assert.Equal(bodyLen, outputLen) var options internal.WriteFileOptions //stub - err = s.client.WriteFromFile(name, options.Metadata, f) + err = s.client.WriteFromFile(ctx, name, options.Metadata, f) s.assert.NoError(err) f.Close() @@ -1276,7 +1277,7 @@ func (s *clientTestSuite) TestWriteFromBuffer() { var options internal.WriteFileOptions //stub - err := s.client.WriteFromBuffer(name, options.Metadata, body) + err := s.client.WriteFromBuffer(ctx, name, options.Metadata, body) s.assert.NoError(err) result, err := s.awsS3Client.GetObject(context.Background(), &s3.GetObjectInput{ @@ -1309,7 +1310,7 @@ func (s *clientTestSuite) TestTruncateFile() { s.assert.NoError(err) size := rand.IntN(bodyLen-1) + 1 // minimum size of 1 - err = s.client.TruncateFile(name, int64(size)) + err = s.client.TruncateFile(ctx, name, int64(size)) s.assert.NoError(err) result, err := s.awsS3Client.GetObject(context.Background(), &s3.GetObjectInput{ @@ -1344,7 +1345,10 @@ func (s *clientTestSuite) TestWrite() { offset := rand.IntN(bodyLen-1) + 1 // minimum offset of 1 newData := []byte(randomString(bodyLen - offset)) h := handlemap.NewHandle(name) - err = s.client.Write(internal.WriteFileOptions{Handle: h, Offset: int64(offset), Data: newData}) + err = s.client.Write( + ctx, + internal.WriteFileOptions{Handle: h, Offset: int64(offset), Data: newData}, + ) s.assert.NoError(err) result, err := s.awsS3Client.GetObject(context.Background(), &s3.GetObjectInput{ @@ -1368,10 +1372,10 @@ func (s *clientTestSuite) TestGetCommittedBlockListSmallFile() { bodyLen := 1024 body := []byte(randomString(bodyLen)) - err := s.client.WriteFromBuffer(name, nil, body) + err := s.client.WriteFromBuffer(ctx, name, nil, body) s.assert.NoError(err) - blockList, err := s.client.GetCommittedBlockList(name) + blockList, err := s.client.GetCommittedBlockList(ctx, name) s.assert.NoError(err) s.assert.NotNil(blockList) @@ -1385,10 +1389,10 @@ func (s *clientTestSuite) TestGetCommittedBlockListMultipartFile() { bodyLen := int(partSize*2 + partSize/2) body := randomString(bodyLen) - err := s.client.WriteFromBuffer(name, nil, []byte(body)) + err := s.client.WriteFromBuffer(ctx, name, nil, []byte(body)) s.assert.NoError(err) - blockList, err := s.client.GetCommittedBlockList(name) + blockList, err := s.client.GetCommittedBlockList(ctx, name) s.assert.NoError(err) s.assert.NotNil(blockList) @@ -1418,7 +1422,7 @@ func (s *clientTestSuite) TestGetCommittedBlockListNonExistentFile() { defer s.cleanupTest() name := generateFileName() - blockList, err := s.client.GetCommittedBlockList(name) + blockList, err := s.client.GetCommittedBlockList(ctx, name) s.assert.Error(err) s.assert.Equal(syscall.ENOENT, err) diff --git a/component/s3storage/config.go b/component/s3storage/config.go index abe768eb4..607bd365b 100644 --- a/component/s3storage/config.go +++ b/component/s3storage/config.go @@ -28,6 +28,7 @@ package s3storage import ( "errors" "fmt" + "time" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/common/config" @@ -55,6 +56,7 @@ type Options struct { UsePathStyle bool `config:"use-path-style" yaml:"use-path-style,omitempty"` DisableUsage bool `config:"disable-usage" yaml:"disable-usage,omitempty"` EnableDirMarker bool `config:"enable-dir-marker" yaml:"enable-dir-marker,omitempty"` + HealthCheckIntervalSec int `config:"health-check-interval-sec" yaml:"health-check-interval-sec,omitempty"` } type ConfigSecrets struct { @@ -62,6 +64,11 @@ type ConfigSecrets struct { SecretKey *memguard.Enclave } +const ( + defaultHealthCheckInterval = 10 * time.Second + maxHealthCheckInterval = 90 * time.Second +) + // ParseAndValidateConfig : Parse and validate config func ParseAndValidateConfig(s3 *S3Storage, opt Options, secrets ConfigSecrets) error { log.Trace("ParseAndValidateConfig : Parsing config") @@ -153,6 +160,27 @@ func ParseAndValidateConfig(s3 *S3Storage, opt Options, secrets ConfigSecrets) e } s3.stConfig.disableSymlink = !enableSymlinks + s3.stConfig.healthCheckInterval = defaultHealthCheckInterval + if config.IsSet("s3storage.health-check-interval-sec") { + specifiedInterval := time.Duration(opt.HealthCheckIntervalSec) * time.Second + switch { + case specifiedInterval < 1*time.Second: + log.Warn( + "S3storage : health-check-interval-sec=%d... using 1s instead", + opt.HealthCheckIntervalSec, + ) + s3.stConfig.healthCheckInterval = 1 * time.Second + case specifiedInterval > maxHealthCheckInterval: + log.Warn( + "S3storage : health-check-interval-sec=%d... using %ds instead", + opt.HealthCheckIntervalSec, + maxHealthCheckInterval.Seconds(), + ) + default: + s3.stConfig.healthCheckInterval = specifiedInterval + } + } + // TODO: add more config options to customize AWS SDK behavior and import them here return nil diff --git a/component/s3storage/connection.go b/component/s3storage/connection.go index da4f4212d..2b7bf5350 100644 --- a/component/s3storage/connection.go +++ b/component/s3storage/connection.go @@ -26,8 +26,10 @@ package s3storage import ( + "context" "net/url" "os" + "time" "github.com/Seagate/cloudfuse/common" "github.com/Seagate/cloudfuse/internal" @@ -55,6 +57,7 @@ type Config struct { disableSymlink bool disableUsage bool enableDirMarker bool + healthCheckInterval time.Duration } // TODO: move s3AuthConfig to s3auth.go @@ -82,43 +85,60 @@ type S3Connection interface { Configure(cfg Config) error UpdateConfig(cfg Config) error - ListBuckets() ([]string, error) - ListAuthorizedBuckets() ([]string, error) + ConnectionOkay(ctx context.Context) error + ListBuckets(ctx context.Context) ([]string, error) + ListAuthorizedBuckets(ctx context.Context) ([]string, error) // This is just for test, shall not be used otherwise SetPrefixPath(string) error - CreateFile(name string, mode os.FileMode) error - CreateDirectory(name string) error - CreateLink(source string, target string, isSymlink bool) error + CreateFile(ctx context.Context, name string, mode os.FileMode) error + CreateDirectory(ctx context.Context, name string) error + CreateLink(ctx context.Context, source string, target string, isSymlink bool) error - DeleteFile(name string) error - DeleteDirectory(name string) error + DeleteFile(ctx context.Context, name string) error + DeleteDirectory(ctx context.Context, name string) error - RenameFile(string, string, bool) error - RenameDirectory(string, string) error + RenameFile(ctx context.Context, source string, target string, isSymLink bool) error + RenameDirectory(ctx context.Context, source string, target string) error - GetAttr(name string) (attr *internal.ObjAttr, err error) + GetAttr(ctx context.Context, name string) (attr *internal.ObjAttr, err error) // Standard operations to be supported by any account type - List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error) - - ReadToFile(name string, offset int64, count int64, fi *os.File) error - ReadBuffer(name string, offset int64, length int64, isSymlink bool) ([]byte, error) - ReadInBuffer(name string, offset int64, length int64, data []byte) error - - WriteFromFile(name string, metadata map[string]*string, fi *os.File) error - WriteFromBuffer(name string, metadata map[string]*string, data []byte) error - Write(options internal.WriteFileOptions) error - GetFileBlockOffsets(name string) (*common.BlockOffsetList, error) - - TruncateFile(string, int64) error - StageAndCommit(name string, bol *common.BlockOffsetList) error - - GetCommittedBlockList(string) (*internal.CommittedBlockList, error) - StageBlock(string, []byte, string) error - CommitBlocks(string, []string) error + List( + ctx context.Context, + prefix string, + marker *string, + count int32, + ) ([]*internal.ObjAttr, *string, error) + + ReadToFile(ctx context.Context, name string, offset int64, count int64, fi *os.File) error + ReadBuffer( + ctx context.Context, + name string, + offset int64, + length int64, + isSymlink bool, + ) ([]byte, error) + ReadInBuffer(ctx context.Context, name string, offset int64, length int64, data []byte) error + + WriteFromFile(ctx context.Context, name string, metadata map[string]*string, fi *os.File) error + WriteFromBuffer( + ctx context.Context, + name string, + metadata map[string]*string, + data []byte, + ) error + Write(ctx context.Context, options internal.WriteFileOptions) error + GetFileBlockOffsets(ctx context.Context, name string) (*common.BlockOffsetList, error) + + TruncateFile(ctx context.Context, name string, size int64) error + StageAndCommit(ctx context.Context, name string, bol *common.BlockOffsetList) error + + GetCommittedBlockList(ctx context.Context, name string) (*internal.CommittedBlockList, error) + StageBlock(name string, data []byte, id string) error + CommitBlocks(ctx context.Context, name string, blockList []string) error NewCredentialKey(_, _ string) error - GetUsedSize() (uint64, error) + GetUsedSize(ctx context.Context) (uint64, error) } diff --git a/component/s3storage/s3storage.go b/component/s3storage/s3storage.go index dc31ba95d..b6f91d862 100644 --- a/component/s3storage/s3storage.go +++ b/component/s3storage/s3storage.go @@ -29,6 +29,7 @@ import ( "context" "errors" "fmt" + "sync" "sync/atomic" "syscall" "time" @@ -48,6 +49,16 @@ type S3Storage struct { internal.BaseComponent Storage S3Connection stConfig Config + state connectionState + ctx context.Context + cancelFn context.CancelFunc +} + +type connectionState struct { + sync.Mutex + lastConnectionAttempt *time.Time + firstOffline *time.Time + retryTicker *time.Ticker } const compName = "s3storage" @@ -123,6 +134,9 @@ func (s3 *S3Storage) Configure(isParent bool) error { log.Err("S3Storage::Configure : Failed to validate storage account [%s]", err.Error()) return err } + // first connection attempt is now + currentTime := time.Now() + s3.state.lastConnectionAttempt = ¤tTime return nil } @@ -167,6 +181,16 @@ func (s3 *S3Storage) Start(ctx context.Context) error { // create stats collector for s3storage s3StatsCollector = stats_manager.NewStatsCollector(s3.Name()) log.Debug("Starting s3 stats collector") + // create a shared context for all cloud operations, with ability to cancel + s3.ctx, s3.cancelFn = context.WithCancel(ctx) + // create the retry ticker + s3.state.retryTicker = time.NewTicker(s3.stConfig.healthCheckInterval) + s3.state.retryTicker.Stop() // stop it for now, we will start it when we are offline + go func() { + for range s3.state.retryTicker.C { + s3.CloudConnected() + } + }() return nil } @@ -178,13 +202,73 @@ func (s3 *S3Storage) Stop() error { return nil } +// Online check +func (s3 *S3Storage) CloudConnected() bool { + log.Trace("S3Storage::CloudConnected") + connected := s3.state.firstOffline == nil + // don't check the connection when it's up, or if we are not ready to retry + if connected || !s3.timeToRetry() { + return connected + } + // check connection + ctx, cancelFun := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancelFun() + err := s3.Storage.ConnectionOkay(ctx) + nowConnected := s3.updateConnectionState(err) + return nowConnected +} + +func (s3 *S3Storage) timeToRetry() bool { + timeSinceLastAttempt := time.Since(*s3.state.lastConnectionAttempt) + switch { + case timeSinceLastAttempt < s3.stConfig.healthCheckInterval: + // minimum delay before retrying + return false + case timeSinceLastAttempt > 90*time.Second: + // maximum delay + return true + default: + // when between the minimum and maximum delay, we use an exponential backoff + timeOfflineAtLastAttempt := s3.state.lastConnectionAttempt.Sub(*s3.state.firstOffline) + return timeSinceLastAttempt > timeOfflineAtLastAttempt + } +} + +func (s3 *S3Storage) updateConnectionState(err error) bool { + s3.state.Lock() + defer s3.state.Unlock() + currentTime := time.Now() + s3.state.lastConnectionAttempt = ¤tTime + connected := !errors.Is(err, &common.CloudUnreachableError{}) + wasConnected := s3.state.firstOffline == nil + stateChanged := connected != wasConnected + if stateChanged { + log.Warn("S3Storage::updateConnectionState : connected is now: %t", connected) + if connected { + s3.state.firstOffline = nil + // reset the context to allow new requests + s3.ctx, s3.cancelFn = context.WithCancel(context.Background()) + // stop the retry ticker + s3.state.retryTicker.Stop() + } else { + s3.state.firstOffline = ¤tTime + // cancel all outstanding requests + s3.cancelFn() + log.Warn("S3Storage::updateConnectionState : cancelled all outstanding requests") + // reset the ticker to retry the connection + s3.state.retryTicker.Reset(s3.stConfig.healthCheckInterval) + } + } + return connected +} + // ------------------------- Bucket listing ------------------------------------------- func (s3 *S3Storage) ListBuckets() ([]string, error) { - return s3.Storage.ListBuckets() + return s3.Storage.ListBuckets(s3.ctx) } func (s3 *S3Storage) ListAuthorizedBuckets() ([]string, error) { - return s3.Storage.ListAuthorizedBuckets() + return s3.Storage.ListAuthorizedBuckets(s3.ctx) } // ------------------------- Core Operations ------------------------------------------- @@ -193,7 +277,10 @@ func (s3 *S3Storage) ListAuthorizedBuckets() ([]string, error) { func (s3 *S3Storage) CreateDir(options internal.CreateDirOptions) error { log.Trace("S3Storage::CreateDir : %s", options.Name) - err := s3.Storage.CreateDirectory(internal.TruncateDirName(options.Name)) + err := s3.Storage.CreateDirectory(s3.ctx, internal.TruncateDirName(options.Name)) + if s3.stConfig.enableDirMarker { + s3.updateConnectionState(err) + } if err == nil { s3StatsCollector.PushEvents( @@ -210,7 +297,8 @@ func (s3 *S3Storage) CreateDir(options internal.CreateDirOptions) error { func (s3 *S3Storage) DeleteDir(options internal.DeleteDirOptions) error { log.Trace("S3Storage::DeleteDir : %s", options.Name) - err := s3.Storage.DeleteDirectory(internal.TruncateDirName(options.Name)) + err := s3.Storage.DeleteDirectory(s3.ctx, internal.TruncateDirName(options.Name)) + s3.updateConnectionState(err) if err == nil { s3StatsCollector.PushEvents(deleteDir, options.Name, nil) @@ -234,7 +322,8 @@ func formatListDirName(path string) string { func (s3 *S3Storage) IsDirEmpty(options internal.IsDirEmptyOptions) bool { log.Trace("S3Storage::IsDirEmpty : %s", options.Name) // List up to two objects, since one could be the directory with a trailing slash - list, _, err := s3.Storage.List(formatListDirName(options.Name), nil, 2) + list, _, err := s3.Storage.List(s3.ctx, formatListDirName(options.Name), nil, 2) + s3.updateConnectionState(err) if err != nil { log.Err("S3Storage::IsDirEmpty : error listing [%s]", err) return false @@ -263,7 +352,8 @@ func (s3 *S3Storage) StreamDir( entriesRemaining = maxResultsPerListCall } for entriesRemaining > 0 { - newList, nextMarker, err := s3.Storage.List(path, marker, entriesRemaining) + newList, nextMarker, err := s3.Storage.List(s3.ctx, path, marker, entriesRemaining) + s3.updateConnectionState(err) if err != nil { log.Err("S3Storage::StreamDir : %s Failed to read dir [%s]", options.Name, err) return objectList, "", err @@ -312,7 +402,8 @@ func (s3 *S3Storage) RenameDir(options internal.RenameDirOptions) error { options.Src = internal.TruncateDirName(options.Src) options.Dst = internal.TruncateDirName(options.Dst) - err := s3.Storage.RenameDirectory(options.Src, options.Dst) + err := s3.Storage.RenameDirectory(s3.ctx, options.Src, options.Dst) + s3.updateConnectionState(err) if err == nil { s3StatsCollector.PushEvents( @@ -337,7 +428,8 @@ func (s3 *S3Storage) CreateFile(options internal.CreateFileOptions) (*handlemap. return nil, syscall.EFAULT } - err := s3.Storage.CreateFile(options.Name, options.Mode) + err := s3.Storage.CreateFile(s3.ctx, options.Name, options.Mode) + s3.updateConnectionState(err) if err != nil { return nil, err } @@ -358,7 +450,8 @@ func (s3 *S3Storage) CreateFile(options internal.CreateFileOptions) (*handlemap. func (s3 *S3Storage) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error) { log.Trace("S3Storage::OpenFile : %s", options.Name) - attr, err := s3.Storage.GetAttr(options.Name) + attr, err := s3.Storage.GetAttr(s3.ctx, options.Name) + s3.updateConnectionState(err) if err != nil { return nil, err } @@ -391,7 +484,8 @@ func (s3 *S3Storage) CloseFile(options internal.CloseFileOptions) error { func (s3 *S3Storage) DeleteFile(options internal.DeleteFileOptions) error { log.Trace("S3Storage::DeleteFile : %s", options.Name) - err := s3.Storage.DeleteFile(options.Name) + err := s3.Storage.DeleteFile(s3.ctx, options.Name) + s3.updateConnectionState(err) if err == nil { s3StatsCollector.PushEvents(deleteFile, options.Name, nil) @@ -404,8 +498,9 @@ func (s3 *S3Storage) DeleteFile(options internal.DeleteFileOptions) error { func (s3 *S3Storage) RenameFile(options internal.RenameFileOptions) error { log.Trace("S3Storage::RenameFile : %s to %s", options.Src, options.Dst) - err := s3.Storage.RenameFile(options.Src, options.Dst, false) + err := s3.Storage.RenameFile(s3.ctx, options.Src, options.Dst, false) + s3.updateConnectionState(err) if err == nil { s3StatsCollector.PushEvents( renameFile, @@ -434,7 +529,14 @@ func (s3 *S3Storage) ReadInBuffer(options internal.ReadInBufferOptions) (int, er return 0, nil } - err := s3.Storage.ReadInBuffer(options.Handle.Path, options.Offset, dataLen, options.Data) + err := s3.Storage.ReadInBuffer( + s3.ctx, + options.Handle.Path, + options.Offset, + dataLen, + options.Data, + ) + s3.updateConnectionState(err) if err != nil { log.Err( "S3Storage::ReadInBuffer : Failed to read %s [%s]", @@ -448,20 +550,22 @@ func (s3 *S3Storage) ReadInBuffer(options internal.ReadInBufferOptions) (int, er } func (s3 *S3Storage) WriteFile(options internal.WriteFileOptions) (int, error) { - err := s3.Storage.Write(options) + err := s3.Storage.Write(s3.ctx, options) + s3.updateConnectionState(err) return len(options.Data), err } func (s3 *S3Storage) GetFileBlockOffsets( options internal.GetFileBlockOffsetsOptions, ) (*common.BlockOffsetList, error) { - return s3.Storage.GetFileBlockOffsets(options.Name) + return s3.Storage.GetFileBlockOffsets(s3.ctx, options.Name) } func (s3 *S3Storage) TruncateFile(options internal.TruncateFileOptions) error { log.Trace("S3Storage::TruncateFile : %s to %d bytes", options.Name, options.Size) - err := s3.Storage.TruncateFile(options.Name, options.Size) + err := s3.Storage.TruncateFile(s3.ctx, options.Name, options.Size) + s3.updateConnectionState(err) if err == nil { s3StatsCollector.PushEvents( @@ -476,12 +580,16 @@ func (s3 *S3Storage) TruncateFile(options internal.TruncateFileOptions) error { func (s3 *S3Storage) CopyToFile(options internal.CopyToFileOptions) error { log.Trace("S3Storage::CopyToFile : Read file %s", options.Name) - return s3.Storage.ReadToFile(options.Name, options.Offset, options.Count, options.File) + err := s3.Storage.ReadToFile(s3.ctx, options.Name, options.Offset, options.Count, options.File) + s3.updateConnectionState(err) + return err } func (s3 *S3Storage) CopyFromFile(options internal.CopyFromFileOptions) error { log.Trace("S3Storage::CopyFromFile : Upload file %s", options.Name) - return s3.Storage.WriteFromFile(options.Name, options.Metadata, options.File) + err := s3.Storage.WriteFromFile(s3.ctx, options.Name, options.Metadata, options.File) + s3.updateConnectionState(err) + return err } // Symlink operations @@ -495,8 +603,9 @@ func (s3 *S3Storage) CreateLink(options internal.CreateLinkOptions) error { return syscall.ENOTSUP } log.Trace("S3Storage::CreateLink : Create symlink %s -> %s", options.Name, options.Target) - err := s3.Storage.CreateLink(options.Name, options.Target, true) + err := s3.Storage.CreateLink(s3.ctx, options.Name, options.Target, true) + s3.updateConnectionState(err) if err == nil { s3StatsCollector.PushEvents( createLink, @@ -516,7 +625,8 @@ func (s3 *S3Storage) ReadLink(options internal.ReadLinkOptions) (string, error) } log.Trace("S3Storage::ReadLink : Read symlink %s", options.Name) - data, err := s3.Storage.ReadBuffer(options.Name, 0, 0, true) + data, err := s3.Storage.ReadBuffer(s3.ctx, options.Name, 0, 0, true) + s3.updateConnectionState(err) if err != nil { s3StatsCollector.PushEvents(readLink, options.Name, nil) @@ -529,7 +639,9 @@ func (s3 *S3Storage) ReadLink(options internal.ReadLinkOptions) (string, error) // Attribute operations func (s3 *S3Storage) GetAttr(options internal.GetAttrOptions) (*internal.ObjAttr, error) { //log.Trace("S3Storage::GetAttr : Get attributes of file %s", name) - return s3.Storage.GetAttr(options.Name) + attr, err := s3.Storage.GetAttr(s3.ctx, options.Name) + s3.updateConnectionState(err) + return attr, err } func (s3 *S3Storage) Chmod(options internal.ChmodOptions) error { @@ -557,11 +669,19 @@ func (s3 *S3Storage) Chown(options internal.ChownOptions) error { func (s3 *S3Storage) FlushFile(options internal.FlushFileOptions) error { log.Trace("S3Storage::FlushFile : Flush file %s", options.Handle.Path) - return s3.Storage.StageAndCommit(options.Handle.Path, options.Handle.CacheObj.BlockOffsetList) + err := s3.Storage.StageAndCommit( + s3.ctx, + options.Handle.Path, + options.Handle.CacheObj.BlockOffsetList, + ) + s3.updateConnectionState(err) + return err } func (s3 *S3Storage) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error) { - return s3.Storage.GetCommittedBlockList(name) + cbl, err := s3.Storage.GetCommittedBlockList(s3.ctx, name) + s3.updateConnectionState(err) + return cbl, err } func (s3 *S3Storage) StageData(opt internal.StageDataOptions) error { @@ -569,7 +689,9 @@ func (s3 *S3Storage) StageData(opt internal.StageDataOptions) error { } func (s3 *S3Storage) CommitData(opt internal.CommitDataOptions) error { - return s3.Storage.CommitBlocks(opt.Name, opt.List) + err := s3.Storage.CommitBlocks(s3.ctx, opt.Name, opt.List) + s3.updateConnectionState(err) + return err } const blockSize = 4096 @@ -584,7 +706,8 @@ func (s3 *S3Storage) StatFs() (*common.Statfs_t, bool, error) { // cache_size - used = f_frsize * f_bavail/1024 // cache_size - used = vfs.f_bfree * vfs.f_frsize / 1024 // if cache size is set to 0 then we have the root mount usage - sizeUsed, err := s3.Storage.GetUsedSize() + sizeUsed, err := s3.Storage.GetUsedSize(s3.ctx) + s3.updateConnectionState(err) if err != nil { // TODO: will returning EIO break any applications that depend on StatFs? return nil, true, err diff --git a/component/s3storage/s3storage_test.go b/component/s3storage/s3storage_test.go index fe492d797..c1798b216 100644 --- a/component/s3storage/s3storage_test.go +++ b/component/s3storage/s3storage_test.go @@ -448,6 +448,37 @@ func (s *s3StorageTestSuite) TestListBuckets() { s.assert.Contains(buckets, storageTestConfigurationParameters.BucketName) } +func (s *s3StorageTestSuite) TestCloudConnected() { + defer s.cleanupTest() + s.assert.True(s.s3Storage.CloudConnected()) +} + +func (s *s3StorageTestSuite) TestUpdateConnectionState() { + defer s.cleanupTest() + connected := s.s3Storage.updateConnectionState(&common.CloudUnreachableError{}) + s.assert.False(connected) + s.assert.False(s.s3Storage.CloudConnected()) + connected = s.s3Storage.updateConnectionState(nil) + s.assert.True(connected) + s.assert.True(s.s3Storage.CloudConnected()) +} + +func (s *s3StorageTestSuite) TestCloudOfflineCached() { + defer s.cleanupTest() + s.s3Storage.updateConnectionState(&common.CloudUnreachableError{}) + s.assert.False(s.s3Storage.CloudConnected()) + s.s3Storage.updateConnectionState(nil) +} + +func (s *s3StorageTestSuite) TestCloudOfflineContext() { + defer s.cleanupTest() + s.s3Storage.updateConnectionState(&common.CloudUnreachableError{}) + h, err := s.s3Storage.CreateFile(internal.CreateFileOptions{Name: "file" + randomString(8)}) + s.assert.Nil(h) + s.assert.ErrorIs(err, &common.CloudUnreachableError{}) + s.s3Storage.updateConnectionState(nil) +} + func (s *s3StorageTestSuite) TestCreateDir() { defer s.cleanupTest() // Testing dir and dir/ @@ -3041,6 +3072,7 @@ func (s *s3StorageTestSuite) TestFlushFileUpdateChunkedFile() { rand.Read(updatedBlock) h.CacheObj.BlockOffsetList.BlockList[1].Data = make([]byte, blockSizeBytes) s.s3Storage.Storage.ReadInBuffer( + context.Background(), name, int64(blockSizeBytes), int64(blockSizeBytes), @@ -3097,6 +3129,7 @@ func (s *s3StorageTestSuite) TestFlushFileTruncateUpdateChunkedFile() { h.CacheObj.BlockOffsetList.BlockList[1].Data = make([]byte, blockSizeBytes/2) h.CacheObj.BlockOffsetList.BlockList[1].EndIndex = int64(blockSizeBytes + blockSizeBytes/2) s.s3Storage.Storage.ReadInBuffer( + context.Background(), name, int64(blockSizeBytes), int64(blockSizeBytes)/2, diff --git a/component/s3storage/s3wrappers.go b/component/s3storage/s3wrappers.go index e0a465db6..a035f9cc1 100644 --- a/component/s3storage/s3wrappers.go +++ b/component/s3storage/s3wrappers.go @@ -82,9 +82,19 @@ type renameObjectOptions struct { const symlinkStr = ".rclonelink" const maxResultsPerListCall = 1000 +// check the connection to the S3 service by calling HeadBucket. +func (cl *Client) ConnectionOkay(ctx context.Context) error { + log.Trace("Client::ConnectionOkay : checking connection to S3 service") + _, err := cl.AwsS3Client.HeadBucket( + ctx, + &s3.HeadBucketInput{Bucket: aws.String(cl.Config.AuthConfig.BucketName)}, + ) + return parseS3Err(err, "HeadBucket "+cl.Config.AuthConfig.BucketName) +} + // getObjectMultipartDownload downloads an object to a file using multipart download // which can be much faster for large objects. -func (cl *Client) getObjectMultipartDownload(name string, fi *os.File) error { +func (cl *Client) getObjectMultipartDownload(ctx context.Context, name string, fi *os.File) error { key := cl.getKey(name, false, false) log.Trace("Client::getObjectMultipartDownload : get object %s", key) @@ -97,7 +107,7 @@ func (cl *Client) getObjectMultipartDownload(name string, fi *os.File) error { getObjectInput.ChecksumMode = types.ChecksumModeEnabled } - _, err := cl.downloader.Download(context.Background(), fi, getObjectInput) + _, err := cl.downloader.Download(ctx, fi, getObjectInput) // check for errors if err != nil { attemptedAction := fmt.Sprintf("GetObject(%s)", key) @@ -109,7 +119,7 @@ func (cl *Client) getObjectMultipartDownload(name string, fi *os.File) error { // Wrapper for awsS3Client.GetObject. // Set count = 0 to read to the end of the object. // name is the path to the file. -func (cl *Client) getObject(options getObjectOptions) (io.ReadCloser, error) { +func (cl *Client) getObject(ctx context.Context, options getObjectOptions) (io.ReadCloser, error) { key := cl.getKey(options.name, options.isSymLink, options.isDir) log.Trace("Client::getObject : get object %s (%d+%d)", key, options.offset, options.count) @@ -145,7 +155,7 @@ func (cl *Client) getObject(options getObjectOptions) (io.ReadCloser, error) { getObjectInput.ChecksumMode = types.ChecksumModeEnabled } - result, err := cl.AwsS3Client.GetObject(context.Background(), getObjectInput) + result, err := cl.AwsS3Client.GetObject(ctx, getObjectInput) // check for errors if err != nil { @@ -160,10 +170,9 @@ func (cl *Client) getObject(options getObjectOptions) (io.ReadCloser, error) { // Wrapper for awsS3Client.PutObject. // Pass in the name of the file, an io.Reader with the object data, the size of the upload, // and whether the object is a symbolic link or not. -func (cl *Client) putObject(options putObjectOptions) error { +func (cl *Client) putObject(ctx context.Context, options putObjectOptions) error { key := cl.getKey(options.name, options.isSymLink, options.isDir) log.Trace("Client::putObject : putting object %s", key) - ctx := context.Background() var err error putObjectInput := &s3.PutObjectInput{ @@ -191,11 +200,11 @@ func (cl *Client) putObject(options putObjectOptions) error { // Wrapper for awsS3Client.DeleteObject. // name is the path to the file. -func (cl *Client) deleteObject(name string, isSymLink bool, isDir bool) error { +func (cl *Client) deleteObject(ctx context.Context, name string, isSymLink bool, isDir bool) error { key := cl.getKey(name, isSymLink, isDir) log.Trace("Client::deleteObject : deleting object %s", key) - _, err := cl.AwsS3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ + _, err := cl.AwsS3Client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String(cl.Config.AuthConfig.BucketName), Key: aws.String(key), }) @@ -206,7 +215,7 @@ func (cl *Client) deleteObject(name string, isSymLink bool, isDir bool) error { // Wrapper for awsS3Client.DeleteObjects. // names is a list of paths to the objects. -func (cl *Client) deleteObjects(objects []*internal.ObjAttr) error { +func (cl *Client) deleteObjects(ctx context.Context, objects []*internal.ObjAttr) error { if objects == nil { return nil } @@ -220,7 +229,7 @@ func (cl *Client) deleteObjects(objects []*internal.ObjAttr) error { } } // send keyList for deletion - result, err := cl.AwsS3Client.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{ + result, err := cl.AwsS3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ Bucket: &cl.Config.AuthConfig.BucketName, Delete: &types.Delete{ Objects: keyList, @@ -249,11 +258,16 @@ func (cl *Client) deleteObjects(objects []*internal.ObjAttr) error { // HeadObject() acts just like GetObject, except no contents are returned. // So this is used to get metadata / attributes for an object. // name is the path to the file. -func (cl *Client) headObject(name string, isSymlink bool, isDir bool) (*internal.ObjAttr, error) { +func (cl *Client) headObject( + ctx context.Context, + name string, + isSymlink bool, + isDir bool, +) (*internal.ObjAttr, error) { key := cl.getKey(name, isSymlink, isDir) log.Trace("Client::headObject : object %s", key) - result, err := cl.AwsS3Client.HeadObject(context.Background(), &s3.HeadObjectInput{ + result, err := cl.AwsS3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(cl.Config.AuthConfig.BucketName), Key: aws.String(key), }) @@ -275,15 +289,15 @@ func (cl *Client) headObject(name string, isSymlink bool, isDir bool) (*internal } // Wrapper for awsS3Client.HeadBucket -func (cl *Client) headBucket(bucketName string) (*s3.HeadBucketOutput, error) { - headBucketOutput, err := cl.AwsS3Client.HeadBucket(context.Background(), &s3.HeadBucketInput{ +func (cl *Client) headBucket(ctx context.Context, bucketName string) (*s3.HeadBucketOutput, error) { + headBucketOutput, err := cl.AwsS3Client.HeadBucket(ctx, &s3.HeadBucketInput{ Bucket: aws.String(bucketName), }) return headBucketOutput, parseS3Err(err, "HeadBucket "+bucketName) } // Wrapper for awsS3Client.CopyObject -func (cl *Client) copyObject(options copyObjectOptions) error { +func (cl *Client) copyObject(ctx context.Context, options copyObjectOptions) error { // copy the object to its new key sourceKey := cl.getKey(options.source, options.isSymLink, options.isDir) targetKey := cl.getKey(options.target, options.isSymLink, options.isDir) @@ -300,7 +314,7 @@ func (cl *Client) copyObject(options copyObjectOptions) error { copyObjectInput.ChecksumAlgorithm = cl.Config.checksumAlgorithm } - _, err := cl.AwsS3Client.CopyObject(context.Background(), copyObjectInput) + _, err := cl.AwsS3Client.CopyObject(ctx, copyObjectInput) // check for errors on copy if err != nil { attemptedAction := fmt.Sprintf("copy %s to %s", sourceKey, targetKey) @@ -310,10 +324,8 @@ func (cl *Client) copyObject(options copyObjectOptions) error { return err } -func (cl *Client) renameObject(options renameObjectOptions) error { - err := cl.copyObject( - copyObjectOptions(options), - ) //nolint +func (cl *Client) renameObject(ctx context.Context, options renameObjectOptions) error { + err := cl.copyObject(ctx, copyObjectOptions(options)) if err != nil { log.Err( "Client::renameObject : copyObject(%s->%s) failed. Here's why: %v", @@ -326,7 +338,7 @@ func (cl *Client) renameObject(options renameObjectOptions) error { // Copy of the file is done so now delete the older file // in this case we don't need to check if the file exists, so we use deleteObject, not DeleteFile // this is what S3's DeleteObject spec is meant for: to make sure the object doesn't exist anymore - err = cl.deleteObject(options.source, options.isSymLink, options.isDir) + err = cl.deleteObject(ctx, options.source, options.isSymLink, options.isDir) if err != nil { log.Err( "Client::renameObject : deleteObject(%s) failed. Here's why: %v", @@ -339,9 +351,9 @@ func (cl *Client) renameObject(options renameObjectOptions) error { } // abortMultipartUpload stops a multipart upload and verifys that the parts are deleted. -func (cl *Client) abortMultipartUpload(key string, uploadID string) error { +func (cl *Client) abortMultipartUpload(ctx context.Context, key string, uploadID string) error { _, abortErr := cl.AwsS3Client.AbortMultipartUpload( - context.Background(), + ctx, &s3.AbortMultipartUploadInput{ Bucket: aws.String(cl.Config.AuthConfig.BucketName), Key: aws.String(key), @@ -353,7 +365,7 @@ func (cl *Client) abortMultipartUpload(key string, uploadID string) error { } // AWS states you need to call listparts to verify that multipart upload was properly aborted - resp, listErr := cl.AwsS3Client.ListParts(context.Background(), &s3.ListPartsInput{ + resp, listErr := cl.AwsS3Client.ListParts(ctx, &s3.ListPartsInput{ Bucket: aws.String(cl.Config.AuthConfig.BucketName), Key: aws.String(key), UploadId: &uploadID, @@ -377,12 +389,12 @@ func (cl *Client) abortMultipartUpload(key string, uploadID string) error { } // Wrapper for awsS3Client.ListBuckets -func (cl *Client) ListBuckets() ([]string, error) { +func (cl *Client) ListBuckets(ctx context.Context) ([]string, error) { log.Trace("Client::ListBuckets : Listing buckets") cntList := make([]string, 0) - result, err := cl.AwsS3Client.ListBuckets(context.Background(), &s3.ListBucketsInput{}) + result, err := cl.AwsS3Client.ListBuckets(ctx, &s3.ListBucketsInput{}) if err != nil { log.Err("Client::ListBuckets : Failed to list buckets. Here's why: %v", err) @@ -404,6 +416,7 @@ func (cl *Client) ListBuckets() ([]string, error) { // If count=0 - fetch max entries. // the *string being returned is the token / marker and will be nil when the listing is complete. func (cl *Client) List( + ctx context.Context, prefix string, marker *string, count int32, @@ -462,15 +475,14 @@ func (cl *Client) List( // initialize list to be returned objectAttrList := make([]*internal.ObjAttr, 0) // fetch and process a single result page - output, err := paginator.NextPage(context.Background()) + output, err := paginator.NextPage(ctx) if err != nil { - log.Err( - "Client::List : Failed to list objects in bucket %v with prefix %v. Here's why: %v", - prefix, + attemptedAction := fmt.Sprintf( + "list objects in bucket %v with prefix %v", bucketName, - err, + prefix, ) - return objectAttrList, nil, err + return objectAttrList, nil, parseS3Err(err, attemptedAction) } if output.IsTruncated != nil && *output.IsTruncated { diff --git a/component/s3storage/utils.go b/component/s3storage/utils.go index 6e2f89557..3587eae05 100644 --- a/component/s3storage/utils.go +++ b/component/s3storage/utils.go @@ -26,6 +26,7 @@ package s3storage import ( + "context" "encoding/json" "errors" "fmt" @@ -38,6 +39,8 @@ import ( "github.com/Seagate/cloudfuse/common/log" "github.com/Seagate/cloudfuse/internal" + "github.com/aws/aws-sdk-go-v2/aws/ratelimit" + "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/smithy-go" ) @@ -142,7 +145,24 @@ func parseS3Err(err error, attemptedAction string) error { } } + var maerr *retry.MaxAttemptsError + qeerr := &ratelimit.QuotaExceededError{} + if errors.As(err, &maerr) || errors.As(err, qeerr) || errors.Is(err, context.Canceled) { + log.Err( + "%s : Failed to %s because cloud storage is unreachable", + functionName, + attemptedAction, + ) + return common.NewCloudUnreachableError(err) + } + // unrecognized error - parsing failed + // log error information to debug log + unwrappedErr := err + for unwrappedErr != nil { + log.Debug("Uncaught S3 error is of type \"%T\" and value %v.", unwrappedErr, unwrappedErr) + unwrappedErr = errors.Unwrap(unwrappedErr) + } // print and return the original error log.Err("%s : Failed to %s. Here's why: %v", functionName, attemptedAction, err) return err diff --git a/internal/base_component.go b/internal/base_component.go index 5cca3585f..63943bcb0 100644 --- a/internal/base_component.go +++ b/internal/base_component.go @@ -84,6 +84,13 @@ func (base *BaseComponent) Stop() error { return nil } +func (base *BaseComponent) CloudConnected() bool { + if base.next != nil { + return base.next.CloudConnected() + } + return false +} + // Directory operations func (base *BaseComponent) CreateDir(options CreateDirOptions) error { if base.next != nil { diff --git a/internal/component.go b/internal/component.go index a81a2619c..9fce41956 100644 --- a/internal/component.go +++ b/internal/component.go @@ -71,6 +71,8 @@ type Component interface { Start(context.Context) error Stop() error + CloudConnected() bool + // Directory operations CreateDir(CreateDirOptions) error DeleteDir(DeleteDirOptions) error diff --git a/internal/mock_component.go b/internal/mock_component.go index 6b21c9c54..dca7c402d 100644 --- a/internal/mock_component.go +++ b/internal/mock_component.go @@ -121,6 +121,20 @@ func (mr *MockComponentMockRecorder) CloseFile(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseFile", reflect.TypeOf((*MockComponent)(nil).CloseFile), arg0) } +// CloudConnected mocks base method. +func (m *MockComponent) CloudConnected() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloudConnected") + ret0, _ := ret[0].(bool) + return ret0 +} + +// CloudConnected indicates an expected call of CloudConnected. +func (mr *MockComponentMockRecorder) CloudConnected() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloudConnected", reflect.TypeOf((*MockComponent)(nil).CloudConnected)) +} + // Configure mocks base method. func (m *MockComponent) Configure(arg0 bool) error { m.ctrl.T.Helper() diff --git a/setup/advancedConfig.yaml b/setup/advancedConfig.yaml index 87a8fa97c..87064895e 100644 --- a/setup/advancedConfig.yaml +++ b/setup/advancedConfig.yaml @@ -220,6 +220,7 @@ s3storage: checksum-algorithm: CRC32|CRC32C|SHA1|SHA256 use-path-style: true|false enable-dir-marker: true|false + health-check-interval-sec: # Mount all configuration mountall: diff --git a/setup/baseConfig.yaml b/setup/baseConfig.yaml index 5aa26884d..2dcad3505 100644 --- a/setup/baseConfig.yaml +++ b/setup/baseConfig.yaml @@ -112,6 +112,7 @@ file_cache: create-empty-file: true|false allow-non-empty-temp: true|false cleanup-on-start: true|false + block-offline-access: true|false policy-trace: true|false offload-io: true|false sync-to-flush: true|false @@ -197,6 +198,7 @@ s3storage: use-path-style: true|false disable-usage: true|false enable-dir-marker: true|false + health-check-interval-sec: # Mount all configuration mountall: diff --git a/test-scripts/block.sh b/test-scripts/block.sh new file mode 100755 index 000000000..181d8cf96 --- /dev/null +++ b/test-scripts/block.sh @@ -0,0 +1,4 @@ +#!/bin/bash +#command to block outgoing calls to Lyve Cloud and Azure Blob Storage +sudo iptables -I OUTPUT 1 -d 192.55.0.0/16 -j REJECT +sudo iptables -I OUTPUT 1 -d 20.60.0.0/16 -j REJECT diff --git a/test-scripts/connect.sh b/test-scripts/connect.sh new file mode 100755 index 000000000..c2345be9b --- /dev/null +++ b/test-scripts/connect.sh @@ -0,0 +1,4 @@ +#!/bin/bash +#command to accept outgoing calls to Lyve Cloud and Azure Blob Storage +sudo iptables -D OUTPUT -d 192.55.0.0/16 -j REJECT +sudo iptables -D OUTPUT -d 20.60.0.0/16 -j REJECT