Documentation
¶
Overview ¶
Copyright © 2025 Microsoft <[email protected]>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Copyright © 2025 Microsoft <[email protected]>
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Index ¶
- Constants
- Variables
- func BlobTierAllowed(destTier *blob.AccessTier) bool
- func BlockIDIntToBase64(blockID int) string
- func CreateNewContainer(t *testing.T, a *assert.Assertions, bsc *blobservice.Client) (cc *container.Client, name string)
- func DeleteBlob(jptm IJobPartTransferMgr, pacer pacer)
- func DeleteContainer(a *assert.Assertions, cc *container.Client)
- func DeleteFile(jptm IJobPartTransferMgr, _ pacer)
- func DeleteHNSResource(jptm IJobPartTransferMgr, pacer pacer)
- func DoWithCreateSymlinkOnAzureFilesNFS(ctx context.Context, action func() error, client *file.Client, ...) error
- func FileAttributesFromUint32(attributes uint32) (*file.NTFSFileAttributes, error)
- func FileAttributesToUint32(attributes file.NTFSFileAttributes) uint32
- func GenerateBlockIDsList(count int) []string
- func GetAccountAndKey() (string, string)
- func GetBlobServiceClient() *blobservice.Client
- func GetContainerClient(t *testing.T, bsc *blobservice.Client) (container *container.Client, name string)
- func IsOSErrors(err error, errnos ...syscall.Errno) bool
- func NewAzcopyHTTPClient(maxIdleConns int) *http.Client
- func NewClientOptions(retry policy.RetryOptions, telemetry policy.TelemetryOptions, ...) azcore.ClientOptions
- func NewDestReauthPolicy(cred *common.ScopedAuthenticator) policy.Policy
- func NewNullAutoPacer() *nullAutoPacer
- func NewRequestPriorityPolicy() policy.Policy
- func NewSourceAuthPolicy(cred azcore.TokenCredential) policy.Policy
- func NewTokenBucketPacer(bytesPerSecond int64, expectedBytesPerCoarseRequest int64) *tokenBucketPacer
- func NewVersionPolicy() policy.Policy
- func SetProperties(jptm IJobPartTransferMgr, _ pacer)
- func ValidateTier(jptm IJobPartTransferMgr, blobTier *blob.AccessTier, client IBlobClient, ...) (isValid bool)
- type AddJobPartArgs
- type AdviceType
- func (AdviceType) AccountIOPS() AdviceType
- func (AdviceType) AccountThroughput() AdviceType
- func (AdviceType) ConcurrencyHighCpu() AdviceType
- func (AdviceType) ConcurrencyHitUpperLimit() AdviceType
- func (AdviceType) ConcurrencyNotEnoughTime() AdviceType
- func (AdviceType) ConcurrencyNotTuned() AdviceType
- func (AdviceType) FileShareOrNetwork() AdviceType
- func (AdviceType) MbpsCapped() AdviceType
- func (AdviceType) NetworkErrors() AdviceType
- func (AdviceType) NetworkIsBottleneck() AdviceType
- func (AdviceType) NetworkNotBottleneck() AdviceType
- func (AdviceType) ServerBusy() AdviceType
- func (AdviceType) SmallFilesOrNetwork() AdviceType
- func (AdviceType) VMSize() AdviceType
- type AzureFileParentDirCreator
- type ConcurrencySettings
- type ConcurrencyTuner
- type ConfiguredBool
- type ConfiguredInt
- type CoordinatorChannels
- type DatalakeClientStub
- type ErrorEx
- type FileClientStub
- type FolderCreationTracker
- type HandleInfo
- type HandleNFSPermissions
- type IBlobClient
- type IBlobSourceInfoProvider
- type ICustomLocalOpener
- type IJobMgr
- type IJobPartMgr
- type IJobPartPlanHeader
- type IJobPartTransferMgr
- type ILocalSourceInfoProvider
- type INFSPropertyBearingSourceInfoProvider
- type IRemoteSourceInfoProvider
- type ISMBPropertyBearingSourceInfoProvider
- type ISourceInfoProvider
- type ISymlinkBearingSourceInfoProvider
- type IUNIXPropertyBearingSourceInfoProvider
- type JPPTCompatibleFolderCreationTracker
- type JobPartCreatedMsg
- type JobPartPlanDstBlob
- type JobPartPlanDstFile
- type JobPartPlanDstLocal
- type JobPartPlanFileName
- func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest)
- func (jpfn JobPartPlanFileName) Delete() error
- func (jppfn *JobPartPlanFileName) Exists() bool
- func (jppfn *JobPartPlanFileName) GetJobPartPlanPath() string
- func (jpfn JobPartPlanFileName) Map() *JobPartPlanMMF
- func (jpfn JobPartPlanFileName) Parse() (jobID common.JobID, partNumber common.PartNumber, err error)
- type JobPartPlanHeader
- func (jpph *JobPartPlanHeader) CommandString() string
- func (jpph *JobPartPlanHeader) JobPartStatus() common.JobStatus
- func (jpph *JobPartPlanHeader) JobStatus() common.JobStatus
- func (jpph *JobPartPlanHeader) SetJobPartStatus(newJobStatus common.JobStatus)
- func (jpph *JobPartPlanHeader) SetJobStatus(newJobStatus common.JobStatus)
- func (jpph *JobPartPlanHeader) Transfer(transferIndex uint32) *JobPartPlanTransfer
- func (jpph *JobPartPlanHeader) TransferSrcDstRelatives(transferIndex uint32) (relSource, relDest string)
- func (jpph *JobPartPlanHeader) TransferSrcDstStrings(transferIndex uint32) (source, destination string, isFolder bool)
- func (jpph *JobPartPlanHeader) TransferSrcPropertiesAndMetadata(transferIndex uint32) (h common.ResourceHTTPHeaders, metadata common.Metadata, blobType blob.BlobType, ...)
- type JobPartPlanMMF
- type JobPartPlanTransfer
- type JpptFolderIndex
- type LogOptions
- type NFSProperties
- type NullConcurrencyTuner
- func (n *NullConcurrencyTuner) GetFinalState() (finalReason string, finalRecommendedConcurrency int)
- func (n *NullConcurrencyTuner) GetRecommendedConcurrency(currentMbps int, highCpuUsage bool) (newConcurrency int, reason string)
- func (n *NullConcurrencyTuner) RequestCallbackWhenStable(callback func()) (callbackAccepted bool)
- type PacerAdmin
- type PartNumber
- type PerformanceAdvisor
- type PipelineNetworkStats
- func (s *PipelineNetworkStats) AverageE2EMilliseconds() int
- func (s *PipelineNetworkStats) GetTotalRetries() int64
- func (s *PipelineNetworkStats) IOPSServerBusyPercentage() float32
- func (s *PipelineNetworkStats) NetworkErrorPercentage() float32
- func (s *PipelineNetworkStats) OperationsPerSecond() int
- func (s *PipelineNetworkStats) OtherServerBusyPercentage() float32
- func (s *PipelineNetworkStats) ThroughputServerBusyPercentage() float32
- func (s *PipelineNetworkStats) TotalServerBusyPercentage() float32
- type RequestLogOptions
- type RetryCodes
- type RetryFunc
- type SrcProperties
- type StatTAdapter
- func (s StatTAdapter) ATime() time.Time
- func (s StatTAdapter) Attribute() uint64
- func (s StatTAdapter) AttributeMask() uint64
- func (s StatTAdapter) BTime() time.Time
- func (s StatTAdapter) CTime() time.Time
- func (s StatTAdapter) Device() uint64
- func (s StatTAdapter) Extended() bool
- func (s StatTAdapter) FileMode() uint32
- func (s StatTAdapter) Group() uint32
- func (s StatTAdapter) INode() uint64
- func (s StatTAdapter) MTime() time.Time
- func (s StatTAdapter) NLink() uint64
- func (s StatTAdapter) Owner() uint32
- func (s StatTAdapter) RDevice() uint64
- func (s StatTAdapter) StatxMask() uint32
- type StatxTAdapter
- func (s StatxTAdapter) ATime() time.Time
- func (s StatxTAdapter) Attribute() uint64
- func (s StatxTAdapter) AttributeMask() uint64
- func (s StatxTAdapter) BTime() time.Time
- func (s StatxTAdapter) CTime() time.Time
- func (s StatxTAdapter) Device() uint64
- func (s StatxTAdapter) Extended() bool
- func (s StatxTAdapter) FileMode() uint32
- func (s StatxTAdapter) Group() uint32
- func (s StatxTAdapter) INode() uint64
- func (s StatxTAdapter) MTime() time.Time
- func (s StatxTAdapter) NLink() uint64
- func (s StatxTAdapter) Owner() uint32
- func (s StatxTAdapter) RDevice() uint64
- func (s StatxTAdapter) StatxMask() uint32
- type StorageErrorCodes
- type TransferFetcher
- type TransferInfo
- type TypedNFSPermissionsHolder
- type TypedNFSPropertyHolder
- type TypedSMBPropertyHolder
- type XferChannels
Constants ¶
const ( CustomHeaderMaxBytes = 256 MetadataMaxBytes = 1000 // If > 65536, then jobPartPlanBlobData's MetadataLength field's type must change BlobTagsMaxByte = 4000 )
const ( ConcurrencyReasonNone = "" ConcurrencyReasonTunerDisabled = "tuner disabled" // used as the final (non-finished) state for null tuner )
const ( FileAttributeNone uint32 = 0 FileAttributeReadonly uint32 = 1 FileAttributeHidden uint32 = 2 FileAttributeSystem uint32 = 4 FileAttributeArchive uint32 = 32 FileAttributeTemporary uint32 = 256 FileAttributeOffline uint32 = 4096 FileAttributeNotContentIndexed uint32 = 8192 FileAttributeNoScrubData uint32 = 131072 )
This is intended for easy conversion to/from local file attributes
const (
BlockBlobDefaultData = "AzCopy Random Test Data"
)
const DataSchemaVersion common.Version = 19
dataSchemaVersion defines the data schema version of JobPart order files supported by current version of azcopy To be Incremented every time when we release azcopy with changed dataSchema
const DownloadMaxRetryDelay = time.Second * 60
const DownloadRetryDelay = time.Second * 1
const DownloadTryTimeout = time.Minute * 15
TODO: consider to unify the retry options.
const JobPartPlanFileNameFormat = "%v--%05d.steV%d"
const MaxRetryPerDownloadBody = 5
download related
const PacerTimeToWaitInMs = 50
pacer related
const StorageErrorCodesWildcard = "*"
const TagsHeaderMaxLength = 2000
const UploadMaxRetryDelay = time.Second * 60
const UploadMaxTries = 20
upload related
const UploadRetryDelay = time.Second * 1
const (
XMsRequestPriority = "x-ms-request-priority"
)
const (
XMsVersion = "x-ms-version"
)
Variables ¶
var ( // ServiceAPIVersionOverride is a global variable in package ste which is a key to Service Api Version Value set in the every Job's context. ServiceAPIVersionOverride = serviceAPIVersionOverride{} // DefaultServiceApiVersion is the default value of service api version that is set as value to the ServiceAPIVersionOverride in every Job's context. DefaultServiceApiVersion = common.GetEnvironmentVariable(common.EEnvironmentVariable.DefaultServiceApiVersion()) )
var ADLSFlushThreshold uint32 = 7500 // The # of blocks to flush at a time-- Implemented only for CI.
var DebugSkipFiles = make(map[string]bool)
debug knob
var EAdviceType = AdviceType{"", ""}
var EnvironmentMimeMap map[string]string
var FilesServiceMaxSDDLSize = 8000
Files supports SDDLs up to and equal to 8kb. Because this isn't KiB, We're going to infer that it's 8x1000, not 8x1024.
var ( // GlobalRequestPriority at risk of becoming yet another angle in which temporary solutions become very permanent // attempts to avoid the current issue of context usage. Context usage will get fixed up in a follow-up PR (PBI # 34685770), but // to save me (Adele) from going bonkers, I'm going to push this out with a jank solution to begin. GlobalRequestPriority = -1 )
var LogBlobConversionOnce = &sync.Once{}
var UploadTryTimeout = time.Minute * 15
Functions ¶
func BlobTierAllowed ¶
func BlobTierAllowed(destTier *blob.AccessTier) bool
// TODO: Infer availability based upon blob size as well, for premium page blobs.
func BlockIDIntToBase64 ¶ added in v10.22.0
BlockIDIntToBase64 functions convert an int block ID to a base-64 string and vice versa
func CreateNewContainer ¶ added in v10.22.0
func CreateNewContainer(t *testing.T, a *assert.Assertions, bsc *blobservice.Client) (cc *container.Client, name string)
func DeleteBlob ¶
func DeleteBlob(jptm IJobPartTransferMgr, pacer pacer)
func DeleteContainer ¶ added in v10.22.0
func DeleteContainer(a *assert.Assertions, cc *container.Client)
func DeleteFile ¶
func DeleteFile(jptm IJobPartTransferMgr, _ pacer)
func DeleteHNSResource ¶ added in v10.19.0
func DeleteHNSResource(jptm IJobPartTransferMgr, pacer pacer)
func DoWithCreateSymlinkOnAzureFilesNFS ¶ added in v10.31.0
func DoWithCreateSymlinkOnAzureFilesNFS( ctx context.Context, action func() error, client *file.Client, shareClient *share.Client, pacer pacer, jptm IJobPartTransferMgr, ) error
DoWithCreateSymlinkOnAzureFilesNFS tries to create a symlink, with retry logic for parent not found and resource already exists.
func FileAttributesFromUint32 ¶ added in v10.21.0
func FileAttributesFromUint32(attributes uint32) (*file.NTFSFileAttributes, error)
func FileAttributesToUint32 ¶ added in v10.21.0
func FileAttributesToUint32(attributes file.NTFSFileAttributes) uint32
func GenerateBlockIDsList ¶ added in v10.22.0
func GetAccountAndKey ¶ added in v10.22.0
func GetBlobServiceClient ¶ added in v10.22.0
func GetBlobServiceClient() *blobservice.Client
get blob account service client
func GetContainerClient ¶ added in v10.22.0
func NewAzcopyHTTPClient ¶
NewAzcopyHTTPClient creates a new HTTP client. We must minimize use of this, and instead maximize reuse of the returned client object. Why? Because that makes our connection pooling more efficient, and prevents us exhausting the number of available network sockets on resource-constrained Linux systems. (E.g. when 'ulimit -Hn' is low).
func NewClientOptions ¶ added in v10.21.0
func NewClientOptions(retry policy.RetryOptions, telemetry policy.TelemetryOptions, transport policy.Transporter, log LogOptions, srcCred *common.ScopedToken, dstCred *common.ScopedAuthenticator) azcore.ClientOptions
func NewDestReauthPolicy ¶ added in v10.28.0
func NewDestReauthPolicy(cred *common.ScopedAuthenticator) policy.Policy
func NewNullAutoPacer ¶ added in v10.15.0
func NewNullAutoPacer() *nullAutoPacer
func NewRequestPriorityPolicy ¶ added in v10.31.0
NewRequestPriorityPolicy creates a new requestPriorityPolicy, which should be placed before a NewVersionPolicy.
func NewSourceAuthPolicy ¶ added in v10.23.0
func NewSourceAuthPolicy(cred azcore.TokenCredential) policy.Policy
func NewTokenBucketPacer ¶ added in v10.15.0
func NewVersionPolicy ¶ added in v10.22.0
func SetProperties ¶ added in v10.16.0
func SetProperties(jptm IJobPartTransferMgr, _ pacer)
func ValidateTier ¶
func ValidateTier(jptm IJobPartTransferMgr, blobTier *blob.AccessTier, client IBlobClient, ctx context.Context, performQuietly bool) (isValid bool)
Types ¶
type AddJobPartArgs ¶ added in v10.22.0
type AddJobPartArgs struct {
PartNum PartNumber
PlanFile JobPartPlanFileName
ExistingPlanMMF *JobPartPlanMMF
// this is required in S2S transfers authenticating to src
// via oAuth
SourceTokenCred *string
// These clients are valid if this fits the FromTo. i.e if
// we're uploading
SrcClient *common.ServiceClient
DstClient *common.ServiceClient
SrcIsOAuth bool // true if source is authenticated via token
ScheduleTransfers bool
// This channel will be closed once all transfers in this part are done
CompletionChan chan struct{}
}
type AdviceType ¶
type AdviceType struct {
// contains filtered or unexported fields
}
func (AdviceType) AccountIOPS ¶
func (AdviceType) AccountIOPS() AdviceType
func (AdviceType) AccountThroughput ¶
func (AdviceType) AccountThroughput() AdviceType
func (AdviceType) ConcurrencyHighCpu ¶
func (AdviceType) ConcurrencyHighCpu() AdviceType
func (AdviceType) ConcurrencyHitUpperLimit ¶
func (AdviceType) ConcurrencyHitUpperLimit() AdviceType
func (AdviceType) ConcurrencyNotEnoughTime ¶
func (AdviceType) ConcurrencyNotEnoughTime() AdviceType
func (AdviceType) ConcurrencyNotTuned ¶
func (AdviceType) ConcurrencyNotTuned() AdviceType
func (AdviceType) FileShareOrNetwork ¶
func (AdviceType) FileShareOrNetwork() AdviceType
func (AdviceType) MbpsCapped ¶
func (AdviceType) MbpsCapped() AdviceType
func (AdviceType) NetworkErrors ¶
func (AdviceType) NetworkErrors() AdviceType
func (AdviceType) NetworkIsBottleneck ¶
func (AdviceType) NetworkIsBottleneck() AdviceType
func (AdviceType) NetworkNotBottleneck ¶
func (AdviceType) NetworkNotBottleneck() AdviceType
func (AdviceType) ServerBusy ¶
func (AdviceType) ServerBusy() AdviceType
func (AdviceType) SmallFilesOrNetwork ¶
func (AdviceType) SmallFilesOrNetwork() AdviceType
func (AdviceType) VMSize ¶
func (AdviceType) VMSize() AdviceType
type AzureFileParentDirCreator ¶
type AzureFileParentDirCreator struct{}
namespace for functions related to creating parent directories in Azure File to avoid free floating global funcs
func (AzureFileParentDirCreator) CreateDirToRoot ¶
func (d AzureFileParentDirCreator) CreateDirToRoot(ctx context.Context, shareClient *share.Client, directoryClient *directory.Client, t FolderCreationTracker) error
func (AzureFileParentDirCreator) CreateParentDirToRoot ¶
func (d AzureFileParentDirCreator) CreateParentDirToRoot(ctx context.Context, fileClient *file.Client, shareClient *share.Client, t FolderCreationTracker) error
CreateParentDirToRoot creates parent directories of the Azure file if file's parent directory doesn't exist.
type ConcurrencySettings ¶
type ConcurrencySettings struct {
// InitialMainPoolSize is the initial size of the main goroutine pool that transfers the data
// (i.e. executes chunkfuncs)
InitialMainPoolSize int
// MaxMainPoolSize is a number >= InitialMainPoolSize, representing max size we will grow the main pool to
MaxMainPoolSize *ConfiguredInt
// TransferInitiationPoolSize is the size of the auxiliary goroutine pool that initiates transfers
// (i.e. creates chunkfuncs)
TransferInitiationPoolSize *ConfiguredInt
// EnumerationPoolSize is size of auxiliary goroutine pool used in enumerators (only some of which are in fact parallelized)
EnumerationPoolSize *ConfiguredInt
// ParallelStatFiles says whether file.Stat calls should be parallelized during enumeration. May help enumeration performance
// on Linux, but is not necessary and should not be activate on Windows.
ParallelStatFiles *ConfiguredBool
// MaxIdleConnections is the max number of idle TCP connections to keep open
MaxIdleConnections int
// MaxOpenFiles is the max number of file handles that we should have open at any time
// Currently (July 2019) this is only used for downloads, which is where we wouldn't
// otherwise have strict control of the number of open files.
// For uploads, the number of open files is effectively controlled by
// TransferInitiationPoolSize, since all the file IO (except retries) happens in
// transfer initiation.
MaxOpenDownloadFiles int
// CheckCpuWhenTuning determines whether CPU usage should be taken into account when auto-tuning
CheckCpuWhenTuning *ConfiguredBool
}
ConcurrencySettings stores the set of related numbers that govern concurrency levels in the STE
func NewConcurrencySettings ¶
func NewConcurrencySettings(maxFileAndSocketHandles int) ConcurrencySettings
NewConcurrencySettings gets concurrency settings by referring to the environment variable AZCOPY_CONCURRENCY_VALUE (if set) and to properties of the machine where we are running
func (ConcurrencySettings) AutoTuneMainPool ¶
func (c ConcurrencySettings) AutoTuneMainPool() bool
AutoTuneMainPool says whether the main pool size should by dynamically tuned
type ConcurrencyTuner ¶
type ConcurrencyTuner interface {
// GetRecommendedConcurrency is called repeatedly, at intervals decided by the caller,
// to compute recommended concurrency levels
GetRecommendedConcurrency(currentMbps int, highCpuUsage bool) (newConcurrency int, reason string)
// RequestCallbackWhenStable lets interested parties ask the concurrency tuner to call them back when the tuner has reached a stable level
RequestCallbackWhenStable(callback func()) (callbackAccepted bool)
// GetFinalState returns the final state of the tuner
GetFinalState() (finalReason string, finalRecommendedConcurrency int)
// contains filtered or unexported methods
}
func NewAutoConcurrencyTuner ¶
func NewAutoConcurrencyTuner(initial, max int, isBenchmarking bool) ConcurrencyTuner
type ConfiguredBool ¶
type ConfiguredBool struct {
Value bool
IsUserSpecified bool
EnvVarName string
DefaultSourceDesc string
}
ConfiguredBool is a boolean which may be optionally configured by user through an environment variable
func GetParallelStatFiles ¶ added in v10.12.2
func GetParallelStatFiles() *ConfiguredBool
func (*ConfiguredBool) GetDescription ¶
func (b *ConfiguredBool) GetDescription() string
type ConfiguredInt ¶
type ConfiguredInt struct {
Value int
IsUserSpecified bool
EnvVarName string
DefaultSourceDesc string
}
ConfiguredInt is an integer which may be optionally configured by user through an environment variable
func GetEnumerationPoolSize ¶ added in v10.12.2
func GetEnumerationPoolSize() *ConfiguredInt
func (*ConfiguredInt) GetDescription ¶
func (i *ConfiguredInt) GetDescription() string
type CoordinatorChannels ¶
type CoordinatorChannels struct {
// contains filtered or unexported fields
}
type DatalakeClientStub ¶ added in v10.22.0
type ErrorEx ¶
type ErrorEx struct {
// contains filtered or unexported fields
}
func (ErrorEx) ErrorCodeAndString ¶
TODO: consider rolling MSRequestID into this, so that all places that use this can pick up, and log, the request ID too
func (ErrorEx) MSRequestID ¶
MSRequestID gets the request ID guid associated with the failed request. Returns "" if there isn't one (either no request, or there is a request but it doesn't have the header)
type FileClientStub ¶ added in v10.22.0
type FileClientStub interface {
URL() string
}
type FolderCreationTracker ¶ added in v10.12.0
type FolderCreationTracker common.FolderCreationTracker
func NewFolderCreationTracker ¶ added in v10.12.0
func NewFolderCreationTracker(fpo common.FolderPropertyOption, fetcher TransferFetcher) FolderCreationTracker
NewFolderCreationTracker creates a folder creation tracker taking in a TransferFetcher (typically created by NewTransferFetcher) A TransferFetcher is used in place of an IJobMgr to make testing easier to implement.
type HandleInfo ¶ added in v10.17.0
type HandleInfo struct {
common.ByHandleFileInformation
}
func (HandleInfo) FileAccessTime ¶ added in v10.30.0
func (hi HandleInfo) FileAccessTime() time.Time
func (HandleInfo) FileAttributes ¶ added in v10.17.0
func (hi HandleInfo) FileAttributes() (*file.NTFSFileAttributes, error)
func (HandleInfo) FileCreationTime ¶ added in v10.17.0
func (hi HandleInfo) FileCreationTime() time.Time
func (HandleInfo) FileLastWriteTime ¶ added in v10.17.0
func (hi HandleInfo) FileLastWriteTime() time.Time
type HandleNFSPermissions ¶ added in v10.30.0
type HandleNFSPermissions struct {
common.UnixStatAdapter
}
func (HandleNFSPermissions) GetFileMode ¶ added in v10.30.0
func (h HandleNFSPermissions) GetFileMode() *string
func (HandleNFSPermissions) GetGroup ¶ added in v10.30.0
func (h HandleNFSPermissions) GetGroup() *string
func (HandleNFSPermissions) GetOwner ¶ added in v10.30.0
func (h HandleNFSPermissions) GetOwner() *string
type IBlobClient ¶ added in v10.21.0
type IBlobClient interface {
URL() string
GetAccountInfo(ctx context.Context, o *blob.GetAccountInfoOptions) (blob.GetAccountInfoResponse, error)
}
IBlobClient is an interface to allow ValidateTier to accept any type of client
type IBlobSourceInfoProvider ¶
type IBlobSourceInfoProvider interface {
IRemoteSourceInfoProvider
// BlobTier returns source's blob tier.
BlobTier() *blob.AccessTier
// BlobType returns source's blob type.
BlobType() blob.BlobType
}
IBlobSourceInfoProvider is the abstraction of the methods needed to prepare blob copy source.
type ICustomLocalOpener ¶
type ICustomLocalOpener interface {
ISourceInfoProvider
Open(path string) (*os.File, error)
}
type IJobMgr ¶
type IJobMgr interface {
JobID() common.JobID
JobPartMgr(partNum PartNumber) (IJobPartMgr, bool)
// Throughput() XferThroughput
// If existingPlanMMF is nil, a new MMF is opened.
AddJobPart(args *AddJobPartArgs) IJobPartMgr
SetIncludeExclude(map[string]int, map[string]int)
IncludeExclude() (map[string]int, map[string]int)
ResumeTransfers(appCtx context.Context)
ResetFailedTransfersCount()
AllTransfersScheduled() bool
ConfirmAllTransfersScheduled()
ResetAllTransfersScheduled()
GetTotalNumFilesProcessed() int64
AddTotalNumFilesProcessed(numFiles int64)
Reset(context.Context, string) IJobMgr
PipelineLogInfo() LogOptions
ReportJobPartDone(jobPartProgressInfo)
Context() context.Context
Cancel()
// TODO: added for debugging purpose. remove later
OccupyAConnection()
// TODO: added for debugging purpose. remove later
ReleaseAConnection()
// TODO: added for debugging purpose. remove later
ActiveConnections() int64
GetPerfInfo() (displayStrings []string, constraint common.PerfConstraint)
// Close()
ChunkStatusLogger() common.ChunkStatusLogger
HttpClient() *http.Client
PipelineNetworkStats() *PipelineNetworkStats
common.ILoggerCloser
/* Status related functions */
SendJobPartCreatedMsg(msg JobPartCreatedMsg)
SendXferDoneMsg(msg xferDoneMsg)
ListJobSummary() common.ListJobSummaryResponse
ResurrectSummary(js common.ListJobSummaryResponse)
/* Ported from jobsAdmin() */
ScheduleTransfer(priority common.JobPriority, jptm IJobPartTransferMgr)
ScheduleChunk(priority common.JobPriority, chunkFunc chunkFunc)
/* Some comment */
IterateJobParts(readonly bool, f func(k common.PartNumber, v IJobPartMgr))
TransferDirection() common.TransferDirection
AddSuccessfulBytesInActiveFiles(n int64)
SuccessfulBytesInActiveFiles() uint64
CancelPauseJobOrder(desiredJobStatus common.JobStatus) common.CancelPauseResumeResponse
IsDaemon() bool
GetJobErrorHandler() common.JobErrorHandler
// Cleanup Functions
DeferredCleanupJobMgr()
// contains filtered or unexported methods
}
func NewJobMgr ¶ added in v10.15.0
func NewJobMgr(concurrency ConcurrencySettings, jobID common.JobID, appCtx context.Context, cpuMon common.CPUMonitor, level common.LogLevel, commandString string, tuner ConcurrencyTuner, pacer PacerAdmin, slicePool common.ByteSlicePooler, cacheLimiter common.CacheLimiter, fileCountLimiter common.CacheLimiter, jobLogger common.ILoggerResetable, daemonMode bool, jobErrorHandler common.JobErrorHandler) IJobMgr
type IJobPartMgr ¶
type IJobPartMgr interface {
Plan() *JobPartPlanHeader
ScheduleTransfers(jobCtx context.Context)
StartJobXfer(jptm IJobPartTransferMgr)
ReportTransferDone(status common.TransferStatus) uint32
GetOverwriteOption() common.OverwriteOption
GetForceIfReadOnly() bool
AutoDecompress() bool
ScheduleChunks(chunkFunc chunkFunc)
RescheduleTransfer(jptm IJobPartTransferMgr)
BlobTypeOverride() common.BlobType
BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier)
ShouldPutMd5() bool
DeleteDestinationFileIfNecessary() bool
SAS() (string, string)
// CancelJob()
Close()
// TODO: added for debugging purpose. remove later
OccupyAConnection()
// TODO: added for debugging purpose. remove later
ReleaseAConnection()
SlicePool() common.ByteSlicePooler
CacheLimiter() common.CacheLimiter
FileCountLimiter() common.CacheLimiter
ExclusiveDestinationMap() *common.ExclusiveStringMap
ChunkStatusLogger() common.ChunkStatusLogger
common.ILogger
// These functions return Container/fileshare clients.
// They must be type asserted before use. In cases where they dont
// make sense (say SrcServiceClient for upload) they are il
SrcServiceClient() *common.ServiceClient
DstServiceClient() *common.ServiceClient
SourceIsOAuth() bool
SecurityInfoPersistenceManager() *securityInfoPersistenceManager
FolderDeletionManager() common.FolderDeletionManager
CpkInfo() *blob.CPKInfo
CpkScopeInfo() *blob.CPKScopeInfo
IsSourceEncrypted() bool
/* Status Manager Updates */
SendXferDoneMsg(msg xferDoneMsg)
PropertiesToTransfer() common.SetPropertiesFlags
ResetFailedTransfersCount() // Resets number of failed transfers after a job is resumed
GetJobErrorHandler() common.JobErrorHandler
// contains filtered or unexported methods
}
type IJobPartPlanHeader ¶ added in v10.23.0
type IJobPartPlanHeader interface {
CommandString() string
JobPartStatus() common.JobStatus
JobStatus() common.JobStatus
SetJobPartStatus(newJobStatus common.JobStatus)
SetJobStatus(newJobStatus common.JobStatus)
Transfer(transferIndex uint32) *JobPartPlanTransfer
TransferSrcDstRelatives(transferIndex uint32) (relSource string, relDest string)
TransferSrcDstStrings(transferIndex uint32) (source string, destination string, isFolder bool)
TransferSrcPropertiesAndMetadata(transferIndex uint32) (h common.ResourceHTTPHeaders, metadata common.Metadata, blobType blob.BlobType, blobTier blob.AccessTier, s2sGetPropertiesInBackend bool, DestLengthValidation bool, s2sSourceChangeValidation bool, s2sInvalidMetadataHandleOption common.InvalidMetadataHandleOption, entityType common.EntityType, blobVersionID string, blobSnapshotID string, blobTags common.BlobTags)
}
type IJobPartTransferMgr ¶
type IJobPartTransferMgr interface {
FromTo() common.FromTo
Info() *TransferInfo
ResourceDstData(dataFileToXfer []byte) (headers common.ResourceHTTPHeaders, metadata common.Metadata, blobTags common.BlobTags, cpkOptions common.CpkOptions)
LastModifiedTime() time.Time
PreserveLastModifiedTime() (time.Time, bool)
ShouldPutMd5() bool
DeleteDestinationFileIfNecessary() bool
MD5ValidationOption() common.HashValidationOption
BlobTypeOverride() common.BlobType
BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier)
JobHasLowFileCount() bool
// ScheduleChunk(chunkFunc chunkFunc)
Context() context.Context
SlicePool() common.ByteSlicePooler
CacheLimiter() common.CacheLimiter
WaitUntilLockDestination(ctx context.Context) error
EnsureDestinationUnlocked()
HoldsDestinationLock() bool
StartJobXfer()
GetOverwriteOption() common.OverwriteOption
GetForceIfReadOnly() bool
ShouldDecompress() bool
GetSourceCompressionType() (common.CompressionType, error)
ReportChunkDone(id common.ChunkID) (lastChunk bool, chunksDone uint32)
TransferStatusIgnoringCancellation() common.TransferStatus
SetStatus(status common.TransferStatus)
SetErrorCode(errorCode int32)
SetNumberOfChunks(numChunks uint32)
SetActionAfterLastChunk(f func())
ReportTransferDone() uint32
RescheduleTransfer()
ScheduleChunks(chunkFunc chunkFunc)
SetDestinationIsModified()
Cancel()
WasCanceled() bool
IsLive() bool
IsDeadBeforeStart() bool
IsDeadInflight() bool
// TODO: added for debugging purpose. remove later
OccupyAConnection()
// TODO: added for debugging purpose. remove later
ReleaseAConnection()
SrcServiceClient() *common.ServiceClient
DstServiceClient() *common.ServiceClient
GetS2SSourceTokenCredential(ctx context.Context) (token *string, err error)
FailActiveUpload(where string, err error)
FailActiveDownload(where string, err error)
FailActiveUploadWithStatus(where string, err error, failureStatus common.TransferStatus)
FailActiveDownloadWithStatus(where string, err error, failureStatus common.TransferStatus)
FailActiveS2SCopy(where string, err error)
FailActiveS2SCopyWithStatus(where string, err error, failureStatus common.TransferStatus)
// TODO: Cleanup FailActiveUpload/FailActiveUploadWithStatus & FailActiveS2SCopy/FailActiveS2SCopyWithStatus
FailActiveSend(where string, err error)
FailActiveSendWithStatus(where string, err error, failureStatus common.TransferStatus)
LogUploadError(source, destination, errorMsg string, status int)
LogDownloadError(source, destination, errorMsg string, status int)
LogS2SCopyError(source, destination, errorMsg string, status int)
LogSendError(source, destination, errorMsg string, status int)
LogError(resource, context string, err error)
LogTransferInfo(level common.LogLevel, source, destination, msg string)
LogTransferStart(source, destination, description string)
LogChunkStatus(id common.ChunkID, reason common.WaitReason)
ChunkStatusLogger() common.ChunkStatusLogger
LogAtLevelForCurrentTransfer(level common.LogLevel, msg string)
GetOverwritePrompter() *overwritePrompter
GetFolderCreationTracker() FolderCreationTracker
common.ILogger
DeleteSnapshotsOption() common.DeleteSnapshotsOption
PermanentDeleteOption() common.PermanentDeleteOption
SecurityInfoPersistenceManager() *securityInfoPersistenceManager
FolderDeletionManager() common.FolderDeletionManager
GetDestinationRoot() string
ShouldInferContentType() bool
CpkInfo() *blob.CPKInfo
CpkScopeInfo() *blob.CPKScopeInfo
IsSourceEncrypted() bool
PropertiesToTransfer() common.SetPropertiesFlags
ResetSourceSize() // sets source size to 0 (made to be used by setProperties command to make number of bytes transferred = 0)
SuccessfulBytesTransferred() int64
TransferIndex() (partNum, transferIndex uint32)
RestartedTransfer() bool
GetJobErrorHandler() common.JobErrorHandler
}
type ILocalSourceInfoProvider ¶
type ILocalSourceInfoProvider interface {
ISourceInfoProvider
OpenSourceFile() (common.CloseableReaderAt, error)
}
type INFSPropertyBearingSourceInfoProvider ¶ added in v10.30.0
type INFSPropertyBearingSourceInfoProvider interface {
ISourceInfoProvider
GetNFSProperties() (TypedNFSPropertyHolder, error)
GetNFSPermissions() (TypedNFSPermissionsHolder, error)
GetNFSDefaultPerms() (*string, *string, *string, error)
}
type IRemoteSourceInfoProvider ¶
type IRemoteSourceInfoProvider interface {
ISourceInfoProvider
// SourceURL returns source's URL.
PreSignedSourceURL() (string, error)
// SourceSize returns size of source
SourceSize() int64
// RawSource returns raw source
RawSource() string
}
IRemoteSourceInfoProvider is the abstraction of the methods needed to prepare remote copy source.
type ISMBPropertyBearingSourceInfoProvider ¶
type ISMBPropertyBearingSourceInfoProvider interface {
ISourceInfoProvider
GetSDDL() (string, error)
GetSMBProperties() (TypedSMBPropertyHolder, error)
}
type ISourceInfoProvider ¶
type ISourceInfoProvider interface {
// Properties returns source's properties.
Properties() (*SrcProperties, error)
// GetLastModifiedTime returns the source's latest last modified time. Not used when
// EntityType() == Folder
GetFreshFileLastModifiedTime() (time.Time, error)
IsLocal() bool
EntityType() common.EntityType
GetMD5(offset, count int64) ([]byte, error)
}
ISourceInfoProvider is the abstraction of generic source info provider which provides source's properties.
type ISymlinkBearingSourceInfoProvider ¶ added in v10.18.0
type ISymlinkBearingSourceInfoProvider interface {
ISourceInfoProvider
ReadLink() (string, error)
}
type IUNIXPropertyBearingSourceInfoProvider ¶ added in v10.16.0
type IUNIXPropertyBearingSourceInfoProvider interface {
ISourceInfoProvider
GetUNIXProperties() (common.UnixStatAdapter, error)
HasUNIXProperties() bool
}
type JPPTCompatibleFolderCreationTracker ¶ added in v10.12.0
type JPPTCompatibleFolderCreationTracker interface {
FolderCreationTracker
RegisterPropertiesTransfer(folder string, partNum PartNumber, transferIndex uint32)
}
type JobPartCreatedMsg ¶ added in v10.15.0
type JobPartPlanDstBlob ¶
type JobPartPlanDstBlob struct {
BlobType common.BlobType
// represents user decision to interpret the content-encoding from source file
NoGuessMimeType bool
// Specifies the length of MIME content type of the blob
ContentTypeLength uint16
// Specifies the MIME content type of the blob. The default type is application/octet-stream
ContentType [CustomHeaderMaxBytes]byte
// Specifies length of content encoding which have been applied to the blob.
ContentEncodingLength uint16
// Specifies which content encodings have been applied to the blob.
ContentEncoding [CustomHeaderMaxBytes]byte
// Specifies length of content language which has been applied to the blob.
ContentLanguageLength uint16
// Specifies which content language has been applied to the blob.
ContentLanguage [CustomHeaderMaxBytes]byte
// Specifies length of content disposition which has been applied to the blob.
ContentDispositionLength uint16
// Specifies the content disposition of the blob
ContentDisposition [CustomHeaderMaxBytes]byte
// Specifies the length of the cache control which has been applied to the blob.
CacheControlLength uint16
// Specifies the cache control of the blob
CacheControl [CustomHeaderMaxBytes]byte
// Specifies the tier if this is a block or page blob
BlockBlobTier common.BlockBlobTier
PageBlobTier common.PageBlobTier
// Controls uploading of MD5 hashes
PutMd5 bool
MetadataLength uint16
Metadata [MetadataMaxBytes]byte
BlobTagsLength uint16
BlobTags [BlobTagsMaxByte]byte
CpkInfo bool
IsSourceEncrypted bool
CpkScopeInfo [CustomHeaderMaxBytes]byte
CpkScopeInfoLength uint16
// Specifies the maximum size of block which determines the number of chunks and chunk size of a transfer
BlockSize int64
// Specifies the maximum size of a blob which can be uploaded by a single PUT request.
PutBlobSize int64
SetPropertiesFlags common.SetPropertiesFlags
DeleteDestinationFileIfNecessary bool
}
JobPartPlanDstBlob holds additional settings required when the destination is a blob
type JobPartPlanDstFile ¶ added in v10.19.0
type JobPartPlanDstFile struct {
TrailingDot common.TrailingDotOption
}
JobPartPlanDstFile holds additional settings required when the destination is a file
type JobPartPlanDstLocal ¶
type JobPartPlanDstLocal struct {
// Specifies whether the timestamp of destination file has to be set to the modified time of source file
PreserveLastModifiedTime bool
// says how MD5 verification failures should be actioned
MD5VerificationOption common.HashValidationOption
}
jobPartPlanDstLocal holds additional settings required when the destination is a local file
type JobPartPlanFileName ¶
type JobPartPlanFileName string
func (JobPartPlanFileName) Create ¶
func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest)
createJobPartPlanFile creates the memory map JobPartPlanHeader using the given JobPartOrder and JobPartPlanBlobData
func (JobPartPlanFileName) Delete ¶
func (jpfn JobPartPlanFileName) Delete() error
func (*JobPartPlanFileName) Exists ¶ added in v10.14.0
func (jppfn *JobPartPlanFileName) Exists() bool
func (*JobPartPlanFileName) GetJobPartPlanPath ¶
func (jppfn *JobPartPlanFileName) GetJobPartPlanPath() string
func (JobPartPlanFileName) Map ¶
func (jpfn JobPartPlanFileName) Map() *JobPartPlanMMF
func (JobPartPlanFileName) Parse ¶
func (jpfn JobPartPlanFileName) Parse() (jobID common.JobID, partNumber common.PartNumber, err error)
TODO: This needs testing
type JobPartPlanHeader ¶
type JobPartPlanHeader struct {
// Once set, the following fields are constants; they should never be modified
Version common.Version // The version of data schema format of header; see the dataSchemaVersion constant
StartTime int64 // The start time of this part
JobID common.JobID // Job Part's JobID
PartNum common.PartNumber // Job Part's part number (0+)
SourceRootLength uint16 // The length of the source root path
SourceRoot [1000]byte // The root directory of the source
SourceExtraQueryLength uint16
SourceExtraQuery [1000]byte // Extra query params applicable to the source
DestinationRootLength uint16 // The length of the destination root path
DestinationRoot [1000]byte // The root directory of the destination
DestExtraQueryLength uint16
DestExtraQuery [1000]byte // Extra query params applicable to the dest
IsFinalPart bool // True if this is the Job's last part; else false
ForceWrite common.OverwriteOption // True if the existing blobs needs to be overwritten.
ForceIfReadOnly bool // Supplements ForceWrite with an additional setting for Azure Files. If true, the read-only attribute will be cleared before we overwrite
AutoDecompress bool // if true, source data with encodings that represent compression are automatically decompressed when downloading
Priority common.JobPriority // The Job Part's priority
TTLAfterCompletion uint32 // Time to live after completion is used to persists the file on disk of specified time after the completion of JobPartOrder
FromTo common.FromTo // The location of the transfer's source & destination
Fpo common.FolderPropertyOption // option specifying how folders will be handled
CommandStringLength uint32
NumTransfers uint32 // The number of transfers in the Job part
LogLevel common.LogLevel // This Job Part's minimal log level
DstBlobData JobPartPlanDstBlob // Additional data for blob destinations
DstLocalData JobPartPlanDstLocal // Additional data for local destinations
DstFileData JobPartPlanDstFile // Additional data for file destinations
PreservePermissions common.PreservePermissionsOption
PreserveInfo bool
PreservePOSIXProperties bool
// S2SGetPropertiesInBackend represents whether to enable get S3 objects' or Azure files' properties during s2s copy in backend.
S2SGetPropertiesInBackend bool
// S2SSourceChangeValidation represents whether user wants to check if source has changed after enumerating.
S2SSourceChangeValidation bool
// DestLengthValidation represents whether the user wants to check if the destination has a different content-length
DestLengthValidation bool
// S2SInvalidMetadataHandleOption represents how user wants to handle invalid metadata.
S2SInvalidMetadataHandleOption common.InvalidMetadataHandleOption
// BlobFSRecursiveDelete represents whether the user wants to make a recursive call to the DFS endpoint or not
BlobFSRecursiveDelete bool
// For delete operation specify what to do with snapshots
DeleteSnapshotsOption common.DeleteSnapshotsOption
// Determine what to do with soft-deleted snapshots
PermanentDeleteOption common.PermanentDeleteOption
RehydratePriority common.RehydratePriorityType
SymlinkHandling common.SymlinkHandlingType
// contains filtered or unexported fields
}
JobPartPlanHeader represents the header of Job Part's memory-mapped file
func (*JobPartPlanHeader) CommandString ¶
func (jpph *JobPartPlanHeader) CommandString() string
CommandString returns the command string given by user when job was created
func (*JobPartPlanHeader) JobPartStatus ¶ added in v10.15.0
func (jpph *JobPartPlanHeader) JobPartStatus() common.JobStatus
func (*JobPartPlanHeader) JobStatus ¶
func (jpph *JobPartPlanHeader) JobStatus() common.JobStatus
Status returns the job status stored in JobPartPlanHeader in thread-safe manner
func (*JobPartPlanHeader) SetJobPartStatus ¶ added in v10.15.0
func (jpph *JobPartPlanHeader) SetJobPartStatus(newJobStatus common.JobStatus)
func (*JobPartPlanHeader) SetJobStatus ¶
func (jpph *JobPartPlanHeader) SetJobStatus(newJobStatus common.JobStatus)
SetJobStatus sets the job status in JobPartPlanHeader in thread-safe manner
func (*JobPartPlanHeader) Transfer ¶
func (jpph *JobPartPlanHeader) Transfer(transferIndex uint32) *JobPartPlanTransfer
Transfer api gives memory map JobPartPlanTransfer header for given index
func (*JobPartPlanHeader) TransferSrcDstRelatives ¶ added in v10.12.0
func (jpph *JobPartPlanHeader) TransferSrcDstRelatives(transferIndex uint32) (relSource, relDest string)
func (*JobPartPlanHeader) TransferSrcDstStrings ¶
func (jpph *JobPartPlanHeader) TransferSrcDstStrings(transferIndex uint32) (source, destination string, isFolder bool)
TransferSrcDstDetail returns the source and destination string for a transfer at given transferIndex in JobPartOrder Also indication of entity type since that's often necessary to avoid ambiguity about what the source and dest are
func (*JobPartPlanHeader) TransferSrcPropertiesAndMetadata ¶
func (jpph *JobPartPlanHeader) TransferSrcPropertiesAndMetadata(transferIndex uint32) (h common.ResourceHTTPHeaders, metadata common.Metadata, blobType blob.BlobType, blobTier blob.AccessTier, s2sGetPropertiesInBackend bool, DestLengthValidation bool, s2sSourceChangeValidation bool, s2sInvalidMetadataHandleOption common.InvalidMetadataHandleOption, entityType common.EntityType, blobVersionID string, blobSnapshotID string, blobTags common.BlobTags)
TransferSrcPropertiesAndMetadata returns the SrcHTTPHeaders, properties and metadata for a transfer at given transferIndex in JobPartOrder TODO: Refactor return type to an object
type JobPartPlanMMF ¶
func (*JobPartPlanMMF) Plan ¶
func (mmf *JobPartPlanMMF) Plan() *JobPartPlanHeader
func (*JobPartPlanMMF) Unmap ¶
func (mmf *JobPartPlanMMF) Unmap()
type JobPartPlanTransfer ¶
type JobPartPlanTransfer struct {
// SrcOffset represents the actual start offset transfer header written in JobPartOrder file
SrcOffset int64
// SrcLength represents the actual length of source string for specific transfer
SrcLength int16
// DstLength represents the actual length of destination string for specific transfer
DstLength int16
// ChunkCount represents the num of chunks a transfer is split into
// ChunkCount uint16 // TODO: Remove this, we need to determine it at runtime
// EntityType indicates whether this is a file or a folder
// We use a dedicated field for this because the alternative (of doing something fancy the names) was too complex and error-prone
EntityType common.EntityType
// ModifiedTime represents the last time at which source was modified before start of transfer stored as nanoseconds.
ModifiedTime int64
// SourceSize represents the actual size of the source on disk
SourceSize int64
// CompletionTime represents the time at which transfer was completed
CompletionTime uint64
// For S2S copy, per Transfer source's properties
// TODO: ensure the length is enough
SrcContentTypeLength int16
SrcContentEncodingLength int16
SrcContentLanguageLength int16
SrcContentDispositionLength int16
SrcCacheControlLength int16
SrcContentMD5Length int16
SrcMetadataLength int16
SrcBlobTypeLength int16
SrcBlobTierLength int16
SrcBlobVersionIDLength int16
SrcBlobSnapshotIDLength int16
SrcBlobTagsLength int16
// contains filtered or unexported fields
}
JobPartPlanTransfer represent the header of Job Part's Transfer in Memory Map File
func (*JobPartPlanTransfer) ErrorCode ¶
func (jppt *JobPartPlanTransfer) ErrorCode() int32
ErrorCode returns the transfer's errorCode.
func (*JobPartPlanTransfer) SetErrorCode ¶
func (jppt *JobPartPlanTransfer) SetErrorCode(errorCode int32, overwrite bool)
SetErrorCode sets the error code of the error if transfer failed. overWrite flags if set to true overWrites the atomicErrorCode. If overWrite flag is set to false, then errorCode won't be overwritten.
func (*JobPartPlanTransfer) SetTransferStatus ¶
func (jppt *JobPartPlanTransfer) SetTransferStatus(status common.TransferStatus, overWrite bool)
SetTransferStatus sets the transfer's status overWrite flags if set to true overWrites the failed status. If overWrite flag is set to false, then status of transfer is set to failed won't be overWritten. overWrite flag is used while resuming the failed transfers where the errorCode are set to default i.e 0
func (*JobPartPlanTransfer) TransferStatus ¶
func (jppt *JobPartPlanTransfer) TransferStatus() common.TransferStatus
TransferStatus returns the transfer's status
type JpptFolderIndex ¶ added in v10.31.0
type JpptFolderIndex struct {
PartNum PartNumber
TransferIndex uint32
}
type LogOptions ¶ added in v10.21.0
type LogOptions struct {
// TODO : Unravel LogOptions and RequestLogOptions
RequestLogOptions RequestLogOptions
Log func(level common.LogLevel, message string)
// ShouldLog is called periodically allowing you to return whether the specified LogLevel should be logged or not.
// An application can return different values over the its lifetime; this allows the application to dynamically
// alter what is logged. NOTE: This method can be called by multiple goroutines simultaneously so make sure
// you implement it in a goroutine-safe way. If nil, nothing is logged (the equivalent of returning LogNone).
// Usually, the function will be implemented simply like this: return level <= LogWarning
ShouldLog func(level common.LogLevel) bool
}
type NFSProperties ¶ added in v10.30.0
type NullConcurrencyTuner ¶ added in v10.15.0
type NullConcurrencyTuner struct {
FixedValue int
}
func (*NullConcurrencyTuner) GetFinalState ¶ added in v10.15.0
func (n *NullConcurrencyTuner) GetFinalState() (finalReason string, finalRecommendedConcurrency int)
func (*NullConcurrencyTuner) GetRecommendedConcurrency ¶ added in v10.15.0
func (n *NullConcurrencyTuner) GetRecommendedConcurrency(currentMbps int, highCpuUsage bool) (newConcurrency int, reason string)
func (*NullConcurrencyTuner) RequestCallbackWhenStable ¶ added in v10.15.0
func (n *NullConcurrencyTuner) RequestCallbackWhenStable(callback func()) (callbackAccepted bool)
type PacerAdmin ¶ added in v10.15.0
type PacerAdmin interface {
// GetTotalTraffic returns the cumulative count of all traffic that has been processed
GetTotalTraffic() int64
// contains filtered or unexported methods
}
type PartNumber ¶
type PartNumber = common.PartNumber
type PerformanceAdvisor ¶
type PerformanceAdvisor struct {
// contains filtered or unexported fields
}
func NewPerformanceAdvisor ¶
func NewPerformanceAdvisor(stats *PipelineNetworkStats, commandLineMbpsCap float64, mbps int64, finalReason string, finalConcurrency int, dir common.TransferDirection, avgBytesPerFile int64, isToAzureFiles bool) *PerformanceAdvisor
func (*PerformanceAdvisor) GetAdvice ¶
func (p *PerformanceAdvisor) GetAdvice() []common.PerformanceAdvice
GetPerfAdvice returns one or many performance advice objects, in priority order, with the highest priority advice first
type PipelineNetworkStats ¶ added in v10.15.0
type PipelineNetworkStats struct {
// contains filtered or unexported fields
}
func (*PipelineNetworkStats) AverageE2EMilliseconds ¶ added in v10.15.0
func (s *PipelineNetworkStats) AverageE2EMilliseconds() int
func (*PipelineNetworkStats) GetTotalRetries ¶ added in v10.15.0
func (s *PipelineNetworkStats) GetTotalRetries() int64
func (*PipelineNetworkStats) IOPSServerBusyPercentage ¶ added in v10.15.0
func (s *PipelineNetworkStats) IOPSServerBusyPercentage() float32
func (*PipelineNetworkStats) NetworkErrorPercentage ¶ added in v10.15.0
func (s *PipelineNetworkStats) NetworkErrorPercentage() float32
func (*PipelineNetworkStats) OperationsPerSecond ¶ added in v10.15.0
func (s *PipelineNetworkStats) OperationsPerSecond() int
func (*PipelineNetworkStats) OtherServerBusyPercentage ¶ added in v10.15.0
func (s *PipelineNetworkStats) OtherServerBusyPercentage() float32
func (*PipelineNetworkStats) ThroughputServerBusyPercentage ¶ added in v10.15.0
func (s *PipelineNetworkStats) ThroughputServerBusyPercentage() float32
func (*PipelineNetworkStats) TotalServerBusyPercentage ¶ added in v10.15.0
func (s *PipelineNetworkStats) TotalServerBusyPercentage() float32
type RequestLogOptions ¶
type RequestLogOptions struct {
// LogWarningIfTryOverThreshold logs a warning if a tried operation takes longer than the specified
// duration (-1=no logging; 0=default threshold).
LogWarningIfTryOverThreshold time.Duration
// SyslogDisabled is a flag to check if logging to Syslog/Windows-Event-Logger is enabled or not
// We by default print to Syslog/Windows-Event-Logger.
// If SyslogDisabled is not provided explicitly, the default value will be false.
SyslogDisabled bool
}
RequestLogOptions configures the retry policy's behavior.
type RetryCodes ¶ added in v10.22.0
type RetryCodes map[int]StorageErrorCodes // where int is the HTTP status code
var RetryStatusCodes RetryCodes
Defines the retry policy rules
func ParseRetryCodes ¶ added in v10.22.0
func ParseRetryCodes(s string) (RetryCodes, error)
ParseRetryCodes takes a string and returns a RetryCodes object Format: <http status code>: <storage error code>, <storage error code>; <http status code>: <storage error code>; <http status code> Remove entire status codes with a negative status code. Remove storage error codes by specifying them under a status code.
type RetryFunc ¶ added in v10.29.0
func GetShouldRetry ¶ added in v10.29.0
func GetShouldRetry(log *LogOptions) RetryFunc
type SrcProperties ¶
type StatTAdapter ¶ added in v10.18.0
func (StatTAdapter) ATime ¶ added in v10.18.0
func (s StatTAdapter) ATime() time.Time
func (StatTAdapter) Attribute ¶ added in v10.18.0
func (s StatTAdapter) Attribute() uint64
func (StatTAdapter) AttributeMask ¶ added in v10.18.0
func (s StatTAdapter) AttributeMask() uint64
func (StatTAdapter) BTime ¶ added in v10.18.0
func (s StatTAdapter) BTime() time.Time
func (StatTAdapter) CTime ¶ added in v10.18.0
func (s StatTAdapter) CTime() time.Time
func (StatTAdapter) Device ¶ added in v10.18.0
func (s StatTAdapter) Device() uint64
func (StatTAdapter) Extended ¶ added in v10.18.0
func (s StatTAdapter) Extended() bool
func (StatTAdapter) FileMode ¶ added in v10.18.0
func (s StatTAdapter) FileMode() uint32
func (StatTAdapter) Group ¶ added in v10.18.0
func (s StatTAdapter) Group() uint32
func (StatTAdapter) INode ¶ added in v10.18.0
func (s StatTAdapter) INode() uint64
func (StatTAdapter) MTime ¶ added in v10.18.0
func (s StatTAdapter) MTime() time.Time
func (StatTAdapter) NLink ¶ added in v10.18.0
func (s StatTAdapter) NLink() uint64
func (StatTAdapter) Owner ¶ added in v10.18.0
func (s StatTAdapter) Owner() uint32
func (StatTAdapter) RDevice ¶ added in v10.18.0
func (s StatTAdapter) RDevice() uint64
func (StatTAdapter) StatxMask ¶ added in v10.18.0
func (s StatTAdapter) StatxMask() uint32
type StatxTAdapter ¶ added in v10.18.0
func (StatxTAdapter) ATime ¶ added in v10.18.0
func (s StatxTAdapter) ATime() time.Time
func (StatxTAdapter) Attribute ¶ added in v10.18.0
func (s StatxTAdapter) Attribute() uint64
func (StatxTAdapter) AttributeMask ¶ added in v10.18.0
func (s StatxTAdapter) AttributeMask() uint64
func (StatxTAdapter) BTime ¶ added in v10.18.0
func (s StatxTAdapter) BTime() time.Time
func (StatxTAdapter) CTime ¶ added in v10.18.0
func (s StatxTAdapter) CTime() time.Time
func (StatxTAdapter) Device ¶ added in v10.18.0
func (s StatxTAdapter) Device() uint64
func (StatxTAdapter) Extended ¶ added in v10.18.0
func (s StatxTAdapter) Extended() bool
func (StatxTAdapter) FileMode ¶ added in v10.18.0
func (s StatxTAdapter) FileMode() uint32
func (StatxTAdapter) Group ¶ added in v10.18.0
func (s StatxTAdapter) Group() uint32
func (StatxTAdapter) INode ¶ added in v10.18.0
func (s StatxTAdapter) INode() uint64
func (StatxTAdapter) MTime ¶ added in v10.18.0
func (s StatxTAdapter) MTime() time.Time
func (StatxTAdapter) NLink ¶ added in v10.18.0
func (s StatxTAdapter) NLink() uint64
func (StatxTAdapter) Owner ¶ added in v10.18.0
func (s StatxTAdapter) Owner() uint32
func (StatxTAdapter) RDevice ¶ added in v10.18.0
func (s StatxTAdapter) RDevice() uint64
func (StatxTAdapter) StatxMask ¶ added in v10.18.0
func (s StatxTAdapter) StatxMask() uint32
type StorageErrorCodes ¶ added in v10.22.0
type StorageErrorCodes map[string]bool // where map[string]bool is the set of storage error codes; true = retry, = no retry
func MergeStorageErrorCodes ¶ added in v10.29.0
func MergeStorageErrorCodes(original, delta StorageErrorCodes) StorageErrorCodes
func ParseStorageErrorCodes ¶ added in v10.22.0
func ParseStorageErrorCodes(s string, doRetry bool) StorageErrorCodes
ParseStorageErrorCodes takes a string and returns a StorageErrorCodes object Format: comma separated list of strings that represent storage error codes
func (StorageErrorCodes) GetWildcardPolicy ¶ added in v10.29.0
func (s StorageErrorCodes) GetWildcardPolicy() bool
func (StorageErrorCodes) SetWildcardPolicy ¶ added in v10.29.0
func (s StorageErrorCodes) SetWildcardPolicy(policy bool)
type TransferFetcher ¶ added in v10.31.0
type TransferFetcher = func(index JpptFolderIndex) *JobPartPlanTransfer
func NewTransferFetcher ¶ added in v10.31.0
func NewTransferFetcher(jobMgr IJobMgr) TransferFetcher
type TransferInfo ¶
type TransferInfo struct {
JobID common.JobID
BlockSize int64
PutBlobSize int64
Source string
SourceSize int64
Destination string
EntityType common.EntityType
PreservePermissions common.PreservePermissionsOption
PreserveInfo bool
PreservePOSIXProperties bool
BlobFSRecursiveDelete bool
// Paths of targets excluding the container/fileshare name.
// ie. for https://acc1.blob.core.windows.net/c1/a/b/c/d.txt,
// SourceFilePath (or destination) would be a/b/c/d.txt.
// If they point to local resources, these strings would be empty.
SrcContainer string
DstContainer string
SrcFilePath string
DstFilePath string
// Transfer info for S2S copy
SrcProperties
S2SGetPropertiesInBackend bool
S2SSourceChangeValidation bool
DestLengthValidation bool
S2SInvalidMetadataHandleOption common.InvalidMetadataHandleOption
// Blob
SrcBlobType blob.BlobType // used for both S2S and for downloads to local from blob
S2SSrcBlobTier blob.AccessTier // AccessTierType (string) is used to accommodate service-side support matrix change.
RehydratePriority blob.RehydratePriority
VersionID string
SnapshotID string
}
TransferInfo is a per path object that needs to be transferred
func (*TransferInfo) IsFilePropertiesTransfer ¶ added in v10.18.0
func (i *TransferInfo) IsFilePropertiesTransfer() bool
func (*TransferInfo) IsFolderPropertiesTransfer ¶
func (i *TransferInfo) IsFolderPropertiesTransfer() bool
func (*TransferInfo) ShouldTransferLastWriteTime ¶
func (i *TransferInfo) ShouldTransferLastWriteTime() bool
We don't preserve LMTs on folders. The main reason is that preserving folder LMTs at download time is very difficult, because it requires us to keep track of when the last file has been saved in each folder OR just do all the folders at the very end. This is because if we modify the contents of a folder after setting its LMT, then the LMT will change because Windows and Linux (and presumably MacOS) automatically update the folder LMT when the contents are changed. The possible solutions to this problem may become difficult on very large jobs (e.g. 10s or hundreds of millions of files, with millions of directories). The secondary reason is that folder LMT's don't actually tell the user anything particularly useful. Specifically, they do NOT tell you when the folder contents (recursively) were last updated: in Azure Files they are never updated when folder contents change; and in NTFS they are only updated when immediate children are changed (not grandchildren).
type TypedNFSPermissionsHolder ¶ added in v10.30.0
type TypedNFSPropertyHolder ¶ added in v10.30.0
type TypedSMBPropertyHolder ¶
type XferChannels ¶
type XferChannels struct {
// contains filtered or unexported fields
}
Source Files
¶
- ErrorExt.go
- JobPartPlan.go
- JobPartPlanFileName.go
- concurrency.go
- concurrencyTuner.go
- destReauthPolicy.go
- downloader-azureFiles.go
- downloader-azureFiles_linux.go
- downloader-blob.go
- downloader-blobFS.go
- downloader-blobFS_linux.go
- downloader-blob_linux.go
- downloader.go
- emptyCloseableReaderAt.go
- fileAttributesHelper.go
- folderCreationTracker.go
- jobStatusManager.go
- md5Comparer.go
- mgr-JobMgr.go
- mgr-JobPartMgr.go
- mgr-JobPartTransferMgr.go
- overwritePrompter.go
- pacedReadSeeker.go
- pacer-autoPacer.go
- pacer-nullAutoPacer.go
- pacer-tokenBucketPacer.go
- pageRangeOptimizer.go
- performanceAdvisor.go
- putListNeed.go
- remoteObjectExists.go
- request_priority_policy.go
- s2sCopier-URLToBlob.go
- securityInfoPersistenceManager.go
- sender-appendBlob.go
- sender-appendBlobFromLocal.go
- sender-appendBlobFromURL.go
- sender-azureFile-helper.go
- sender-azureFile.go
- sender-azureFileFromLocal.go
- sender-azureFileFromURL.go
- sender-blobFS.go
- sender-blobFSFromLocal.go
- sender-blobFolders.go
- sender-blobFolders_linux.go
- sender-blobSymlinks.go
- sender-blobSymlinks_linux.go
- sender-blockBlob.go
- sender-blockBlobFromLocal.go
- sender-blockBlobFromURL.go
- sender-pageBlob.go
- sender-pageBlobFromLocal.go
- sender-pageBlobFromURL.go
- sender.go
- sourceAuthPolicy.go
- sourceInfoProvider-Benchmark.go
- sourceInfoProvider-Blob.go
- sourceInfoProvider-File.go
- sourceInfoProvider-GCP.go
- sourceInfoProvider-Local.go
- sourceInfoProvider-Local_linux.go
- sourceInfoProvider-S3.go
- sourceInfoProvider.go
- ste-pathUtils.go
- xfer-anyToRemote-file.go
- xfer-anyToRemote-fileProperties.go
- xfer-anyToRemote-folder.go
- xfer-anyToRemote-symlink.go
- xfer-deleteBlob.go
- xfer-deleteBlobFS.go
- xfer-deleteFile.go
- xfer-remoteToLocal-file.go
- xfer-remoteToLocal-folder.go
- xfer-remoteToLocal-symlink.go
- xfer-setProperties.go
- xfer.go
- xferLogPolicy.go
- xferRetryHelper.go
- xferRetryHelper_unix.go
- xferRetryNotificationPolicy.go
- xferStatsPolicy.go
- xferVersionPolicy.go