crash.software
Projects
Pull Requests
Issues
Builds
dolt
Code
Files
Commits
Branches
Tags
Pull Requests
Code Comments
Code Compare
Issues
List
Boards
Milestones
Builds
Statistics
Contributions
Source Lines
Child Projects
Projects
STRLCPY
dolt
Files
🤬
Sign In
2e4d56cf
ROOT
/
go
/
libraries
/
doltcore
/
remotestorage
/
chunk_store.go
Search
History
1401 lines
|
ISO-8859-1
|
39 KB
Blame
Outline
2
4
8
No wrap
Soft wrap
Outline
ErrCacheCapacityExceeded
DownloadHedger
:
*Hedger
init
()
ErrUploadFailed
globalHttpFetcher
:
HTTPFetcher
_
:
nbs.TableFileStore
_
:
nbs.NBSCompressedChunkStore
_
:
chunks.ChunkStore
_
:
chunks.LoggingChunkStore
downThroughputCheck
downRetryCount
uploadRetryCount
tracer
downloadBackOff
(ctx context.Context)
:
backoff.BackOff
uploadBackOff
(ctx context.Context)
:
backoff.BackOff
HedgeDownloadSizeLimit
HTTPFetcher
Do
(req *http.Request)
:
(*http.Response, error)
ConcurrencyParams
ConcurrentSmallFetches
:
int
ConcurrentLargeFetches
:
int
LargeFetchSize
:
int
DoltChunkStore
repoId
:
*remotesapi.RepoId
repoPath
:
string
repoToken
:
*atomic.Value
host
:
string
csClient
:
remotesapi.ChunkStoreServiceClient
cache
:
ChunkCache
metadata
:
*remotesapi.GetRepoMetadataResponse
nbf
:
*types.NomsBinFormat
httpFetcher
:
HTTPFetcher
concurrency
:
ConcurrencyParams
stats
:
cacheStats
logger
:
chunks.DebugLogger
NewDoltChunkStoreFromPath
(ctx context.Context, nbf *types.NomsBinFormat, path, host string, csClient remotesapi.ChunkStoreServiceClient)
:
(*DoltChunkStore, error)
cacheStats
Hits
:
uint32
CacheHits
()
:
uint32
CacheStats
CacheHits
()
:
uint32
getLocsBatchSize
GetRange
sortRangesBySize
(ranges []*GetRange)
resourcePathToUrlFunc
locationRefresh
RefreshAfter
:
time.Time
RefreshRequest
:
*remotesapi.RefreshTableFileUrlRequest
URL
:
string
lastRefresh
:
time.Time
mu
:
*sync.Mutex
refreshTableFileURLRetryDuration
dlLocations
ranges
:
map[string]*GetRange
refreshes
:
map[string]*locationRefresh
newDlLocations
()
:
dlLocations
RepoRequest
SetRepoId
(*remotesapi.RepoId)
SetRepoToken
(string)
SetRepoPath
(string)
maxHasManyBatchSize
Sizer
Size
()
:
int64
HttpPostUpload
(ctx context.Context, httpFetcher HTTPFetcher, post *remotesapi.HttpPostTableFile, contentHash []byte, contentLength int64, body io.ReadCloser)
:
error
aggregateDownloads
(aggDistance uint64, resourceGets map[string]*GetRange)
:
[]*GetRange
chunkAggDistance
MaxFetchSize
defaultConcurrency
:
ConcurrencyParams
logDownloadStats
(span trace.Span, originalGets map[string]*GetRange, computedGets []*GetRange)
urlFactoryFunc
hedgedRangeDownloadWithRetries
(ctx context.Context, stats StatsRecorder, fetcher HTTPFetcher, offset, length uint64, urlStrF urlFactoryFunc)
:
([]byte, error)
rangeDownloadWithRetries
(ctx context.Context, stats StatsRecorder, fetcher HTTPFetcher, offset, length uint64, hedgeN int, urlStrF urlFactoryFunc)
:
([]byte, error)
collapseBuffers
(bufs [][]byte, length uint64)
:
[]byte
getTableFiles
(dcs *DoltChunkStore, infoList []*remotesapi.TableFileInfo)
:
[]nbs.TableFile
DoltRemoteTableFile
dcs
:
*DoltChunkStore
info
:
*remotesapi.TableFileInfo
FileID
()
:
string
NumChunks
()
:
int
Open
(ctx context.Context)
:
(io.ReadCloser, uint64, error)
ErrRemoteTableFileGet
sanitizeSignedUrl
(url string)
:
string
*DoltChunkStore
WithHTTPFetcher
(fetcher HTTPFetcher)
:
*DoltChunkStore
WithNoopChunkCache
()
:
*DoltChunkStore
WithChunkCache
(cache ChunkCache)
:
*DoltChunkStore
WithDownloadConcurrency
(concurrency ConcurrencyParams)
:
*DoltChunkStore
SetLogger
(logger chunks.DebugLogger)
logf
(fmt string, args ...interface{})
getRepoId
()
:
(*remotesapi.RepoId, string)
Get
(ctx context.Context, h hash.Hash)
:
(chunks.Chunk, error)
GetMany
(ctx context.Context, hashes hash.HashSet, found func(context.Context, *chunks.Chunk))
:
error
GetManyCompressed
(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk))
:
error
getDLLocs
(ctx context.Context, hashes []hash.Hash)
:
(dlLocations, error)
readChunksAndCache
(ctx context.Context, hashes hash.HashSet, notCached []hash.Hash, found func(context.Context, nbs.CompressedChunk))
:
error
Has
(ctx context.Context, h hash.Hash)
:
(bool, error)
HasMany
(ctx context.Context, hashes hash.HashSet)
:
(hash.HashSet, error)
Put
(ctx context.Context, c chunks.Chunk)
:
error
Version
()
:
string
Rebase
(ctx context.Context)
:
error
refreshRepoMetadata
(ctx context.Context)
:
error
Root
(ctx context.Context)
:
(hash.Hash, error)
Commit
(ctx context.Context, current, last hash.Hash)
:
(bool, error)
Stats
()
:
interface{}
StatsSummary
()
:
string
Close
()
:
error
uploadChunks
(ctx context.Context)
:
(map[hash.Hash]int, error)
uploadTableFileWithRetries
(ctx context.Context, tableFileId hash.Hash, numChunks uint64, tableFileContentHash []byte, getContent func() (io.ReadCloser, uint64, error))
:
error
httpPostUpload
(ctx context.Context, post *remotesapi.HttpPostTableFile, contentHash []byte, contentLength int64, body io.ReadCloser)
:
error
downloadChunks
(ctx context.Context, dlLocs dlLocations, chunkChan chan nbs.CompressedChunk)
:
error
SupportedOperations
()
:
nbs.TableFileStoreOps
WriteTableFile
(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error))
:
error
AddTableFilesToManifest
(ctx context.Context, fileIdToNumChunks map[string]int)
:
error
PruneTableFiles
(ctx context.Context)
:
error
Sources
(ctx context.Context)
:
(hash.Hash, []nbs.TableFile, []nbs.TableFile, error)
Size
(ctx context.Context)
:
(uint64, error)
SetRootChunk
(ctx context.Context, root, previous hash.Hash)
:
error
*GetRange
ResourcePath
()
:
string
Append
(other *GetRange)
Sort
()
ChunkStartOffset
(i int)
:
uint64
ChunkEndOffset
(i int)
:
uint64
GapBetween
(i, j int)
:
uint64
SplitAtGaps
(maxGapBytes uint64)
:
[]*GetRange
NumChunks
()
:
int
RangeLen
()
:
uint64
NumBytesInRanges
()
:
uint64
ChunkByteRange
(i int)
:
(uint64, uint64)
GetDownloadFunc
(ctx context.Context, stats StatsRecorder, fetcher HTTPFetcher, chunkChan chan nbs.CompressedChunk, pathToUrl resourcePathToUrlFunc)
:
func() error
*locationRefresh
Add
(resp *remotesapi.DownloadLoc)
GetURL
(ctx context.Context, lastError error, client remotesapi.ChunkStoreServiceClient)
:
(string, error)
*dlLocations
Add
(resp *remotesapi.DownloadLoc)
All occurrences
Please wait...
Page is in error, reload to recover