chunk_store.go at v0.41.7 - STRLCPY/dolt

ROOT /

go /

remotestorage /

chunk_store.go

1394 lines | ISO-8859-1 | 38 KB

Blame Outline

Outline

ErrCacheCapacityExceeded

DownloadHedger : *Hedger

ErrUploadFailed

globalHttpFetcher : HTTPFetcher

_ : nbs.TableFileStore

_ : nbs.NBSCompressedChunkStore

_ : chunks.ChunkStore

_ : chunks.LoggingChunkStore

downThroughputCheck

uploadRetryCount

uploadRetryParams

downRetryParams

HedgeDownloadSizeLimit

Do(req *http.Request) : (*http.Response, error)

ConcurrencyParams

ConcurrentSmallFetches : int

ConcurrentLargeFetches : int

LargeFetchSize : int

NewDoltChunkStoreFromPath(ctx context.Context, nbf *types.NomsBinFormat, path, host string, csClient remotesapi.ChunkStoreServiceClient) : (*DoltChunkStore, error)

CacheHits() : uint32

CacheHits() : uint32

getLocsBatchSize

sortRangesBySize(ranges []*GetRange)

resourcePathToUrlFunc

locationRefresh

RefreshAfter : time.Time

RefreshRequest : *remotesapi.RefreshTableFileUrlRequest

lastRefresh : time.Time

mu : *sync.Mutex

refreshTableFileURLRetryDuration

ranges : map[string]*GetRange

refreshes : map[string]*locationRefresh

newDlLocations() : dlLocations

SetRepoId(*remotesapi.RepoId)

SetRepoToken(string)

SetRepoPath(string)

maxHasManyBatchSize

HttpPostUpload(ctx context.Context, httpFetcher HTTPFetcher, post *remotesapi.HttpPostTableFile, contentHash []byte, contentLength int64, body io.ReadCloser) : error

aggregateDownloads(aggDistance uint64, resourceGets map[string]*GetRange) : []*GetRange

chunkAggDistance

defaultConcurrency : ConcurrencyParams

logDownloadStats(span trace.Span, originalGets map[string]*GetRange, computedGets []*GetRange)

hedgedRangeDownloadWithRetries(ctx context.Context, stats StatsRecorder, fetcher HTTPFetcher, offset, length uint64, urlStrF urlFactoryFunc) : ([]byte, error)

rangeDownloadWithRetries(ctx context.Context, stats StatsRecorder, fetcher HTTPFetcher, offset, length uint64, hedgeN int, urlStrF urlFactoryFunc) : ([]byte, error)

collapseBuffers(bufs [][]byte, length uint64) : []byte

getTableFiles(dcs *DoltChunkStore, infoList []*remotesapi.TableFileInfo) : []nbs.TableFile

DoltRemoteTableFile

dcs : *DoltChunkStore

info : *remotesapi.TableFileInfo

FileID() : string

NumChunks() : int

Open(ctx context.Context) : (io.ReadCloser, uint64, error)

ErrRemoteTableFileGet

sanitizeSignedUrl(url string) : string

*DoltChunkStore

WithHTTPFetcher(fetcher HTTPFetcher) : *DoltChunkStore

WithNoopChunkCache() : *DoltChunkStore

WithChunkCache(cache ChunkCache) : *DoltChunkStore

WithDownloadConcurrency(concurrency ConcurrencyParams) : *DoltChunkStore

SetLogger(logger chunks.DebugLogger)

logf(fmt string, args ...interface{})

getRepoId() : (*remotesapi.RepoId, string)

Get(ctx context.Context, h hash.Hash) : (chunks.Chunk, error)

GetMany(ctx context.Context, hashes hash.HashSet, found func(context.Context, *chunks.Chunk)) : error

GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) : error

getDLLocs(ctx context.Context, hashes []hash.Hash) : (dlLocations, error)

readChunksAndCache(ctx context.Context, hashes hash.HashSet, notCached []hash.Hash, found func(context.Context, nbs.CompressedChunk)) : error

Has(ctx context.Context, h hash.Hash) : (bool, error)

HasMany(ctx context.Context, hashes hash.HashSet) : (hash.HashSet, error)

Put(ctx context.Context, c chunks.Chunk) : error

Version() : string

Rebase(ctx context.Context) : error

refreshRepoMetadata(ctx context.Context) : error

Root(ctx context.Context) : (hash.Hash, error)

Commit(ctx context.Context, current, last hash.Hash) : (bool, error)

Stats() : interface{}

StatsSummary() : string

Close() : error

uploadChunks(ctx context.Context) : (map[hash.Hash]int, error)

uploadTableFileWithRetries(ctx context.Context, tableFileId hash.Hash, numChunks uint64, tableFileContentHash []byte, getContent func() (io.ReadCloser, uint64, error)) : error

httpPostUpload(ctx context.Context, post *remotesapi.HttpPostTableFile, contentHash []byte, contentLength int64, body io.ReadCloser) : error

downloadChunks(ctx context.Context, dlLocs dlLocations, chunkChan chan nbs.CompressedChunk) : error

SupportedOperations() : nbs.TableFileStoreOps

WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) : error

AddTableFilesToManifest(ctx context.Context, fileIdToNumChunks map[string]int) : error

PruneTableFiles(ctx context.Context) : error

Sources(ctx context.Context) : (hash.Hash, []nbs.TableFile, []nbs.TableFile, error)

Size(ctx context.Context) : (uint64, error)

SetRootChunk(ctx context.Context, root, previous hash.Hash) : error

*locationRefresh

Add(resp *remotesapi.DownloadLoc)

GetURL(ctx context.Context, lastError error, client remotesapi.ChunkStoreServiceClient) : (string, error)

Add(resp *remotesapi.DownloadLoc)

Please wait...

Page is in error, reload to recover