From 03d836005484959402ecf81c56dfc9b14fa1addf Mon Sep 17 00:00:00 2001 From: Matsvei Maksimau Date: Mon, 20 Oct 2025 13:40:09 +0200 Subject: [PATCH 1/4] Add explicit buffers for AWS uploader and downloader --- storage/s3.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/storage/s3.go b/storage/s3.go index 3313be66e..410c5b6d0 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -50,6 +50,9 @@ const ( // the key of the object metadata which is used to handle retry decision on NoSuchUpload error metadataKeyRetryID = "s5cmd-upload-retry-id" + + DownloaderBufferSize = 64 * 1024 + UploaderBufferSize = 512 * 1024 ) // Re-used AWS sessions dramatically improve performance. @@ -102,10 +105,15 @@ func newS3Storage(ctx context.Context, opts Options) (*S3, error) { return nil, err } + downloader := s3manager.NewDownloader(awsSession) + downloader.BufferProvider = s3manager.NewPooledBufferedWriterReadFromProvider(DownloaderBufferSize) + uploader := s3manager.NewUploader(awsSession) + uploader.BufferProvider = s3manager.NewBufferedReadSeekerWriteToPool(UploaderBufferSize) + return &S3{ api: s3.New(awsSession), - downloader: s3manager.NewDownloader(awsSession), - uploader: s3manager.NewUploader(awsSession), + downloader: downloader, + uploader: uploader, endpointURL: endpointURL, dryRun: opts.DryRun, useListObjectsV1: opts.UseListObjectsV1, From 0e791a36c4ce9ad069888c8a21b562a46e2ce000 Mon Sep 17 00:00:00 2001 From: Matsvei Maksimau Date: Mon, 20 Oct 2025 13:49:45 +0200 Subject: [PATCH 2/4] fix docker image --- .github/workflows/ci.yml | 2 +- storage/mock_storage.go | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eceb7f72f..fa0e39f2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: runs-on: ${{ matrix.os }}-latest services: minio: - image: ${{ (matrix.os == 'ubuntu') && 'bitnami/minio:2023.7.18' || ''}} + image: ${{ (matrix.os == 'ubuntu') && 'quay.io/minio/minio' || ''}} ports: - 45677:9000 options: >- diff --git a/storage/mock_storage.go b/storage/mock_storage.go index cd76cf2b7..921beb306 100644 --- a/storage/mock_storage.go +++ b/storage/mock_storage.go @@ -1,10 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. // Source: storage.go -// -// Generated by this command: -// -// mockgen -source=storage.go -destination=mock_storage.go -package=storage Storage -// // Package storage is a generated GoMock package. package storage @@ -13,8 +8,8 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" url "github.com/peak/s5cmd/v2/storage/url" - gomock "go.uber.org/mock/gomock" ) // MockStorage is a mock of Storage interface. @@ -49,7 +44,7 @@ func (m *MockStorage) Copy(ctx context.Context, src, dst *url.URL, metadata Meta } // Copy indicates an expected call of Copy. -func (mr *MockStorageMockRecorder) Copy(ctx, src, dst, metadata any) *gomock.Call { +func (mr *MockStorageMockRecorder) Copy(ctx, src, dst, metadata interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockStorage)(nil).Copy), ctx, src, dst, metadata) } @@ -63,7 +58,7 @@ func (m *MockStorage) Delete(ctx context.Context, src *url.URL) error { } // Delete indicates an expected call of Delete. -func (mr *MockStorageMockRecorder) Delete(ctx, src any) *gomock.Call { +func (mr *MockStorageMockRecorder) Delete(ctx, src interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStorage)(nil).Delete), ctx, src) } @@ -77,7 +72,7 @@ func (m *MockStorage) List(ctx context.Context, src *url.URL, followSymlinks boo } // List indicates an expected call of List. -func (mr *MockStorageMockRecorder) List(ctx, src, followSymlinks any) *gomock.Call { +func (mr *MockStorageMockRecorder) List(ctx, src, followSymlinks interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockStorage)(nil).List), ctx, src, followSymlinks) } @@ -91,7 +86,7 @@ func (m *MockStorage) MultiDelete(ctx context.Context, urls <-chan *url.URL) <-c } // MultiDelete indicates an expected call of MultiDelete. -func (mr *MockStorageMockRecorder) MultiDelete(ctx, urls any) *gomock.Call { +func (mr *MockStorageMockRecorder) MultiDelete(ctx, urls interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MultiDelete", reflect.TypeOf((*MockStorage)(nil).MultiDelete), ctx, urls) } @@ -106,7 +101,7 @@ func (m *MockStorage) Stat(ctx context.Context, src *url.URL) (*Object, error) { } // Stat indicates an expected call of Stat. -func (mr *MockStorageMockRecorder) Stat(ctx, src any) *gomock.Call { +func (mr *MockStorageMockRecorder) Stat(ctx, src interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockStorage)(nil).Stat), ctx, src) } From 236a09f96320bab34dd07162145035ce235d6e3f Mon Sep 17 00:00:00 2001 From: Matsvei Maksimau Date: Mon, 20 Oct 2025 13:52:05 +0200 Subject: [PATCH 3/4] fix docker image --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa0e39f2c..6e619a28f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: runs-on: ${{ matrix.os }}-latest services: minio: - image: ${{ (matrix.os == 'ubuntu') && 'quay.io/minio/minio' || ''}} + image: ${{ (matrix.os == 'ubuntu') && 'quay.io/minio/minio:latest' || '' }} ports: - 45677:9000 options: >- @@ -56,6 +56,7 @@ jobs: env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin + command: server /data --console-address ":9001" steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 From b62167d56cb45b3277044a50b4aeeb323277e9a1 Mon Sep 17 00:00:00 2001 From: Matsvei Maksimau Date: Mon, 20 Oct 2025 13:59:50 +0200 Subject: [PATCH 4/4] fix mock --- storage/mock_storage.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/storage/mock_storage.go b/storage/mock_storage.go index 921beb306..f444808f2 100644 --- a/storage/mock_storage.go +++ b/storage/mock_storage.go @@ -1,5 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: storage.go +// +// Generated by this command: +// +// mockgen -source=storage.go -destination=mock_storage.go -package=storage Storage +// // Package storage is a generated GoMock package. package storage @@ -8,14 +13,15 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" url "github.com/peak/s5cmd/v2/storage/url" + gomock "go.uber.org/mock/gomock" ) // MockStorage is a mock of Storage interface. type MockStorage struct { ctrl *gomock.Controller recorder *MockStorageMockRecorder + isgomock struct{} } // MockStorageMockRecorder is the mock recorder for MockStorage. @@ -44,7 +50,7 @@ func (m *MockStorage) Copy(ctx context.Context, src, dst *url.URL, metadata Meta } // Copy indicates an expected call of Copy. -func (mr *MockStorageMockRecorder) Copy(ctx, src, dst, metadata interface{}) *gomock.Call { +func (mr *MockStorageMockRecorder) Copy(ctx, src, dst, metadata any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Copy", reflect.TypeOf((*MockStorage)(nil).Copy), ctx, src, dst, metadata) } @@ -58,7 +64,7 @@ func (m *MockStorage) Delete(ctx context.Context, src *url.URL) error { } // Delete indicates an expected call of Delete. -func (mr *MockStorageMockRecorder) Delete(ctx, src interface{}) *gomock.Call { +func (mr *MockStorageMockRecorder) Delete(ctx, src any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockStorage)(nil).Delete), ctx, src) } @@ -72,7 +78,7 @@ func (m *MockStorage) List(ctx context.Context, src *url.URL, followSymlinks boo } // List indicates an expected call of List. -func (mr *MockStorageMockRecorder) List(ctx, src, followSymlinks interface{}) *gomock.Call { +func (mr *MockStorageMockRecorder) List(ctx, src, followSymlinks any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockStorage)(nil).List), ctx, src, followSymlinks) } @@ -86,7 +92,7 @@ func (m *MockStorage) MultiDelete(ctx context.Context, urls <-chan *url.URL) <-c } // MultiDelete indicates an expected call of MultiDelete. -func (mr *MockStorageMockRecorder) MultiDelete(ctx, urls interface{}) *gomock.Call { +func (mr *MockStorageMockRecorder) MultiDelete(ctx, urls any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MultiDelete", reflect.TypeOf((*MockStorage)(nil).MultiDelete), ctx, urls) } @@ -101,7 +107,7 @@ func (m *MockStorage) Stat(ctx context.Context, src *url.URL) (*Object, error) { } // Stat indicates an expected call of Stat. -func (mr *MockStorageMockRecorder) Stat(ctx, src interface{}) *gomock.Call { +func (mr *MockStorageMockRecorder) Stat(ctx, src any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stat", reflect.TypeOf((*MockStorage)(nil).Stat), ctx, src) }