troyready · troyready · May 3, 2025 · May 3, 2025
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Support for files larger than 5GB
+
 ### Fixed
 - Bumped aws sdk versions to fix tests
 

diff --git a/README.md b/README.md
@@ -62,6 +62,19 @@ git lfs install
 
 That's it. On push/pull, you'll be prompted for Cognito credentials.
 
+### Handling Files >= 5GB
+
+The backend S3 storage service won't accept files larger than 5GB using Git LFS's normal basic transfer agent. When attempting to upload them, git-lfs-s3 will reject requests that don't claim support for its custom `multipart3upload` [transfer adapter](https://github.com/git-lfs/git-lfs/blob/main/docs/custom-transfers.md).
+
+A python script implementing this is [located here](./git-lfs-multiparts3upload). Download it, place it in your $PATH (e.g. `/usr/local/bin`), ensure it's executable, and configure your repo to use it:
+
+```bash
+git config --add lfs.customtransfer.multipart3upload.path git-lfs-multiparts3upload
+git config --add lfs.customtransfer.multipart3upload.direction upload
+```
+
+(or set the `--global` option to save the options in your user .gitconfig for use with all repositories and allow easy `git clone`ing of repos)
+
 ## Further Customization Ideas
 
 * Add an API Gateway custom domain to the API to get a better URL

diff --git a/git-lfs-multiparts3upload b/git-lfs-multiparts3upload
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+Git LFS multipart3upload transfer agent
+"""
+
+import json
+import sys
+import tempfile
+from typing import Dict, Any, Optional
+import urllib.error
+import urllib.request
+
+def write_error_log(message:str, exit=True) -> None:
+    with tempfile.NamedTemporaryFile(delete=False, prefix='git-lfs-multipart3upload-', suffix='.log') as tmp_file:
+        tmp_file.write(message.encode('utf-8'))
+        if exit:
+            sys.exit(1)
+
+class MultipartS3UploadTransferAgent:
+    def read_line(self) -> Optional[str]:
+        """Read a line from stdin."""
+        try:
+            return sys.stdin.readline().strip()
+        except Exception as e:
+            write_error_log(f"Error reading from stdin: {e}")
+
+    def write_line(self, data: Dict[str, Any]) -> None:
+        """Write a line to stdout."""
+        try:
+            sys.stdout.write(json.dumps(data) + "\n")
+            sys.stdout.flush()
+        except Exception as e:
+            write_error_log(f"Error writing to stdout: {e}")
+
+    def send_progress(self, oid: str, bytes_so_far: int, bytes_since_last: int) -> None:
+        """Send a progress update."""
+        self.write_line({
+            "event": "progress",
+            "oid": oid,
+            "bytesSoFar": bytes_so_far,
+            "bytesSinceLast": bytes_since_last,
+        })
+
+    def put_request(self, url: str, data: bytes) -> Dict[str, Any]:
+        """Make a PUT request with the given data."""
+        try:
+            request = urllib.request.Request(
+                url,
+                data=data,
+                method='PUT'
+            )
+            with urllib.request.urlopen(request) as response:
+                if response.status not in (200, 201):
+                    raise urllib.error.HTTPError(
+                        url, response.status,
+                        f"PUT request failed with status {response.status}",
+                        response.headers, None
+                    )
+                return response.headers
+        except urllib.error.URLError as e:
+            write_error_log(f"Failed to make PUT request: {e}")
+            sys.exit(1)
+
+    def handle_upload(self, transfer_request: Dict[str, Any]) -> None:
+        """Handle an upload transfer request."""
+        try:
+            oid = transfer_request["oid"]
+            size = transfer_request["size"]
+            multipart_s3_upload_href = json.loads(transfer_request["action"]["href"])
+            presigned_urls = multipart_s3_upload_href["presignedurls"]
+
+            part_number = 1
+            total_uploaded = 0
+            chunk_size = size // len(presigned_urls)
+            completed_parts = []
+            if size % len(presigned_urls) != 0:
+                chunk_size = min(size // (len(presigned_urls) - 1), 5_000_000_000) # s3 max upload chunk of 5GB
+
+            with open(transfer_request["path"], 'rb') as f:
+                for presigned_url in presigned_urls:
+                    if part_number == len(presigned_urls):
+                        chunk = f.read() # For the last part, read the remaining bytes
+                    else:
+                        chunk = f.read(chunk_size)
+
+                    part_upload_response_headers = self.put_request(presigned_url, chunk)
+                    completed_parts.append({'ETag': part_upload_response_headers['ETag'], 'PartNumber': part_number})
+                    total_uploaded += len(chunk)
+                    self.send_progress(oid, total_uploaded, len(chunk))
+                    part_number += 1
+
            self.put_request(
+                multipart_s3_upload_href["completionurl"],
+                json.dumps({
+                  "MultipartUpload": {"Parts": completed_parts},
+                  "UploadId": multipart_s3_upload_href["uploadid"],
+                }).encode('utf-8'))
+
+            self.write_line({
+                "event": "complete",
+                "oid": oid
+            })
+
+        except Exception as e:
+            write_error_log(f"Error during upload: {e}", exit=False)
+            self.write_line({
+                "event": "error",
+                "oid": transfer_request["oid"],
+                "error": {
+                    "code": 1,
+                    "message": str(e)
+                }
+            })
+
+    def run(self):
+        """Main event loop."""
+        try:
+            init_line = self.read_line()
+            if not init_line:
+                return
+
+            init_data = json.loads(init_line)
+            if init_data["event"] != "init":
+                raise ValueError("Expected init event")
+            if init_data["operation"] != "upload":
+                self.write_line({
+                    "event": "error",
+                    "oid": init_data["oid"],
+                    "error": {
+                        "code": 32,
+                        "message": "Unsupported operation"
+                    }
+                })
+                sys.exit(1)
+
+            self.write_line({}) # Confirm initialization
+
+            while True:
+                line = self.read_line()
+                if not line:
+                    break
+
+                transfer_request = json.loads(line)
+
+                if transfer_request["event"] == "terminate":
+                    break
+                elif transfer_request["event"] == "upload":
+                    self.handle_upload(transfer_request)
+
+        except Exception as e:
+            write_error_log(f"Fatal error: {e}")
+
+if __name__ == "__main__":
+    agent = MultipartS3UploadTransferAgent()
+    agent.run()
diff --git a/serverless.yml b/serverless.yml
@@ -53,6 +53,7 @@ functions:
             resultTtlInSeconds: 0
             identitySource: method.request.header.Authorization
             type: request
+    memorySize: 512
   locks:
     handler: src/locks/locks.handler
     iamRoleStatements:
@@ -89,14 +90,48 @@ functions:
           authorizer: *authorizer
       - http:
           path: locks
-          method: get  # list locks
+          method: get # list locks
           integration: lambda-proxy
           authorizer: *authorizer
       - http:
           path: locks
-          method: post  # create lock
+          method: post # create lock
           integration: lambda-proxy
           authorizer: *authorizer
+  completemultipartupload:
+    handler: src/completemultipartupload/handler.handler
+    iamRoleStatements:
+      - Action:
+          - s3:PutObject
+        Effect: "Allow"
+        Resource:
+          Fn::Join:
+            - ""
+            - - Fn::GetAtt:
+                  - StorageBucket
+                  - Arn
+              - "/*"
+      - Action:
+          - s3:DeleteObject
+          - s3:GetObject
+        Effect: "Allow"
+        Resource:
+          Fn::Join:
+            - ""
+            - - Fn::GetAtt:
+                  - StorageBucket
+                  - Arn
+              - "/*.multipartuploadcomplete"
+    events:
+      - s3:
+          bucket:
+            Ref: StorageBucket
+          event: s3:ObjectCreated:Put
+          existing: true
+          rules:
+            - suffix: .multipartuploadcomplete
+    memorySize: 512
+    timeout: 900 # complete multipart upload can take minutes
   authorizer:
     handler: src/authorizer/authorizer.handler
     environment:
@@ -128,10 +163,17 @@ resources:
         ResponseType: UNAUTHORIZED
         RestApiId:
           Ref: ApiGatewayRestApi
-        StatusCode: '401'
+        StatusCode: "401"
     StorageBucket:
       Type: AWS::S3::Bucket
       DeletionPolicy: Retain
+      Properties:
+        LifecycleConfiguration:
+          Rules:
+            - AbortIncompleteMultipartUpload:
+                DaysAfterInitiation: 1
+              Id: DeleteIncompleteMultipartUploads
+              Status: Enabled
     LockTable:
       Type: AWS::DynamoDB::Table
       DeletionPolicy: Retain
@@ -154,20 +196,20 @@ resources:
             KeyType: HASH
     UserPool:
       Type: AWS::Cognito::UserPool
-      Properties: 
+      Properties:
         AdminCreateUserConfig:
           AllowAdminCreateUserOnly: true
         UserPoolName: "${self:service}-${opt:stage}"
         UserPoolTags:
           STAGE: "${opt:stage}"
     UserPoolClient:
       Type: AWS::Cognito::UserPoolClient
-      Properties: 
+      Properties:
         ClientName: "${self:service}-${opt:stage}"
-        ExplicitAuthFlows: 
+        ExplicitAuthFlows:
           - ADMIN_NO_SRP_AUTH
         GenerateSecret: false
-        SupportedIdentityProviders: 
+        SupportedIdentityProviders:
           - COGNITO
         UserPoolId:
           Ref: UserPool