Multipart upload

.gitignore

@@ -24,4 +24,5 @@ nbdist/
 .nb-gradle/
 
 .env*
-.vscode
+.vscode
+.volumes/

Makefile

@@ -13,4 +13,20 @@ include $(MAKEFILE_DIR)/targets.mk
 ### Application specific targets can be added to this file ###
 -include targets.mk
 
-### Or application sepcific targets can be added here ###
+### Or application sepcific targets can be added here ###
+
+start:
+	@echo ">>> Starting sftpd"
+	@docker-compose -f docker-compose-local.yml up sftpd
+
+stop:
+	@echo ">>> Stopping docker-compose"
+	@docker-compose -f docker-compose-local.yml stop
+
+logs:
+	@echo ">>> Starting application"
+	@docker-compose -f docker-compose-local.yml logs -f --tail="all"
+
+mysql-connect:
+	@echo ">>> Connecting to mysql"
+	@docker-compose -f docker-compose-local.yml exec mysql mysql -uroot -proot sftpd

README.md

@@ -34,6 +34,11 @@ Running the app
 AWS_REGION="ca-central-1" AWS_ACCESS_KEY_ID="ACCESSKEYHERE" AWS_SECRET_ACCESS_KEY="SECRETKEYHERE" java -jar target/s3-sftp-1.0.1-SNAPSHOT.jar
 ```
 
+### Running the app localy
+```
+make start AWS_ACCESS_KEY_ID=ACCESSKEYHERE AWS_SECRET_ACCESS_KEY=SECRETKEYHERE
+```
+
 ### How to configure your app
 
 Example configuration:

docker-compose-local.yml

@@ -0,0 +1,30 @@
+version: '2.1'
+
+services:
+    mysql:
+        image: mysql:5.7
+        restart: always
+        environment:
+            MYSQL_DATABASE: sftpd
+            MYSQL_ROOT_PASSWORD: root
+        healthcheck:
+            test:  mysql -uroot -proot -e 'SHOW TABLES' sftpd
+
+    sftpd:
+        image: maven:3-jdk-8-alpine
+        working_dir: /usr/share/statflo
+        command: mvn clean spring-boot:run
+        ports:
+            - "2222:2222"
+        environment:
+            - AWS_REGION=ca-central-1
+            - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID- }
+            - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY- }
+
+            - SPRING_PROFILES_ACTIVE=default,local
+        volumes:
+            - "./:/usr/share/statflo"
+            - "./.volumes/m2:/root/.m2"
+        depends_on:
+            mysql:
+                condition: service_healthy

pom.xml

@@ -25,6 +25,13 @@
         <org.apache.sshd.version>1.6.0</org.apache.sshd.version>
 	</properties>
 
+	<repositories>
+		<repository>
+			<id>jitpack.io</id>
+			<url>https://jitpack.io</url>
+		</repository>
+	</repositories>
+
 	<dependencies>
 
 		<dependency>
@@ -61,12 +68,18 @@
 			<groupId>commons-codec</groupId>
 			<artifactId>commons-codec</artifactId>
 		</dependency>
-		
+<!--
 		<dependency>
 			<groupId>com.upplication</groupId>
 			<artifactId>s3fs</artifactId>
 			<version>2.2.2</version>
 		</dependency>
+-->
+		<dependency>
+			<groupId>com.github.Statflo</groupId>
+			<artifactId>Amazon-S3-FileSystem-NIO2</artifactId>
+			<version>master-SNAPSHOT</version>
+		</dependency>
 
 		<dependency>
 			<groupId>com.amazonaws</groupId>

src/main/java/com/dataexchange/server/aws/CustomAmazonS3Client.java

@@ -1,5 +1,6 @@
 package com.dataexchange.server.aws;
 
+import com.amazonaws.AmazonServiceException;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.SdkClientException;
 import com.amazonaws.auth.AWSCredentialsProvider;
@@ -39,4 +40,18 @@ public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) throws S
 
         return super.copyObject(copyObjectRequest);
     }
+
+    @Override
+    public InitiateMultipartUploadResult initiateMultipartUpload(final InitiateMultipartUploadRequest request) throws SdkClientException, AmazonServiceException {
+        ObjectMetadata metadata = request.getObjectMetadata();
+
+        if (metadata == null) {
+            metadata = new ObjectMetadata();
+        }
+
+        metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+        request.setObjectMetadata(metadata);
+
+        return super.initiateMultipartUpload(request);
+    }
 }

src/main/java/com/dataexchange/server/conf/SftpServer.java

@@ -47,6 +47,11 @@
 @EnableConfigurationProperties(SftpServer.SftpServerConfiguration.class)
 public class SftpServer {
 
+    public static final Map<String, ?> ENV = ImmutableMap.<String, Object>builder()
+        .put(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS, CustomAmazonS3ClientFactory.class.getName())
+        .put(S3FileSystemProvider.MULTIPART_UPLOAD_ENABLED, "true")
+        .build();
+
     @Value("${app.sftp.aws.bucket-name}")
     private String bucketName;
 
@@ -106,10 +111,8 @@ public SshServer sshServer(PasswordAuthenticator passwordAuthenticator, Tracking
 
 
     private Path getS3BucketPath() throws URISyntaxException, IOException {
-        Map<String, ?> env = ImmutableMap.<String, Object>builder()
-                .put(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS, CustomAmazonS3ClientFactory.class.getName()).build();
-
-        FileSystem fileSystem = FileSystems.newFileSystem(new URI("s3:///"), env, Thread.currentThread().getContextClassLoader());
+        final URI uri = new URI("s3:///");
+        final FileSystem fileSystem = FileSystems.newFileSystem(uri, ENV, Thread.currentThread().getContextClassLoader());
 
         return fileSystem.getPath("/" + bucketName);
     }
@@ -198,6 +201,6 @@ public String getPassword() {
         public void setPassword(String password) {
             this.password = password;
         }
-        
+
     }
 }

src/main/java/com/dataexchange/server/sshd/file/UserRootedFileSystemFactory.java

@@ -1,5 +1,6 @@
 package com.dataexchange.server.sshd.file;
 
+import com.dataexchange.server.conf.SftpServer;
 import org.apache.sshd.common.file.root.RootedFileSystemProvider;
 import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
 import org.apache.sshd.common.session.Session;
@@ -9,7 +10,6 @@
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
 import java.nio.file.Path;
-import java.util.Collections;
 
 public class UserRootedFileSystemFactory extends VirtualFileSystemFactory {
 
@@ -25,7 +25,7 @@ public FileSystem createFileSystem(Session session) throws IOException {
             throw new InvalidPathException(username, "Cannot resolve home directory");
         }
 
-        return new RootedFileSystemProvider().newFileSystem(dir, Collections.emptyMap());
+        return new RootedFileSystemProvider().newFileSystem(dir, SftpServer.ENV);
     }
 
     @Override

src/main/resources/application-local.yaml

@@ -0,0 +1,43 @@
+app:
+  sftp:
+    users:
+      -
+        username: test
+        password: test
+    aws:
+      bucket-name: "test-sftp-uploader"
+    hostkey-algorithm: "RSA"
+    hostkey-private: |
+      -----BEGIN RSA PRIVATE KEY-----
+      MIIEowIBAAKCAQEArXpNaJAOXsgNe8OIFzWrKMC0D5yE3GV8TPjdAWy8n7MjOole
+      DbEH5DzO1fKlLMCQGITjZuouSY3yE8zMC9xs4hE5e+KTSStreK1E9X3csSszuOLr
+      +olj/C3ZcMKNq2s0TPBu691cRnTwVaL1V75V32rfCHz8NShCxyYXNiahycrOL2Bm
+      9Cu8AMZcm1e9POgqABZbP2cP5MDy+Vw4RMw8XI8G6ClCQBlJayNAhqMDDw+80A/Z
+      MBt3OyDOJ7o2xq/sJAIhppck+Fw43W04BIQwIrmEucBs1c6Np/+ePvk5jKhII5iE
+      2+7nQizwwUJQmzdmTcciLJ8CMhZ7FDUobXFHwQIDAQABAoIBAQCtLbq495Zg6DUN
+      qFTjVqBXPoiCTS4VFvrPnGCEW5ixlJNGjHtoD6JhJpMU4fYihtlSoNailoU+yIUx
+      XvETpijMRYWtNrLyCHZxMgf1SoBWHVq051PL9/voc5zuBEqJJaxAYoVY0NKdQ3/o
+      /ZUO0uPG9VlRj/P23QpM5KmnVUmNt1L6nml9v9J/yeO/GlMtv5FEwJm11uHeeWGn
+      Qfh/4B6hyDXsCOCAhD5C7G4hWChzs6q/IRI+swiocSf5S5u6N2h1HdZz3W0Q92lQ
+      BAFypWjCPDlMHzRgbtMUKQ2n8bv78svEe7kAgZvEX8dqnE0EJGLlJP2fo1nv6BQt
+      VmWMTMsBAoGBANQHndt+gXx75amGfTYyIK51dXahagfMiz4JGPBrzrZ4zOW10opU
+      zvvB0p2ApaKr8+jZtRLxHZToof/8Cck2RUodwS00he/Ikbz7GdwqDoiyWbjVTaut
+      +HJNn3zUxHSMDTHxU+DOVrIb79vqUweAYlq2ManPRzi7yDVkZv3ssr0FAoGBANF0
+      AoNV8w8gGDfV8O6P/wTSXmkRfvk8WkTOkOoOHCTHXt4QuTZ5Q6v7ad0BlSj/A+sN
+      qScpn0GQsDNdgqSCoXdXDgKWjUqKvcwN+SLq5f0uPeGFkL3GuGraK+jAaxjXml4q
+      eb3MoSzMmlHXO7LorbNXcYXen180j34jFgcOijyNAoGAcXoeFzIulPQuZzuOWeoc
+      KaK9mh8rgz8KqsntqgbY3Dns12gnZVeca4tBcIBApb5HzTBvQ9L91QtoYHuAtUqx
+      bMhd8haxNDLxJOZy4a00mB5z/INUjXSPioVWvVWWSxYFndwd2ywzpCFljxprE5G3
+      kfr+u3YvFQMTVNgGcxIyWw0CgYBps/Q4t72MWpQeSZfnrT/z/P6sY6oCuhSMOPsl
+      Q62sWflkdGgkN5qRRrYsVMokAF7H1wMQwo6elhh2M7DkuRAnnAV0BOqVPeN/AWZD
+      2uvqk9Wi5LchLxKOyj/ynHjv9o7vCtyZjpWgeDlBKC/2oDpZEb424xzm/S2VaLuL
+      IClfWQKBgF/aQ0Jxhg1LCTK/gZmqyxXyvxu7x2Utedxt7hI6ia3eSvfXowgcOXMT
+      mfixttpPUobQhk43y5/h3N7gFGKxZQV6/ZFeBG7Dvck2NWgutNwqvbhFtfhVf37E
+      PP+E8udcWsXfU2ZjPAIrApFNIwIYtEOsisy2QTZkN8BaTKcHq6+S
+      -----END RSA PRIVATE KEY-----
+
+spring:
+  datasource:
+    url: "jdbc:mysql://mysql/sftpd?useSSL=false&serverTimezone=UTC"
+    username: "root"
+    password: "root"