From eb1bfca671fbfa1de5cbbfc1b09482ece2e1cddc Mon Sep 17 00:00:00 2001 From: FabioBatSilva Date: Tue, 7 Aug 2018 16:06:30 -0400 Subject: [PATCH 1/3] Local setup using docker-compose --- .gitignore | 3 +- Makefile | 18 +++++++++- README.md | 5 +++ docker-compose-local.yml | 30 ++++++++++++++++ src/main/resources/application-local.yaml | 43 +++++++++++++++++++++++ 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 docker-compose-local.yml create mode 100644 src/main/resources/application-local.yaml diff --git a/.gitignore b/.gitignore index 3595f6b..888db3b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ nbdist/ .nb-gradle/ .env* -.vscode \ No newline at end of file +.vscode +.volumes/ \ No newline at end of file diff --git a/Makefile b/Makefile index 9974871..a6c3e1d 100644 --- a/Makefile +++ b/Makefile @@ -13,4 +13,20 @@ include $(MAKEFILE_DIR)/targets.mk ### Application specific targets can be added to this file ### -include targets.mk -### Or application sepcific targets can be added here ### \ No newline at end of file +### Or application sepcific targets can be added here ### + +start: + @echo ">>> Starting sftpd" + @docker-compose -f docker-compose-local.yml up sftpd + +stop: + @echo ">>> Stopping docker-compose" + @docker-compose -f docker-compose-local.yml stop + +logs: + @echo ">>> Starting application" + @docker-compose -f docker-compose-local.yml logs -f --tail="all" + +mysql-connect: + @echo ">>> Connecting to mysql" + @docker-compose -f docker-compose-local.yml exec mysql mysql -uroot -proot sftpd \ No newline at end of file diff --git a/README.md b/README.md index 7859eee..75eca7f 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,11 @@ Running the app AWS_REGION="ca-central-1" AWS_ACCESS_KEY_ID="ACCESSKEYHERE" AWS_SECRET_ACCESS_KEY="SECRETKEYHERE" java -jar target/s3-sftp-1.0.1-SNAPSHOT.jar ``` +### Running the app localy +``` +make start AWS_ACCESS_KEY_ID=ACCESSKEYHERE AWS_SECRET_ACCESS_KEY=SECRETKEYHERE +``` + ### How to configure your app Example configuration: diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 0000000..60da8be --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,30 @@ +version: '2.1' + +services: + mysql: + image: mysql:5.7 + restart: always + environment: + MYSQL_DATABASE: sftpd + MYSQL_ROOT_PASSWORD: root + healthcheck: + test: mysql -uroot -proot -e 'SHOW TABLES' sftpd + + sftpd: + image: maven:3-jdk-8-alpine + working_dir: /usr/share/statflo + command: mvn clean spring-boot:run + ports: + - "2222:2222" + environment: + - AWS_REGION=ca-central-1 + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID- } + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY- } + + - SPRING_PROFILES_ACTIVE=default,local + volumes: + - "./:/usr/share/statflo" + - "./.volumes/m2:/root/.m2" + depends_on: + mysql: + condition: service_healthy \ No newline at end of file diff --git a/src/main/resources/application-local.yaml b/src/main/resources/application-local.yaml new file mode 100644 index 0000000..0f36813 --- /dev/null +++ b/src/main/resources/application-local.yaml @@ -0,0 +1,43 @@ +app: + sftp: + users: + - + username: test + password: test + aws: + bucket-name: "test-sftp-uploader" + hostkey-algorithm: "RSA" + hostkey-private: | + -----BEGIN RSA PRIVATE KEY----- + MIIEowIBAAKCAQEArXpNaJAOXsgNe8OIFzWrKMC0D5yE3GV8TPjdAWy8n7MjOole + DbEH5DzO1fKlLMCQGITjZuouSY3yE8zMC9xs4hE5e+KTSStreK1E9X3csSszuOLr + +olj/C3ZcMKNq2s0TPBu691cRnTwVaL1V75V32rfCHz8NShCxyYXNiahycrOL2Bm + 9Cu8AMZcm1e9POgqABZbP2cP5MDy+Vw4RMw8XI8G6ClCQBlJayNAhqMDDw+80A/Z + MBt3OyDOJ7o2xq/sJAIhppck+Fw43W04BIQwIrmEucBs1c6Np/+ePvk5jKhII5iE + 2+7nQizwwUJQmzdmTcciLJ8CMhZ7FDUobXFHwQIDAQABAoIBAQCtLbq495Zg6DUN + qFTjVqBXPoiCTS4VFvrPnGCEW5ixlJNGjHtoD6JhJpMU4fYihtlSoNailoU+yIUx + XvETpijMRYWtNrLyCHZxMgf1SoBWHVq051PL9/voc5zuBEqJJaxAYoVY0NKdQ3/o + /ZUO0uPG9VlRj/P23QpM5KmnVUmNt1L6nml9v9J/yeO/GlMtv5FEwJm11uHeeWGn + Qfh/4B6hyDXsCOCAhD5C7G4hWChzs6q/IRI+swiocSf5S5u6N2h1HdZz3W0Q92lQ + BAFypWjCPDlMHzRgbtMUKQ2n8bv78svEe7kAgZvEX8dqnE0EJGLlJP2fo1nv6BQt + VmWMTMsBAoGBANQHndt+gXx75amGfTYyIK51dXahagfMiz4JGPBrzrZ4zOW10opU + zvvB0p2ApaKr8+jZtRLxHZToof/8Cck2RUodwS00he/Ikbz7GdwqDoiyWbjVTaut + +HJNn3zUxHSMDTHxU+DOVrIb79vqUweAYlq2ManPRzi7yDVkZv3ssr0FAoGBANF0 + AoNV8w8gGDfV8O6P/wTSXmkRfvk8WkTOkOoOHCTHXt4QuTZ5Q6v7ad0BlSj/A+sN + qScpn0GQsDNdgqSCoXdXDgKWjUqKvcwN+SLq5f0uPeGFkL3GuGraK+jAaxjXml4q + eb3MoSzMmlHXO7LorbNXcYXen180j34jFgcOijyNAoGAcXoeFzIulPQuZzuOWeoc + KaK9mh8rgz8KqsntqgbY3Dns12gnZVeca4tBcIBApb5HzTBvQ9L91QtoYHuAtUqx + bMhd8haxNDLxJOZy4a00mB5z/INUjXSPioVWvVWWSxYFndwd2ywzpCFljxprE5G3 + kfr+u3YvFQMTVNgGcxIyWw0CgYBps/Q4t72MWpQeSZfnrT/z/P6sY6oCuhSMOPsl + Q62sWflkdGgkN5qRRrYsVMokAF7H1wMQwo6elhh2M7DkuRAnnAV0BOqVPeN/AWZD + 2uvqk9Wi5LchLxKOyj/ynHjv9o7vCtyZjpWgeDlBKC/2oDpZEb424xzm/S2VaLuL + IClfWQKBgF/aQ0Jxhg1LCTK/gZmqyxXyvxu7x2Utedxt7hI6ia3eSvfXowgcOXMT + mfixttpPUobQhk43y5/h3N7gFGKxZQV6/ZFeBG7Dvck2NWgutNwqvbhFtfhVf37E + PP+E8udcWsXfU2ZjPAIrApFNIwIYtEOsisy2QTZkN8BaTKcHq6+S + -----END RSA PRIVATE KEY----- + +spring: + datasource: + url: "jdbc:mysql://mysql/sftpd?useSSL=false&serverTimezone=UTC" + username: "root" + password: "root" From 80a0cc41f88d96b070f59261c9305db1754315fb Mon Sep 17 00:00:00 2001 From: FabioBatSilva Date: Thu, 9 Aug 2018 11:28:03 -0400 Subject: [PATCH 2/3] Use statflo version of s3fs --- pom.xml | 15 ++++++++++++++- .../server/aws/CustomAmazonS3Client.java | 15 +++++++++++++++ .../com/dataexchange/server/conf/SftpServer.java | 13 ++++++++----- .../sshd/file/UserRootedFileSystemFactory.java | 4 ++-- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index a0b71a3..ce5abb3 100644 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,13 @@ 1.6.0 + + + jitpack.io + https://jitpack.io + + + @@ -61,12 +68,18 @@ commons-codec commons-codec - + + + com.github.Statflo + Amazon-S3-FileSystem-NIO2 + develop-SNAPSHOT + com.amazonaws diff --git a/src/main/java/com/dataexchange/server/aws/CustomAmazonS3Client.java b/src/main/java/com/dataexchange/server/aws/CustomAmazonS3Client.java index 6c6f29c..8d6118e 100644 --- a/src/main/java/com/dataexchange/server/aws/CustomAmazonS3Client.java +++ b/src/main/java/com/dataexchange/server/aws/CustomAmazonS3Client.java @@ -1,5 +1,6 @@ package com.dataexchange.server.aws; +import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentialsProvider; @@ -39,4 +40,18 @@ public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) throws S return super.copyObject(copyObjectRequest); } + + @Override + public InitiateMultipartUploadResult initiateMultipartUpload(final InitiateMultipartUploadRequest request) throws SdkClientException, AmazonServiceException { + ObjectMetadata metadata = request.getObjectMetadata(); + + if (metadata == null) { + metadata = new ObjectMetadata(); + } + + metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); + request.setObjectMetadata(metadata); + + return super.initiateMultipartUpload(request); + } } diff --git a/src/main/java/com/dataexchange/server/conf/SftpServer.java b/src/main/java/com/dataexchange/server/conf/SftpServer.java index 271726b..4ce2555 100644 --- a/src/main/java/com/dataexchange/server/conf/SftpServer.java +++ b/src/main/java/com/dataexchange/server/conf/SftpServer.java @@ -47,6 +47,11 @@ @EnableConfigurationProperties(SftpServer.SftpServerConfiguration.class) public class SftpServer { + public static final Map ENV = ImmutableMap.builder() + .put(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS, CustomAmazonS3ClientFactory.class.getName()) + .put(S3FileSystemProvider.MULTIPART_UPLOAD_ENABLED, "true") + .build(); + @Value("${app.sftp.aws.bucket-name}") private String bucketName; @@ -106,10 +111,8 @@ public SshServer sshServer(PasswordAuthenticator passwordAuthenticator, Tracking private Path getS3BucketPath() throws URISyntaxException, IOException { - Map env = ImmutableMap.builder() - .put(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS, CustomAmazonS3ClientFactory.class.getName()).build(); - - FileSystem fileSystem = FileSystems.newFileSystem(new URI("s3:///"), env, Thread.currentThread().getContextClassLoader()); + final URI uri = new URI("s3:///"); + final FileSystem fileSystem = FileSystems.newFileSystem(uri, ENV, Thread.currentThread().getContextClassLoader()); return fileSystem.getPath("/" + bucketName); } @@ -198,6 +201,6 @@ public String getPassword() { public void setPassword(String password) { this.password = password; } - + } } diff --git a/src/main/java/com/dataexchange/server/sshd/file/UserRootedFileSystemFactory.java b/src/main/java/com/dataexchange/server/sshd/file/UserRootedFileSystemFactory.java index 97cb969..bfddec8 100644 --- a/src/main/java/com/dataexchange/server/sshd/file/UserRootedFileSystemFactory.java +++ b/src/main/java/com/dataexchange/server/sshd/file/UserRootedFileSystemFactory.java @@ -1,5 +1,6 @@ package com.dataexchange.server.sshd.file; +import com.dataexchange.server.conf.SftpServer; import org.apache.sshd.common.file.root.RootedFileSystemProvider; import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory; import org.apache.sshd.common.session.Session; @@ -9,7 +10,6 @@ import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; -import java.util.Collections; public class UserRootedFileSystemFactory extends VirtualFileSystemFactory { @@ -25,7 +25,7 @@ public FileSystem createFileSystem(Session session) throws IOException { throw new InvalidPathException(username, "Cannot resolve home directory"); } - return new RootedFileSystemProvider().newFileSystem(dir, Collections.emptyMap()); + return new RootedFileSystemProvider().newFileSystem(dir, SftpServer.ENV); } @Override From 2601467a64789fb3fe7e481f484c4db9f0cf8a5c Mon Sep 17 00:00:00 2001 From: FabioBatSilva Date: Wed, 15 Aug 2018 10:26:00 -0400 Subject: [PATCH 3/3] Point to snapshot --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ce5abb3..b6cf49c 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ com.github.Statflo Amazon-S3-FileSystem-NIO2 - develop-SNAPSHOT + master-SNAPSHOT