Generate forward-compatible Java client

.github/workflows/java-api-client.yaml

@@ -9,6 +9,48 @@ on:
         description: Tag/version to publish
 
 jobs:
+  deploy-legacy:                # TODO(1.0): Remove this job after releasing 1.0
+    runs-on: ubuntu-20.04
+    environment: Treeverse signing
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+        # Extract the version to 'version' based on workflow_dispatch or triggered tag in the published event
+      - name: Extract version
+        shell: bash
+        run: |
+          if [ "${{ github.event.inputs.tag }}" != "" ]; then
+            echo "tag=$(echo ${{ github.event.inputs.tag }} | sed s/^v//)" >> $GITHUB_OUTPUT
+          else
+            echo "tag=$(echo ${GITHUB_REF##*/} | sed s/^v//)" >> $GITHUB_OUTPUT
+          fi
+        id: version
+
+      - name: Java generate package
+        run: make client-java-legacy PACKAGE_VERSION=${{ steps.version.outputs.tag }}
+
+      - name: Install secret key for signing
+        run: |
+          cat <(echo -e '${{ secrets.OSSRH_GPG_SECRET_KEY }}') | gpg --batch --import
+          gpg --list-secret-keys --keyid-format LONG
+
+      - name: Set up Java and Maven Central Repository
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+          server-id: ossrh
+          server-username: MAVEN_USERNAME
+          server-password: MAVEN_PASSWORD
+
+      - name: Build and publish package
+        working-directory: clients/java-legacy
+        run: mvn --batch-mode deploy -Dgpg.executable="${GITHUB_WORKSPACE}/scripts/gpg_loopback.sh" --activate-profiles sign-artifacts
+        env:
+          MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
+          MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}
+
   deploy:
     runs-on: ubuntu-20.04
     environment: Treeverse signing

Makefile

@@ -12,7 +12,9 @@ PROTOC=$(DOCKER) run --rm -v $(shell pwd):/mnt $(PROTOC_IMAGE)
 CLIENT_JARS_BUCKET="s3://treeverse-clients-us-east/"
 
 # https://openapi-generator.tech
-OPENAPI_GENERATOR_IMAGE=openapitools/openapi-generator-cli:v5.3.0
+OPENAPI_LEGACY_GENERATOR_IMAGE=openapitools/openapi-generator-cli:v5.3.0
+OPENAPI_LEGACY_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_LEGACY_GENERATOR_IMAGE)
+OPENAPI_GENERATOR_IMAGE=openapitools/openapi-generator-cli:v7.0.0
 OPENAPI_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE)
 
 GOLANGCI_LINT_VERSION=v1.53.3
@@ -113,7 +115,7 @@ client-python: api/swagger.yml  ## Generate SDK for Python client
 	# remove the build folder as it also holds lakefs_client folder which keeps because we skip it during find
 	rm -rf clients/python/build; cd clients/python && \
 		find . -depth -name lakefs_client -prune -o ! \( -name Gemfile -or -name Gemfile.lock -or -name _config.yml -or -name .openapi-generator-ignore -or -name templates -or -name setup.mustache -or -name client.mustache \) -delete
-	$(OPENAPI_GENERATOR) generate \
+	$(OPENAPI_LEGACY_GENERATOR) generate \
 		-i /mnt/$< \
 		-g python \
 		-t /mnt/clients/python/templates \
@@ -124,18 +126,32 @@ client-python: api/swagger.yml  ## Generate SDK for Python client
 		-c /mnt/clients/python-codegen-config.yaml \
 		-o /mnt/clients/python
 
-client-java: api/swagger.yml  ## Generate SDK for Java (and Scala) client
-	rm -rf clients/java
-	$(OPENAPI_GENERATOR) generate \
+client-java-legacy: api/swagger.yml api/java-gen-ignore  ## Generate legacy SDK for Java (and Scala) client
+	rm -rf clients/java-legacy
+	$(OPENAPI_LEGACY_GENERATOR) generate \
 		-i /mnt/$< \
+		--ignore-file-override /mnt/api/java-gen-ignore \
 		-g java \
 		--invoker-package io.lakefs.clients.api \
-		--http-user-agent "lakefs-java-sdk/$(PACKAGE_VERSION)" \
-		--additional-properties=hideGenerationTimestamp=true,artifactVersion=$(PACKAGE_VERSION),parentArtifactId=lakefs-parent,parentGroupId=io.lakefs,parentVersion=0,groupId=io.lakefs,artifactId='api-client',artifactDescription='lakeFS OpenAPI Java client',artifactUrl=https://lakefs.io,apiPackage=io.lakefs.clients.api,modelPackage=io.lakefs.clients.api.model,mainPackage=io.lakefs.clients.api,developerEmail=services@treeverse.io,developerName='Treeverse lakeFS dev',developerOrganization='lakefs.io',developerOrganizationUrl='https://lakefs.io',licenseName=apache2,licenseUrl=http://www.apache.org/licenses/,scmConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmDeveloperConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmUrl=https://github.com/treeverse/lakeFS \
+		--http-user-agent "lakefs-java-sdk/$(PACKAGE_VERSION)-legacy" \
+		--additional-properties hideGenerationTimestamp=true,artifactVersion=$(PACKAGE_VERSION),parentArtifactId=lakefs-parent,parentGroupId=io.lakefs,parentVersion=0,groupId=io.lakefs,artifactId='api-client',artifactDescription='lakeFS OpenAPI Java client legacy SDK',artifactUrl=https://lakefs.io,apiPackage=io.lakefs.clients.api,modelPackage=io.lakefs.clients.api.model,mainPackage=io.lakefs.clients.api,developerEmail=services@treeverse.io,developerName='Treeverse lakeFS dev',developerOrganization='lakefs.io',developerOrganizationUrl='https://lakefs.io',licenseName=apache2,licenseUrl=http://www.apache.org/licenses/,scmConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmDeveloperConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmUrl=https://github.com/treeverse/lakeFS \
+		-o /mnt/clients/java-legacy
+
+client-java: api/swagger.yml api/java-gen-ignore  ## Generate SDK for Java (and Scala) client
+	rm -rf clients/java
+	mkdir -p clients/java
+	cp api/java-gen-ignore clients/java/.openapi-generator-ignore
+	$(OPENAPI_GENERATOR) generate \
+		-i /mnt/api/swagger.yml \
+		-g java \
+		--invoker-package io.lakefs.clients.sdk \
+		--http-user-agent "lakefs-java-sdk/$(PACKAGE_VERSION)-v1" \
+		--additional-properties disallowAdditionalPropertiesIfNotPresent=false,useSingleRequestParameter=true,hideGenerationTimestamp=true,artifactVersion=$(PACKAGE_VERSION),parentArtifactId=lakefs-parent,parentGroupId=io.lakefs,parentVersion=0,groupId=io.lakefs,artifactId='sdk-client',artifactDescription='lakeFS OpenAPI Java client',artifactUrl=https://lakefs.io,apiPackage=io.lakefs.clients.sdk,modelPackage=io.lakefs.clients.sdk.model,mainPackage=io.lakefs.clients.sdk,developerEmail=services@treeverse.io,developerName='Treeverse lakeFS dev',developerOrganization='lakefs.io',developerOrganizationUrl='https://lakefs.io',licenseName=apache2,licenseUrl=http://www.apache.org/licenses/,scmConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmDeveloperConnection=scm:git:git@github.com:treeverse/lakeFS.git,scmUrl=https://github.com/treeverse/lakeFS \
 		-o /mnt/clients/java
 
-.PHONY: clients client-python client-java
-clients: client-python client-java
+
+.PHONY: clients client-python client-java client-java-legacy
+clients: client-python client-java client-java-legacy
 
 package-python: client-python
 	$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt -e HOME=/tmp/ -w /mnt/clients/python $(PYTHON_IMAGE) /bin/bash -c \

api/java-gen-ignore

@@ -0,0 +1,3 @@
+# Files to ignore (skip) when generating Java client
+maven.yml
+.github/

clients/.gitattributes

@@ -1,5 +1,6 @@
-java/**		linguist-generated
-python/**	linguist-generated
-python/Gemfile linguist-generated=false
-python/Gemfile.lock linguist-generated=false
-python/_config.yml linguist-generated=false
+java/**			linguist-generated
+java-legacy/**		linguist-generated
+python/**		linguist-generated
+python/Gemfile		linguist-generated=false
+python/Gemfile.lock	linguist-generated=false
+python/_config.yml	linguist-generated=false

clients/hadoopfs/pom.xml

@@ -93,7 +93,7 @@ To export to S3:
                     <plugin>
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
-                        <version>2.12.4</version>
+                        <version>3.1.2</version>
                         <configuration>
                             <systemPropertyVariables>
                                 <lakefs.access_mode>presigned</lakefs.access_mode>
@@ -231,7 +231,7 @@ To export to S3:
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.12.4</version>
+                <version>3.1.2</version>
                 <configuration>
                     <excludes>
                         <exclude>${exclude.tests}</exclude>
@@ -368,11 +368,24 @@ To export to S3:
 	</dependency>
 
         <dependency>
-            <groupId>org.mockito</groupId>
-            <!-- "inline" currently required in order to mock constructor calls. -->
-            <artifactId>mockito-inline</artifactId>
-            <version>3.10.0</version>
-	    <scope>test</scope>
+            <groupId>org.mock-server</groupId>
+            <artifactId>mockserver-junit-rule-no-dependencies</artifactId>
+            <version>5.14.0</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.immutables</groupId>
+            <artifactId>value</artifactId>
+            <version>2.9.3</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>32.1.2-jre</version>
+            <scope>test</scope>
         </dependency>
     </dependencies>
 </project>

clients/hadoopfs/src/main/java/io/lakefs/Constants.java

@@ -10,6 +10,7 @@ public class Constants {
     public static final String ENDPOINT_KEY_SUFFIX = "endpoint";
     public static final String LIST_AMOUNT_KEY_SUFFIX = "list.amount";
     public static final String ACCESS_MODE_KEY_SUFFIX = "access.mode";
+    public static final String SESSION_ID = "session_id";
 
     public static enum AccessMode {
         SIMPLE,

clients/hadoopfs/src/main/java/io/lakefs/LakeFSClient.java

@@ -41,6 +41,11 @@ public LakeFSClient(String scheme, Configuration conf) throws IOException {
         basicAuth.setUsername(accessKey);
         basicAuth.setPassword(secretKey);
 
+        String sessionId = FSConfiguration.get(conf, scheme, Constants.SESSION_ID);
+        if (sessionId != null) {
+            apiClient.addDefaultCookie("sessionId", sessionId);
+        }
+
         this.objectsApi = new ObjectsApi(apiClient);
         this.stagingApi = new StagingApi(apiClient);
         this.repositoriesApi = new RepositoriesApi(apiClient);

clients/hadoopfs/src/main/java/io/lakefs/LakeFSFileSystem.java

@@ -421,6 +421,7 @@ private boolean renameFile(LakeFSFileStatus srcStatus, Path dst) throws IOExcept
             LOG.debug("renameFile: dst {} exists and is a {}", dst, dstFileStatus.isDirectory() ? "directory" : "file");
             if (dstFileStatus.isDirectory()) {
                 dst = buildObjPathOnExistingDestinationDir(srcStatus.getPath(), dst);
+                LOG.debug("renameFile: use {} to create dst {}", srcStatus.getPath(), dst);
             }
         } catch (FileNotFoundException e) {
             LOG.debug("renameFile: dst does not exist, renaming src {} to a file called dst {}",