From 2bb9133eaa50b38badeac7aad83ee84eb5b7bbfc Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 23 Apr 2024 14:48:55 +0200 Subject: [PATCH 01/13] Extract base test class. --- .../AbstractMicrosoftIISDAVTest.java | 83 +++++++++++++++++++ .../MicrosoftIISDAVLockFeatureTest.java | 19 +---- .../MicrosoftIISDAVReadFeatureTest.java | 19 +---- .../MicrosoftIISDAVTimestampFeatureTest.java | 19 +---- 4 files changed, 86 insertions(+), 54 deletions(-) create mode 100644 webdav/src/test/java/ch/cyberduck/core/dav/microsoft/AbstractMicrosoftIISDAVTest.java diff --git a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/AbstractMicrosoftIISDAVTest.java b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/AbstractMicrosoftIISDAVTest.java new file mode 100644 index 00000000000..fae97383df4 --- /dev/null +++ b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/AbstractMicrosoftIISDAVTest.java @@ -0,0 +1,83 @@ +package ch.cyberduck.core.dav.microsoft; + +/* + * Copyright (c) 2002-2018 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.DisabledPasswordStore; +import ch.cyberduck.core.DisabledProgressListener; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.LoginConnectionService; +import ch.cyberduck.core.LoginOptions; +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.Scheme; +import ch.cyberduck.core.dav.DAVProtocol; +import ch.cyberduck.core.dav.DAVSession; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; +import ch.cyberduck.core.ssl.DefaultX509KeyManager; +import ch.cyberduck.core.ssl.DefaultX509TrustManager; +import ch.cyberduck.test.VaultTest; + +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; +import java.util.HashSet; + +import static org.junit.Assert.fail; + +public class AbstractMicrosoftIISDAVTest extends VaultTest { + + protected DAVSession session; + + @After + public void disconnect() throws Exception { + session.close(); + } + + @Before + public void setup() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new DAVProtocol()))); + final Profile profile = new ProfilePlistReader(factory).read( + this.getClass().getResourceAsStream("/DAV.cyberduckprofile")); + session = new DAVSession(new Host(profile, "winbuild.iterate.ch", profile.getDefaultPort(), "/WebDAV", new Credentials( + PROPERTIES.get("webdav.iis.user"), PROPERTIES.get("webdav.iis.password") + )), new DefaultX509TrustManager(), new DefaultX509KeyManager()); + final LoginConnectionService login = new LoginConnectionService(new DisabledLoginCallback() { + @Override + public Credentials prompt(final Host bookmark, final String title, final String reason, final LoginOptions options) { + fail(reason); + return null; + } + + @Override + public void warn(final Host bookmark, final String title, final String message, final String continueButton, final String disconnectButton, final String preference) { + // + } + }, new DisabledHostKeyCallback(), new TestPasswordStore(), new DisabledProgressListener()); + login.check(session, new DisabledCancelCallback()); + } + + public static class TestPasswordStore extends DisabledPasswordStore { + @Override + public String getPassword(Scheme scheme, int port, String hostname, String user) { + return PROPERTIES.get("webdav.iis.password"); + } + } +} diff --git a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java index 7e6696208aa..c09d5e9dc31 100644 --- a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java +++ b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java @@ -16,33 +16,23 @@ */ import ch.cyberduck.core.AlphanumericRandomStringService; -import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.DisabledConnectionCallback; -import ch.cyberduck.core.DisabledHostKeyCallback; import ch.cyberduck.core.DisabledListProgressListener; import ch.cyberduck.core.DisabledLoginCallback; -import ch.cyberduck.core.Host; import ch.cyberduck.core.Local; import ch.cyberduck.core.Path; import ch.cyberduck.core.PathAttributes; import ch.cyberduck.core.dav.DAVDeleteFeature; import ch.cyberduck.core.dav.DAVLockFeature; -import ch.cyberduck.core.dav.DAVProtocol; -import ch.cyberduck.core.dav.DAVSession; import ch.cyberduck.core.dav.DAVUploadFeature; import ch.cyberduck.core.dav.DAVWriteFeature; import ch.cyberduck.core.features.Delete; import ch.cyberduck.core.http.HttpUploadFeature; import ch.cyberduck.core.io.BandwidthThrottle; import ch.cyberduck.core.io.DisabledStreamListener; -import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.shared.DefaultHomeFinderService; -import ch.cyberduck.core.ssl.DefaultX509KeyManager; -import ch.cyberduck.core.ssl.DisabledX509TrustManager; import ch.cyberduck.core.transfer.TransferStatus; import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; import org.apache.commons.io.IOUtils; import org.junit.Test; @@ -57,17 +47,10 @@ import static org.junit.Assert.*; @Category(IntegrationTest.class) -public class MicrosoftIISDAVLockFeatureTest extends VaultTest { +public class MicrosoftIISDAVLockFeatureTest extends AbstractMicrosoftIISDAVTest { @Test public void testLock() throws Exception { - final Host host = new Host(new DAVProtocol(), "winbuild.iterate.ch", new Credentials( - PROPERTIES.get("webdav.iis.user"), PROPERTIES.get("webdav.iis.password") - )); - host.setDefaultPath("/WebDAV"); - final DAVSession session = new DAVSession(host, new DisabledX509TrustManager(), new DefaultX509KeyManager()); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); final TransferStatus status = new TransferStatus(); final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random()); final byte[] content = "test".getBytes(StandardCharsets.UTF_8); diff --git a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java index 1f918249b36..e0b8c80f063 100644 --- a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java +++ b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java @@ -16,29 +16,19 @@ */ import ch.cyberduck.core.AlphanumericRandomStringService; -import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.DisabledConnectionCallback; -import ch.cyberduck.core.DisabledHostKeyCallback; import ch.cyberduck.core.DisabledListProgressListener; import ch.cyberduck.core.DisabledLoginCallback; -import ch.cyberduck.core.Host; import ch.cyberduck.core.Local; import ch.cyberduck.core.Path; -import ch.cyberduck.core.dav.AbstractDAVTest; import ch.cyberduck.core.dav.DAVDeleteFeature; -import ch.cyberduck.core.dav.DAVProtocol; -import ch.cyberduck.core.dav.DAVSession; import ch.cyberduck.core.dav.DAVTouchFeature; import ch.cyberduck.core.dav.DAVUploadFeature; import ch.cyberduck.core.features.Delete; import ch.cyberduck.core.io.BandwidthThrottle; import ch.cyberduck.core.io.DisabledStreamListener; import ch.cyberduck.core.io.StreamCopier; -import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.shared.DefaultHomeFinderService; -import ch.cyberduck.core.ssl.DefaultX509KeyManager; -import ch.cyberduck.core.ssl.DisabledX509TrustManager; import ch.cyberduck.core.transfer.TransferStatus; import ch.cyberduck.test.IntegrationTest; @@ -56,17 +46,10 @@ import static org.junit.Assert.*; @Category(IntegrationTest.class) -public class MicrosoftIISDAVReadFeatureTest extends AbstractDAVTest { +public class MicrosoftIISDAVReadFeatureTest extends AbstractMicrosoftIISDAVTest { @Test public void testReadMicrosoft() throws Exception { - final Host host = new Host(new DAVProtocol(), "winbuild.iterate.ch", new Credentials( - PROPERTIES.get("webdav.iis.user"), PROPERTIES.get("webdav.iis.password") - )); - host.setDefaultPath("/WebDAV"); - final DAVSession session = new DAVSession(host, new DisabledX509TrustManager(), new DefaultX509KeyManager()); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); final Path test = new DAVTouchFeature(session).touch(new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)), new TransferStatus()); final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random()); final byte[] content = RandomUtils.nextBytes(1023); diff --git a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVTimestampFeatureTest.java b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVTimestampFeatureTest.java index d9a35363aa4..96574de00ef 100644 --- a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVTimestampFeatureTest.java +++ b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVTimestampFeatureTest.java @@ -16,25 +16,15 @@ */ import ch.cyberduck.core.AlphanumericRandomStringService; -import ch.cyberduck.core.Credentials; import ch.cyberduck.core.DefaultPathPredicate; -import ch.cyberduck.core.DisabledCancelCallback; -import ch.cyberduck.core.DisabledHostKeyCallback; import ch.cyberduck.core.DisabledListProgressListener; import ch.cyberduck.core.DisabledLoginCallback; -import ch.cyberduck.core.Host; import ch.cyberduck.core.Path; import ch.cyberduck.core.PathAttributes; -import ch.cyberduck.core.dav.AbstractDAVTest; import ch.cyberduck.core.dav.DAVDeleteFeature; -import ch.cyberduck.core.dav.DAVProtocol; -import ch.cyberduck.core.dav.DAVSession; import ch.cyberduck.core.dav.DAVTouchFeature; import ch.cyberduck.core.features.Delete; -import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.shared.DefaultHomeFinderService; -import ch.cyberduck.core.ssl.DefaultX509KeyManager; -import ch.cyberduck.core.ssl.DisabledX509TrustManager; import ch.cyberduck.core.transfer.TransferStatus; import ch.cyberduck.test.IntegrationTest; @@ -47,17 +37,10 @@ import static org.junit.Assert.*; @Category(IntegrationTest.class) -public class MicrosoftIISDAVTimestampFeatureTest extends AbstractDAVTest { +public class MicrosoftIISDAVTimestampFeatureTest extends AbstractMicrosoftIISDAVTest { @Test public void testSetTimestamp() throws Exception { - final Host host = new Host(new DAVProtocol(), "winbuild.iterate.ch", new Credentials( - PROPERTIES.get("webdav.iis.user"), PROPERTIES.get("webdav.iis.password") - )); - host.setDefaultPath("/WebDAV"); - final DAVSession session = new DAVSession(host, new DisabledX509TrustManager(), new DefaultX509KeyManager()); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); final Path file = new DAVTouchFeature(session).touch(new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)), new TransferStatus()); assertTrue(new MicrosoftIISDAVFindFeature(session).find(file)); assertNotSame(PathAttributes.EMPTY, new MicrosoftIISDAVAttributesFinderFeature(session).find(file)); From f83d1fdaf172526dc46c26d5b8640bce7300eadc Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 23 Apr 2024 14:49:08 +0200 Subject: [PATCH 02/13] Add test. --- .../MicrosoftIISDAVReadFeatureTest.java | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java index e0b8c80f063..3a83fa714d8 100644 --- a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java +++ b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVReadFeatureTest.java @@ -22,7 +22,6 @@ import ch.cyberduck.core.Local; import ch.cyberduck.core.Path; import ch.cyberduck.core.dav.DAVDeleteFeature; -import ch.cyberduck.core.dav.DAVTouchFeature; import ch.cyberduck.core.dav.DAVUploadFeature; import ch.cyberduck.core.features.Delete; import ch.cyberduck.core.io.BandwidthThrottle; @@ -42,6 +41,14 @@ import java.io.OutputStream; import java.util.Collections; import java.util.EnumSet; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; import static org.junit.Assert.*; @@ -49,8 +56,8 @@ public class MicrosoftIISDAVReadFeatureTest extends AbstractMicrosoftIISDAVTest { @Test - public void testReadMicrosoft() throws Exception { - final Path test = new DAVTouchFeature(session).touch(new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)), new TransferStatus()); + public void testReadConcurrency() throws Exception { + final Path test = new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random()); final byte[] content = RandomUtils.nextBytes(1023); final OutputStream out = local.getOutputStream(false); @@ -61,18 +68,34 @@ public void testReadMicrosoft() throws Exception { test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), new TransferStatus().withLength(content.length), new DisabledConnectionCallback()); - assertTrue(new MicrosoftIISDAVFindFeature(session).find(test)); - assertEquals(content.length, new MicrosoftIISDAVListService(session, new MicrosoftIISDAVAttributesFinderFeature(session)).list(test.getParent(), new DisabledListProgressListener()).get(test).attributes().getSize(), 0L); - final TransferStatus status = new TransferStatus(); - status.setLength(-1L); - final InputStream in = new MicrosoftIISDAVReadFeature(session).read(test, status, new DisabledConnectionCallback()); - assertNotNull(in); - final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length); - new StreamCopier(status, status).transfer(in, buffer); - final byte[] reference = new byte[content.length]; - System.arraycopy(content, 0, reference, 0, content.length); - assertArrayEquals(reference, buffer.toByteArray()); - in.close(); - new DAVDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + final ExecutorService service = Executors.newCachedThreadPool(); + final BlockingQueue> queue = new LinkedBlockingQueue<>(); + final CompletionService completion = new ExecutorCompletionService<>(service, queue); + final int num = 5; + for(int i = 0; i < num; i++) { + completion.submit(new Callable() { + @Override + public Void call() throws Exception { + assertTrue(new MicrosoftIISDAVFindFeature(session).find(test)); + assertEquals(content.length, new MicrosoftIISDAVListService(session, new MicrosoftIISDAVAttributesFinderFeature(session)).list(test.getParent(), new DisabledListProgressListener()).get(test).attributes().getSize(), 0L); + final TransferStatus status = new TransferStatus(); + status.setLength(-1L); + final InputStream in = new MicrosoftIISDAVReadFeature(session).read(test, status, new DisabledConnectionCallback()); + assertNotNull(in); + final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length); + new StreamCopier(status, status).transfer(in, buffer); + final byte[] reference = new byte[content.length]; + System.arraycopy(content, 0, reference, 0, content.length); + assertArrayEquals(reference, buffer.toByteArray()); + in.close(); + return null; + } + }); + } + for(int i = 0; i < num; i++) { + completion.take().get(); + } + new DAVDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + local.delete(); } } From 23071a25bf0f993f2cdf329a72bcd7e46baa044a Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 23 Apr 2024 16:17:40 +0200 Subject: [PATCH 03/13] Limit attributes caching to single execution context. --- .../java/ch/cyberduck/core/dav/DAVClient.java | 37 ++++++++++++++++++- .../ch/cyberduck/core/dav/DAVSession.java | 24 +++++++----- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/DAVClient.java b/webdav/src/main/java/ch/cyberduck/core/dav/DAVClient.java index 199791c0906..ede73fc3b2f 100644 --- a/webdav/src/main/java/ch/cyberduck/core/dav/DAVClient.java +++ b/webdav/src/main/java/ch/cyberduck/core/dav/DAVClient.java @@ -21,12 +21,19 @@ import ch.cyberduck.core.preferences.PreferencesFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; import org.apache.http.client.ResponseHandler; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.entity.StringEntity; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -34,6 +41,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -52,11 +60,30 @@ public class DAVClient extends SardineImpl { private final String uri; + private final BasicAuthCache authCache = new BasicAuthCache(); + private final CredentialsProvider authProvider = new BasicCredentialsProvider(); + public DAVClient(final String uri, final HttpClientBuilder http) { super(http); this.uri = uri; } + public void setCredentials(final AuthScope authScope, final Credentials credentials) { + authProvider.setCredentials(authScope, credentials); + } + + @Override + public void enablePreemptiveAuthentication(final String hostname, final int httpPort, final int httpsPort, final Charset credentialsCharset) { + final BasicScheme basicScheme = new BasicScheme(credentialsCharset); + authCache.put(new HttpHost(hostname, httpPort, "http"), basicScheme); + authCache.put(new HttpHost(hostname, httpsPort, "https"), basicScheme); + } + + @Override + public void disablePreemptiveAuthentication() { + authCache.clear(); + } + @Override public T execute(final HttpRequestBase request, final ResponseHandler responseHandler) throws IOException { if(StringUtils.isNotBlank(request.getURI().getRawQuery())) { @@ -65,7 +92,10 @@ public T execute(final HttpRequestBase request, final ResponseHandler res else { request.setURI(URI.create(String.format("%s%s", uri, request.getURI().getRawPath()))); } - return super.execute(request, responseHandler); + final HttpClientContext context = HttpClientContext.create(); + context.setCredentialsProvider(authProvider); + context.setAuthCache(authCache); + return this.execute(context, request, responseHandler); } @Override @@ -76,7 +106,10 @@ protected HttpResponse execute(final HttpRequestBase request) throws IOException else { request.setURI(URI.create(String.format("%s%s", uri, request.getURI().getRawPath()))); } - return super.execute(request); + final HttpClientContext context = HttpClientContext.create(); + context.setCredentialsProvider(authProvider); + context.setAuthCache(authCache); + return this.execute(context, request, null); } @Override diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java index 0d45cc4786f..158b9b1727a 100644 --- a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java +++ b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java @@ -60,16 +60,16 @@ import org.apache.http.Header; import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseInterceptor; import org.apache.http.HttpStatus; import org.apache.http.auth.AuthScope; import org.apache.http.auth.NTCredentials; import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.AuthSchemes; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpHead; -import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.HttpContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -113,6 +113,14 @@ protected DAVClient connect(final Proxy proxy, final HostKeyCallback key, final protected HttpClientBuilder getConfiguration(final Proxy proxy, final LoginCallback prompt) throws ConnectionCanceledException { final HttpClientBuilder configuration = builder.build(proxy, this, prompt); configuration.setRedirectStrategy(new DAVRedirectStrategy(redirect)); + configuration.addInterceptorLast(new HttpResponseInterceptor() { + @Override + public void process(final HttpResponse response, final HttpContext context) { + if(response.containsHeader("Persistent-Auth")) { + client.disablePreemptiveAuthentication(); + } + } + }); return configuration; } @@ -139,27 +147,25 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal username = credentials.getUsername(); domain = new HostPreferences(host).getProperty("webdav.ntlm.domain"); } - final CredentialsProvider provider = new BasicCredentialsProvider(); - provider.setCredentials( + client.setCredentials( new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.NTLM), new NTCredentials(username, credentials.getPassword(), preferences.getProperty("webdav.ntlm.workstation"), domain) ); - provider.setCredentials( + client.setCredentials( new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.SPNEGO), new NTCredentials(username, credentials.getPassword(), preferences.getProperty("webdav.ntlm.workstation"), domain) ); - provider.setCredentials( + client.setCredentials( new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.BASIC), new UsernamePasswordCredentials(username, credentials.getPassword())); - provider.setCredentials( + client.setCredentials( new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.DIGEST), new UsernamePasswordCredentials(username, credentials.getPassword())); - provider.setCredentials( + client.setCredentials( new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM, AuthSchemes.KERBEROS), new UsernamePasswordCredentials(username, credentials.getPassword())); - client.setCredentials(provider); if(preferences.getBoolean("webdav.basic.preemptive")) { switch(proxy.getType()) { case DIRECT: From 89eb1c31e1dd00e84f2019437844c013d305a1d3 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 23 Apr 2024 19:32:49 +0200 Subject: [PATCH 04/13] User no-op user token handler. This ensures connection from the pool can be shared regardless of the authentication state. --- .../ch/cyberduck/core/http/HttpConnectionPoolBuilder.java | 8 ++++++++ defaults/src/main/resources/default.properties | 1 + 2 files changed, 9 insertions(+) diff --git a/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java b/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java index 53e67c7f373..4b1f8902388 100644 --- a/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java +++ b/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java @@ -53,8 +53,10 @@ import org.apache.http.impl.auth.SPNegoSchemeFactory; import org.apache.http.impl.client.AIMDBackoffManager; import org.apache.http.impl.client.DefaultClientConnectionReuseStrategy; +import org.apache.http.impl.client.DefaultUserTokenHandler; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.NoopUserTokenHandler; import org.apache.http.impl.client.WinHttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; @@ -173,6 +175,12 @@ public HttpClientBuilder build(final Proxy proxy, final TranscriptListener liste else { configuration.setConnectionReuseStrategy(new NoConnectionReuseStrategy()); } + if(new HostPreferences(host).getBoolean("http.connections.tokenhandler.enable")) { + configuration.setUserTokenHandler(DefaultUserTokenHandler.INSTANCE); + } + else { + configuration.setUserTokenHandler(NoopUserTokenHandler.INSTANCE); + } // Retry handler for I/O failures configuration.setRetryHandler(new ExtendedHttpRequestRetryHandler( new HostPreferences(host).getInteger("connection.retry"))); diff --git a/defaults/src/main/resources/default.properties b/defaults/src/main/resources/default.properties index 6e4248fb230..dabb139d3cf 100644 --- a/defaults/src/main/resources/default.properties +++ b/defaults/src/main/resources/default.properties @@ -236,6 +236,7 @@ http.compression.enable=true # Integer.MAX_VALUE http.connections.route=2147483647 http.connections.reuse=true +http.connections.tokenhandler.enable=false http.connections.stale.check.ms=5000 # Total number of connections in the pool From 6563a7567e2403e6a0661f04a79fa06dc0c5d3cf Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 24 Apr 2024 09:04:48 +0200 Subject: [PATCH 05/13] Use disable connection state helper instead of providing user token handler. --- .../cyberduck/core/http/HttpConnectionPoolBuilder.java | 9 ++------- defaults/src/main/resources/default.properties | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java b/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java index 4b1f8902388..8c7144c3ce0 100644 --- a/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java +++ b/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java @@ -53,10 +53,8 @@ import org.apache.http.impl.auth.SPNegoSchemeFactory; import org.apache.http.impl.client.AIMDBackoffManager; import org.apache.http.impl.client.DefaultClientConnectionReuseStrategy; -import org.apache.http.impl.client.DefaultUserTokenHandler; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.client.NoopUserTokenHandler; import org.apache.http.impl.client.WinHttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.protocol.HttpContext; @@ -175,11 +173,8 @@ public HttpClientBuilder build(final Proxy proxy, final TranscriptListener liste else { configuration.setConnectionReuseStrategy(new NoConnectionReuseStrategy()); } - if(new HostPreferences(host).getBoolean("http.connections.tokenhandler.enable")) { - configuration.setUserTokenHandler(DefaultUserTokenHandler.INSTANCE); - } - else { - configuration.setUserTokenHandler(NoopUserTokenHandler.INSTANCE); + if(!new HostPreferences(host).getBoolean("http.connections.state.enable")) { + configuration.disableConnectionState(); } // Retry handler for I/O failures configuration.setRetryHandler(new ExtendedHttpRequestRetryHandler( diff --git a/defaults/src/main/resources/default.properties b/defaults/src/main/resources/default.properties index dabb139d3cf..26a7788d102 100644 --- a/defaults/src/main/resources/default.properties +++ b/defaults/src/main/resources/default.properties @@ -236,7 +236,7 @@ http.compression.enable=true # Integer.MAX_VALUE http.connections.route=2147483647 http.connections.reuse=true -http.connections.tokenhandler.enable=false +http.connections.state.enable=false http.connections.stale.check.ms=5000 # Total number of connections in the pool From 670397490d6aa16079c9cf90b2ee18d364c3e479 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 25 Apr 2024 08:04:35 +0200 Subject: [PATCH 06/13] Addendum to 13602aaa. Must mark as non-repeatable to allow buffering contents. --- .../ch/cyberduck/core/http/DelayedHttpEntity.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/ch/cyberduck/core/http/DelayedHttpEntity.java b/core/src/main/java/ch/cyberduck/core/http/DelayedHttpEntity.java index ed3a66e71b5..3c5e10f1581 100644 --- a/core/src/main/java/ch/cyberduck/core/http/DelayedHttpEntity.java +++ b/core/src/main/java/ch/cyberduck/core/http/DelayedHttpEntity.java @@ -66,18 +66,13 @@ public DelayedHttpEntity(final Thread parentThread, final CountDownLatch streamO */ private OutputStream stream; - /** - * Entity written to server - */ - private boolean entityWritten = false; - /** * Parent thread to check if still alive */ private final Thread parentThread; public boolean isRepeatable() { - return !entityWritten; + return false; } public abstract long getContentLength(); @@ -120,12 +115,10 @@ protected void handleIOException(final IOException e) throws IOException { } // Wait for signal when content has been written to the pipe Interruptibles.await(streamClosed, IOException.class, new Interruptibles.ThreadAliveCancelCallback(parentThread)); - // Entity written to server - entityWritten = true; } public boolean isStreaming() { - return !entityWritten; + return true; } /** From eed639d4ac571635570d83f18298dd52adff7972 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 25 Apr 2024 09:16:51 +0200 Subject: [PATCH 07/13] Extract capabilities bean. --- .../ch/cyberduck/core/dav/DAVSession.java | 19 +++++++++++++++---- .../cyberduck/core/dav/DAVUploadFeature.java | 5 +++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java index 158b9b1727a..7390bfcf886 100644 --- a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java +++ b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java @@ -86,8 +86,8 @@ public class DAVSession extends HttpSession { private static final Logger log = LogManager.getLogger(DAVSession.class); private final RedirectCallback redirect; - private final PreferencesReader preferences - = new HostPreferences(host); + private final PreferencesReader preferences = new HostPreferences(host); + private final HttpCapabilities capabilities = new HttpCapabilities(preferences); private ListService list = new DAVListService(this, new DAVAttributesFinderFeature(this)); private Read read = new DAVReadFeature(this); @@ -321,10 +321,10 @@ public T _getFeature(final Class type) { return (T) read; } if(type == Write.class) { - return (T) new DAVWriteFeature(this); + return (T) new DAVWriteFeature(this, capabilities.expectcontinue); } if(type == Upload.class) { - return (T) new DAVUploadFeature(this); + return (T) new DAVUploadFeature(new DAVWriteFeature(this, capabilities.expectcontinue)); } if(type == Delete.class) { return (T) new DAVDeleteFeature(this); @@ -385,4 +385,15 @@ public Void handleResponse(final HttpResponse response) throws IOException { return null; } } + + private static final class HttpCapabilities { + /** + * Support for Expect: Continue + */ + boolean expectcontinue; + + public HttpCapabilities(final PreferencesReader preferences) { + this.expectcontinue = preferences.getBoolean("webdav.expect-continue"); + } + } } diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/DAVUploadFeature.java b/webdav/src/main/java/ch/cyberduck/core/dav/DAVUploadFeature.java index f252aecb558..ba70c024cc3 100644 --- a/webdav/src/main/java/ch/cyberduck/core/dav/DAVUploadFeature.java +++ b/webdav/src/main/java/ch/cyberduck/core/dav/DAVUploadFeature.java @@ -17,6 +17,7 @@ * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ +import ch.cyberduck.core.features.Write; import ch.cyberduck.core.http.HttpUploadFeature; import java.security.MessageDigest; @@ -26,4 +27,8 @@ public class DAVUploadFeature extends HttpUploadFeature { public DAVUploadFeature(final DAVSession session) { super(new DAVWriteFeature(session)); } + + public DAVUploadFeature(final Write writer) { + super(writer); + } } From 549cbf3f6dd7680a593ec3bae64bc895c226ac59 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 25 Apr 2024 09:20:36 +0200 Subject: [PATCH 08/13] Extract handler. --- .../ch/cyberduck/core/dav/DAVSession.java | 78 ++++++++++--------- .../MicrosoftIISFeaturesResponseHandler.java | 52 +++++++++++++ 2 files changed, 93 insertions(+), 37 deletions(-) create mode 100644 webdav/src/main/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISFeaturesResponseHandler.java diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java index 7390bfcf886..a464414efdb 100644 --- a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java +++ b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java @@ -37,6 +37,7 @@ import ch.cyberduck.core.dav.microsoft.MicrosoftIISDAVListService; import ch.cyberduck.core.dav.microsoft.MicrosoftIISDAVReadFeature; import ch.cyberduck.core.dav.microsoft.MicrosoftIISDAVTimestampFeature; +import ch.cyberduck.core.dav.microsoft.MicrosoftIISFeaturesResponseHandler; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConnectionCanceledException; import ch.cyberduck.core.exception.ListCanceledException; @@ -77,7 +78,6 @@ import java.net.URL; import java.nio.charset.Charset; import java.text.MessageFormat; -import java.util.Arrays; import com.github.sardine.impl.SardineException; import com.github.sardine.impl.handler.ValidatingResponseHandler; @@ -89,12 +89,6 @@ public class DAVSession extends HttpSession { private final PreferencesReader preferences = new HostPreferences(host); private final HttpCapabilities capabilities = new HttpCapabilities(preferences); - private ListService list = new DAVListService(this, new DAVAttributesFinderFeature(this)); - private Read read = new DAVReadFeature(this); - private Timestamp timestamp = new DAVTimestampFeature(this); - private AttributesFinder attributes = new DAVAttributesFinderFeature(this); - private Find find = new DAVFindFeature(this); - public DAVSession(final Host host, final X509TrustManager trust, final X509KeyManager key) { this(host, trust, key, new PreferencesRedirectCallback()); } @@ -195,7 +189,7 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal final Path home = new DelegatingHomeFeature(new WorkdirHomeFeature(host), new DefaultPathHomeFeature(host)).find(); final HttpHead head = new HttpHead(new DAVPathEncoder().encode(home)); try { - client.execute(head, new MicrosoftIISFeaturesResponseHandler()); + client.execute(head, new MicrosoftIISFeaturesResponseHandler(capabilities)); } catch(SardineException e) { switch(e.getStatusCode()) { @@ -214,6 +208,7 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal } cancel.verify(); // Possibly only HEAD requests are not allowed + final ListService list = this.getFeature(ListService.class); list.list(home, new DisabledListProgressListener() { @Override public void chunk(final Path parent, final AttributedList list) throws ListCanceledException { @@ -234,7 +229,7 @@ public void chunk(final Path parent, final AttributedList list) throws Lis } cancel.verify(); client.disablePreemptiveAuthentication(); - client.execute(head, new MicrosoftIISFeaturesResponseHandler()); + client.execute(head, new MicrosoftIISFeaturesResponseHandler(capabilities)); } else { throw new DAVExceptionMappingService().map(e); @@ -312,13 +307,21 @@ public Header handleResponse(final HttpResponse response) { @SuppressWarnings("unchecked") public T _getFeature(final Class type) { if(type == ListService.class) { - return (T) list; + if(capabilities.iis) { + return (T) new MicrosoftIISDAVListService(this, new MicrosoftIISDAVAttributesFinderFeature(this)); + } + return (T) new DAVListService(this, new DAVAttributesFinderFeature(this)); } if(type == Directory.class) { return (T) new DAVDirectoryFeature(this); } if(type == Read.class) { - return (T) read; + if(capabilities.iis) { + if(preferences.getBoolean("webdav.microsoftiis.header.translate")) { + return (T) new MicrosoftIISDAVReadFeature(this); + } + } + return (T) new DAVReadFeature(this); } if(type == Write.class) { return (T) new DAVWriteFeature(this, capabilities.expectcontinue); @@ -342,13 +345,24 @@ public T _getFeature(final Class type) { return (T) new DAVCopyFeature(this); } if(type == Find.class) { - return (T) find; + if(capabilities.iis) { + if(preferences.getBoolean("webdav.microsoftiis.header.translate")) { + return (T) new MicrosoftIISDAVFindFeature(this); + } + } + return (T) new DAVFindFeature(this); } if(type == AttributesFinder.class) { - return (T) attributes; + if(capabilities.iis) { + return (T) new MicrosoftIISDAVAttributesFinderFeature(this); + } + return (T) new DAVAttributesFinderFeature(this); } if(type == Timestamp.class) { - return (T) timestamp; + if(capabilities.iis) { + return (T) new MicrosoftIISDAVTimestampFeature(this); + } + return (T) new DAVTimestampFeature(this); } if(type == Quota.class) { return (T) new DAVQuotaFeature(this); @@ -365,35 +379,25 @@ public T _getFeature(final Class type) { return super._getFeature(type); } - private final class MicrosoftIISFeaturesResponseHandler extends ValidatingResponseHandler { - @Override - public Void handleResponse(final HttpResponse response) throws IOException { - if(Arrays.stream(response.getAllHeaders()).anyMatch(header -> - HttpHeaders.SERVER.equals(header.getName()) && StringUtils.contains(header.getValue(), "Microsoft-IIS"))) { - if(log.isInfoEnabled()) { - log.info(String.format("Microsoft-IIS backend detected in response %s", response)); - } - list = new MicrosoftIISDAVListService(DAVSession.this, new MicrosoftIISDAVAttributesFinderFeature(DAVSession.this)); - timestamp = new MicrosoftIISDAVTimestampFeature(DAVSession.this); - attributes = new MicrosoftIISDAVAttributesFinderFeature(DAVSession.this); - if(preferences.getBoolean("webdav.microsoftiis.header.translate")) { - read = new MicrosoftIISDAVReadFeature(DAVSession.this); - find = new MicrosoftIISDAVFindFeature(DAVSession.this); - } - } - this.validateResponse(response); - return null; - } - } - - private static final class HttpCapabilities { + public static final class HttpCapabilities { /** * Support for Expect: Continue */ - boolean expectcontinue; + public boolean expectcontinue; + public boolean iis; public HttpCapabilities(final PreferencesReader preferences) { this.expectcontinue = preferences.getBoolean("webdav.expect-continue"); } + + public HttpCapabilities withExpectcontinue(final boolean expectcontinue) { + this.expectcontinue = expectcontinue; + return this; + } + + public HttpCapabilities withIIS(final boolean iis) { + this.iis = iis; + return this; + } } } diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISFeaturesResponseHandler.java b/webdav/src/main/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISFeaturesResponseHandler.java new file mode 100644 index 00000000000..25bf242b06b --- /dev/null +++ b/webdav/src/main/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISFeaturesResponseHandler.java @@ -0,0 +1,52 @@ +package ch.cyberduck.core.dav.microsoft; + +/* + * Copyright (c) 2002-2024 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.dav.DAVSession; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Arrays; + +import com.github.sardine.impl.handler.ValidatingResponseHandler; + +public final class MicrosoftIISFeaturesResponseHandler extends ValidatingResponseHandler { + private static final Logger log = LogManager.getLogger(MicrosoftIISFeaturesResponseHandler.class); + + private final DAVSession.HttpCapabilities capabilities; + + public MicrosoftIISFeaturesResponseHandler(final DAVSession.HttpCapabilities capabilities) { + this.capabilities = capabilities; + } + + @Override + public Void handleResponse(final HttpResponse response) throws IOException { + if(Arrays.stream(response.getAllHeaders()).anyMatch(header -> + HttpHeaders.SERVER.equals(header.getName()) && StringUtils.contains(header.getValue(), "Microsoft-IIS"))) { + if(log.isInfoEnabled()) { + log.info(String.format("Microsoft-IIS backend detected in response %s", response)); + } + capabilities.withIIS(true); + } + this.validateResponse(response); + return null; + } +} From 75d0601c5e368be072f036e7ac1a9d9de9081c57 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 25 Apr 2024 09:48:16 +0200 Subject: [PATCH 09/13] Extract inner class. --- .../java/ch/cyberduck/core/dav/DAVSession.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java index a464414efdb..564060405c5 100644 --- a/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java +++ b/webdav/src/main/java/ch/cyberduck/core/dav/DAVSession.java @@ -107,14 +107,7 @@ protected DAVClient connect(final Proxy proxy, final HostKeyCallback key, final protected HttpClientBuilder getConfiguration(final Proxy proxy, final LoginCallback prompt) throws ConnectionCanceledException { final HttpClientBuilder configuration = builder.build(proxy, this, prompt); configuration.setRedirectStrategy(new DAVRedirectStrategy(redirect)); - configuration.addInterceptorLast(new HttpResponseInterceptor() { - @Override - public void process(final HttpResponse response, final HttpContext context) { - if(response.containsHeader("Persistent-Auth")) { - client.disablePreemptiveAuthentication(); - } - } - }); + configuration.addInterceptorLast(new MicrosoftIISPersistentAuthResponseInterceptor()); return configuration; } @@ -400,4 +393,13 @@ public HttpCapabilities withIIS(final boolean iis) { return this; } } + + private final class MicrosoftIISPersistentAuthResponseInterceptor implements HttpResponseInterceptor { + @Override + public void process(final HttpResponse response, final HttpContext context) { + if(response.containsHeader("Persistent-Auth")) { + client.disablePreemptiveAuthentication(); + } + } + } } From d77f35b389c67e5c3f554cdb63719f57fcec3aa9 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 25 Apr 2024 18:36:48 +0200 Subject: [PATCH 10/13] Review test. --- .../core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java index c09d5e9dc31..56b521e5acd 100644 --- a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java +++ b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVLockFeatureTest.java @@ -83,5 +83,6 @@ public void testLock() throws Exception { } new DAVLockFeature(session).unlock(test, lock); new DAVDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + local.delete(); } } From dedd98f47b822ca540bfe28ddde5cec21da06cbf Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 25 Apr 2024 18:57:27 +0200 Subject: [PATCH 11/13] Add tests. --- .../MicrosoftIISDAVUploadFeatureTest.java | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVUploadFeatureTest.java diff --git a/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVUploadFeatureTest.java b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVUploadFeatureTest.java new file mode 100644 index 00000000000..ee7546d2857 --- /dev/null +++ b/webdav/src/test/java/ch/cyberduck/core/dav/microsoft/MicrosoftIISDAVUploadFeatureTest.java @@ -0,0 +1,105 @@ +package ch.cyberduck.core.dav.microsoft; + +/* + * Copyright (c) 2002-2024 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.AlphanumericRandomStringService; +import ch.cyberduck.core.DisabledConnectionCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.Local; +import ch.cyberduck.core.Path; +import ch.cyberduck.core.dav.DAVDeleteFeature; +import ch.cyberduck.core.dav.DAVUploadFeature; +import ch.cyberduck.core.exception.InteroperabilityException; +import ch.cyberduck.core.features.Delete; +import ch.cyberduck.core.io.BandwidthThrottle; +import ch.cyberduck.core.io.DisabledStreamListener; +import ch.cyberduck.core.shared.DefaultHomeFinderService; +import ch.cyberduck.core.transfer.TransferStatus; +import ch.cyberduck.test.IntegrationTest; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.RandomUtils; +import org.apache.http.conn.ClientConnectionManager; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.OutputStream; +import java.util.Collections; +import java.util.EnumSet; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; + +@Category(IntegrationTest.class) +public class MicrosoftIISDAVUploadFeatureTest extends AbstractMicrosoftIISDAVTest { + + @Test + public void testUpload() throws Exception { + final Path test = new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); + final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random()); + final byte[] content = RandomUtils.nextBytes(9273); + final OutputStream out = local.getOutputStream(false); + assertNotNull(out); + IOUtils.write(content, out); + out.close(); + new DAVUploadFeature(session).upload( + test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), + new TransferStatus().withLength(content.length), + new DisabledConnectionCallback()); + new DAVDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + local.delete(); + } + + @Test + public void testUploadNoConnectionInPool() throws Exception { + final Path test = new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); + final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random()); + final byte[] content = RandomUtils.nextBytes(7365); + final OutputStream out = local.getOutputStream(false); + assertNotNull(out); + IOUtils.write(content, out); + out.close(); + // Close connections in pool to require new NTLM handshake + final ClientConnectionManager manager = session.getClient().getClient().getConnectionManager(); + manager.closeIdleConnections(0L, TimeUnit.MILLISECONDS); + assertThrows(InteroperabilityException.class, () -> new DAVUploadFeature(session).upload( + test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), + new TransferStatus().withLength(content.length), + new DisabledConnectionCallback())); + local.delete(); + } + + @Test + public void testZeroByteUploadNoConnectionInPool() throws Exception { + final Path test = new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); + final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random()); + final byte[] content = RandomUtils.nextBytes(0); + final OutputStream out = local.getOutputStream(false); + assertNotNull(out); + IOUtils.write(content, out); + out.close(); + // Close connections in pool to require new NTLM handshake + final ClientConnectionManager manager = session.getClient().getClient().getConnectionManager(); + manager.closeIdleConnections(0L, TimeUnit.MILLISECONDS); + new DAVUploadFeature(session).upload( + test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), + new TransferStatus().withLength(content.length), + new DisabledConnectionCallback()); + new DAVDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + local.delete(); + } +} From 0e6202ad89d5528d39492081a2ddee14ebfbb808 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 25 Apr 2024 19:03:19 +0200 Subject: [PATCH 12/13] Add option to prematurely fail requests that would require NTLM handshake in PUT/POST requests enclosign entity that is not supported. --- ...or.java => CustomHttpRequestExecutor.java} | 38 +++++++++++++++++-- .../core/http/HttpConnectionPoolBuilder.java | 2 +- .../src/main/resources/default.properties | 1 + 3 files changed, 37 insertions(+), 4 deletions(-) rename core/src/main/java/ch/cyberduck/core/http/{LoggingHttpRequestExecutor.java => CustomHttpRequestExecutor.java} (63%) diff --git a/core/src/main/java/ch/cyberduck/core/http/LoggingHttpRequestExecutor.java b/core/src/main/java/ch/cyberduck/core/http/CustomHttpRequestExecutor.java similarity index 63% rename from core/src/main/java/ch/cyberduck/core/http/LoggingHttpRequestExecutor.java rename to core/src/main/java/ch/cyberduck/core/http/CustomHttpRequestExecutor.java index 54d6679c0da..d1ba67d74d5 100644 --- a/core/src/main/java/ch/cyberduck/core/http/LoggingHttpRequestExecutor.java +++ b/core/src/main/java/ch/cyberduck/core/http/CustomHttpRequestExecutor.java @@ -18,9 +18,12 @@ * feedback@cyberduck.io */ +import ch.cyberduck.core.Host; import ch.cyberduck.core.PreferencesUseragentProvider; import ch.cyberduck.core.TranscriptListener; import ch.cyberduck.core.UseragentProvider; +import ch.cyberduck.core.exception.RetriableAccessDeniedException; +import ch.cyberduck.core.preferences.HostPreferences; import org.apache.commons.lang3.StringUtils; import org.apache.http.Header; @@ -29,20 +32,28 @@ import org.apache.http.HttpHeaders; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpResponseException; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.message.BasicHeader; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpRequestExecutor; import java.io.IOException; +import java.util.Arrays; -public class LoggingHttpRequestExecutor extends HttpRequestExecutor { +public class CustomHttpRequestExecutor extends HttpRequestExecutor { private final UseragentProvider useragentProvider = new PreferencesUseragentProvider(); + private final Host host; private final TranscriptListener listener; - public LoggingHttpRequestExecutor(final TranscriptListener listener) { + public CustomHttpRequestExecutor(final Host host, final TranscriptListener listener) { + this.host = host; this.listener = listener; } @@ -77,8 +88,29 @@ protected HttpResponse doSendRequest(final HttpRequest request, final HttpClient } final HttpResponse response = super.doSendRequest(request, conn, context); if(null != response) { - // response received as part of an expect-continue handshake + // Response received as part of an expect-continue handshake this.log(response); + if(new HostPreferences(host).getBoolean("request.unauthorized.ntlm.preflight")) { + // Unable to authenticate with NTLM for requests with non-zero entity + if(HttpStatus.SC_UNAUTHORIZED == response.getStatusLine().getStatusCode()) { + if(response.containsHeader(HttpHeaders.WWW_AUTHENTICATE)) { + switch(request.getRequestLine().getMethod()) { + case HttpPut.METHOD_NAME: + case HttpPost.METHOD_NAME: + case HttpPatch.METHOD_NAME: + if(Arrays.asList(response.getAllHeaders()).stream() + .filter(header -> HttpHeaders.WWW_AUTHENTICATE.equals(header.getName())) + .filter(header -> "NTLM".equals(header.getValue())).findAny().isPresent()) { + // Unauthenticated connection cannot proceed with PUT + final HttpResponseException preflight = new HttpResponseException(response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + preflight.initCause(new RetriableAccessDeniedException(String.format("Authentication cannot proceed for %s", + request.getRequestLine().getMethod()))); + throw preflight; + } + } + } + } + } } return response; } diff --git a/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java b/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java index 8c7144c3ce0..c99ded1ede2 100644 --- a/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java +++ b/core/src/main/java/ch/cyberduck/core/http/HttpConnectionPoolBuilder.java @@ -184,7 +184,7 @@ public HttpClientBuilder build(final Proxy proxy, final TranscriptListener liste if(!new HostPreferences(host).getBoolean("http.compression.enable")) { configuration.disableContentCompression(); } - configuration.setRequestExecutor(new LoggingHttpRequestExecutor(listener)); + configuration.setRequestExecutor(new CustomHttpRequestExecutor(host, listener)); // Always register HTTP for possible use with proxy. Contains a number of protocol properties such as the // default port and the socket factory to be used to create the java.net.Socket instances for the given protocol final PoolingHttpClientConnectionManager connectionManager = this.createConnectionManager(this.createRegistry()); diff --git a/defaults/src/main/resources/default.properties b/defaults/src/main/resources/default.properties index 26a7788d102..c5eaa6313c2 100644 --- a/defaults/src/main/resources/default.properties +++ b/defaults/src/main/resources/default.properties @@ -249,6 +249,7 @@ http.socket.buffer=8192 http.credentials.charset=UTF-8 http.request.uri.normalize=false http.request.entity.buffer.limit=5242880 +request.unauthorized.ntlm.preflight=false # Enable or disable verification that the remote host taking part # of a data connection is the same as the host to which the control From 9796900bc1deeeb434e39fff7bdfababc5b858cb Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 28 Apr 2024 18:15:32 +0200 Subject: [PATCH 13/13] Add retry for 417 Expectation Failed reply. --- .../core/nextcloud/NextcloudWriteFeature.java | 4 ++-- .../ch/cyberduck/core/dav/DAVWriteFeature.java | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/nextcloud/src/main/java/ch/cyberduck/core/nextcloud/NextcloudWriteFeature.java b/nextcloud/src/main/java/ch/cyberduck/core/nextcloud/NextcloudWriteFeature.java index ea7f3c33cec..4b27ff6cd8c 100644 --- a/nextcloud/src/main/java/ch/cyberduck/core/nextcloud/NextcloudWriteFeature.java +++ b/nextcloud/src/main/java/ch/cyberduck/core/nextcloud/NextcloudWriteFeature.java @@ -37,8 +37,8 @@ public NextcloudWriteFeature(final DAVSession session) { } @Override - protected List
getHeaders(final Path file, final TransferStatus status) throws UnsupportedException { - final List
headers = super.getHeaders(file, status); + protected List
toHeaders(final Path file, final TransferStatus status, final boolean expectdirective) throws UnsupportedException { + final List
headers = super.toHeaders(file, status, expectdirective); if(null != status.getModified()) { headers.add(new BasicHeader("X-OC-Mtime", String.valueOf(status.getModified() / 1000))); } diff --git a/webdav/src/main/java/ch/cyberduck/core/dav/DAVWriteFeature.java b/webdav/src/main/java/ch/cyberduck/core/dav/DAVWriteFeature.java index 837191f39a9..75ffe3a13d3 100644 --- a/webdav/src/main/java/ch/cyberduck/core/dav/DAVWriteFeature.java +++ b/webdav/src/main/java/ch/cyberduck/core/dav/DAVWriteFeature.java @@ -22,6 +22,7 @@ import ch.cyberduck.core.VoidAttributesAdapter; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConflictException; +import ch.cyberduck.core.exception.InteroperabilityException; import ch.cyberduck.core.exception.UnsupportedException; import ch.cyberduck.core.features.Lock; import ch.cyberduck.core.features.Write; @@ -72,21 +73,29 @@ public DAVWriteFeature(final DAVSession session, final boolean expect) { @Override public HttpResponseOutputStream write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { - return this.write(file, this.getHeaders(file, status), status); + return this.write(file, this.toHeaders(file, status, expect), status); } catch(ConflictException e) { if(expect) { if(null != status.getLockId()) { // Handle 412 Precondition Failed with expired token log.warn(String.format("Retry failure %s with lock id %s removed", e, status.getLockId())); - return this.write(file, this.getHeaders(file, status.withLockId(null)), status); + return this.write(file, this.toHeaders(file, status.withLockId(null), expect), status); } } throw e; } + catch(InteroperabilityException e) { + if(expect) { + // Handle 417 Expectation Failed + log.warn(String.format("Retry failure %s with Expect: Continue removed", e)); + return this.write(file, this.toHeaders(file, status.withLockId(null), false), status); + } + throw e; + } } - protected List
getHeaders(final Path file, final TransferStatus status) throws UnsupportedException { + protected List
toHeaders(final Path file, final TransferStatus status, final boolean expectdirective) throws UnsupportedException { final List
headers = new ArrayList<>(); if(status.isAppend()) { if(status.getLength() == TransferStatus.UNKNOWN_LENGTH) { @@ -102,7 +111,7 @@ protected List
getHeaders(final Path file, final TransferStatus status) } headers.add(new BasicHeader(HttpHeaders.CONTENT_RANGE, header)); } - if(expect) { + if(expectdirective) { if(status.getLength() > 0L) { headers.add(new BasicHeader(HTTP.EXPECT_DIRECTIVE, HTTP.EXPECT_CONTINUE)); }