[JSch-users] Problem transferring files larger than 4GB via SFTP

Nicolas Kukolja

2014-02-19 12:07:59 UTC

<html><head></head><body><div style="font-family: Verdana;font-size: 12.0px;"><div>Hi,</div>

<div> </div>

<div>I am implementing a java application that transfers (large) files to an ftp-server, using sftp (JSch).</div>

<div> </div>

<div>Everything works fine so far with small files. But if a file is larger, the transfer hangs always at about 4GB - 4,1GB and doesn't move on.</div>

<div>After 10 minutes, I get an "java.io.IOException: Pipe closed". The "rekey"-Param (http://www.proftpd.org/docs/contrib/mod_sftp.html#SFTPRekey) of the server is configured to 500MB, and the key exchange seems to work fine, as the server log tells me.</div>

<div> </div>

<div>Does anyone have a suggestion, what I am doing wrong?</div>

<div>If you need any more information, please give me a hint...</div>

<div> </div>

<div>
<div>
<div>Kind regards,</div>

<div>Nicolas</div>

<div> </div>

<div> </div>

<div> </div>
</div>
</div>

<div>Environment:</div>

<div>Client:</div>

<div>- Windows 2008 Server</div>

<div>- Java 1.6.0_45 x64</div>

<div>- Java application using JSch 0.1.50.jar</div>

<div> </div>

<div>Server:</div>

<div>- Solaris 10</div>

<div>- FTP-Server: http://www.proftpd.org</div>

<div> </div>

<div> 
<div> </div>

<div>My connect-method:</div>

<div>
<div>   public boolean connect( Long orderId )<br/>
   {<br/>
      try<br/>
      {<br/>
         this.jSch = new JSch();</div>

<div>         logger.info( "[Order:" + orderId + "] Connecting to SFTP-Server: " + this.username + ":" + this.password + "@" + this.host + ":" + this.port );<br/>
         // create session<br/>
         this.sftpSession = this.jSch.getSession( this.username, this.host, this.port );</div>

<div>         // build config<br/>
         Hashtable<String,String> config = new Hashtable<String,String>();<br/>
         config.put( "StrictHostKeyChecking", "no" );<br/>
         this.sftpSession.setConfig( config );<br/>
         this.sftpSession.setPassword( this.password );</div>

<div>         // establish connection<br/>
         this.sftpSession.connect();</div>

<div>         this.sftpChannel = ( ChannelSftp ) this.sftpSession.openChannel( "sftp" );<br/>
         this.sftpChannel.connect();</div>

<div>         logger.info( "[Order:" + orderId + "] Current directory on SFTP-Server (pwd): " + this.sftpChannel.pwd() );<br/>
         if( this.ftpSubdir != null && this.ftpSubdir.length() > 0 )<br/>
         {<br/>
            logger.info( "[Order:" + orderId + "] Changing to subdirectory on SFTP-Server (cd): " + this.ftpSubdir );<br/>
            this.sftpChannel.cd( this.ftpSubdir );<br/>
            logger.info( "[Order:" + orderId + "] Current directory on SFTP-Server (pwd): " + this.sftpChannel.pwd() );<br/>
         }<br/>
      }<br/>
      catch( JSchException e )<br/>
      {<br/>
         logger.error( "[Order:" + orderId + "] Error connecting to SFTP-Server... (uri: '" + this.sftpUri + "')", e );<br/>
         return false;<br/>
      }<br/>
      catch( SftpException e )<br/>
      {<br/>
         logger.error( "[Order:" + orderId + "] Error connecting to SFTP-Server... (uri: '" + this.sftpUri + "')", e );<br/>
         return false;<br/>
      }<br/>
      return this.sftpChannel.isConnected();<br/>
   }</div>

<div> </div>

<div>My upload-method:</div>

<div>
<div>   public void uploadSynchronized( String source, String target, Long filesize, Long orderId )<br/>
   {<br/>
      long overallFilesize = filesize == null ? 0 : filesize;<br/>
      long overallUploadedBytes = 0;<br/>
      long uploadedBytes = 0;</div>

<div>      try<br/>
      {<br/>
         if( !checkIfFileExists( source ) )<br/>
         {<br/>
            OutputStream tOut = this.sftpChannel.put( target );<br/>
            FileInputStream in = new FileInputStream( source );<br/>
            byte[] bytes = new byte[ this.chunkSize ];<br/>
            int count = in.read( bytes );<br/>
            try<br/>
            {<br/>
               while( count != -1 && count <= this.chunkSize && !this.abort )<br/>
               {<br/>
                  tOut.write( bytes, 0, count );</div>

<div>                  uploadedBytes = uploadedBytes + this.chunkSize;<br/>
                  if( uploadedBytes >= BYTES_TO_LOG )<br/>
                  {</div>

<div>                     overallUploadedBytes = overallUploadedBytes + uploadedBytes;<br/>
                     uploadedBytes = 0;<br/>
                     logger.info( "[Order:" + orderId + "] " + overallUploadedBytes / MEGA_BYTE + "MB of " + overallFilesize / MEGA_BYTE<br/>
                           + "MB uploaded (source: '" + source + "', target: '" + target + "')" );<br/>
                  }<br/>
                  count = in.read( bytes );<br/>
               }<br/>
            }<br/>
            finally<br/>
            {<br/>
               in.close();<br/>
               tOut.close();<br/>
            }<br/>
         }<br/>
         else<br/>
         {<br/>
            this.success = false;<br/>
            logger.error( "[Order:" + orderId + "] Error uploading file... (source: '" + source + "', target: '" + target<br/>
                  + "'): File already exists on SFTP-Server" );<br/>
         }<br/>
      }<br/>
      catch( IOException e )<br/>
      {<br/>
         this.success = false;<br/>
         logger.error( "[Order:" + orderId + "] Error uploading file... (source: '" + source + "', target: '" + target + "')", e );</div>

<div>      }<br/>
      catch( SftpException e )<br/>
      {<br/>
         this.success = false;<br/>
         logger.error( "[Order:" + orderId + "] Error uploading file... (source: '" + source + "', target: '" + target + "')", e );<br/>
      }<br/>
      catch( Exception e )<br/>
      {<br/>
         this.success = false;<br/>
         logger.error( "[Order:" + orderId + "] Error uploading file... (source: '" + source + "', target: '" + target + "')", e );<br/>
      }<br/>
   }</div>
</div>
</div>
</div></div></body></html>