Multiple file support with scp

Paramiko doesn’t provide a scp implementation, so I’ve been using my own for a while.

http://blogs.sun.com/janp/entry/how_the_scp_protocol_works (link now unfortunately dead) provides good documentation about the scp protocol, but it missed out on one detail I needed — how to send more than one file in a given session. In the end I implemented a simple scp logger to see what the protocol was doing during the copying of files. My logger said this:

>>> New command invocation: /usr/bin/scp -d -t /tmp
O: \0
I: C0644 21 a\n
O: \0
I: file a file a file a\n\0
O: \0
I: C0644 21 b\n
O: \0
I: file b file b file b\n\0
O: \0
>>>stdin closed
>>> stdout closed
>>> stderr closed

It turns out its important to wait for those zeros by the way. So, here’s my implementation of the protocol to send more than one file. Turning this into paramiko code is left as an exercise for the reader.

#!/usr/bin/python

import fcntl
import os
import select
import string
import subprocess
import sys
import traceback

def printable(s):
  out = ''

  for c in s:
    if c == '\n':
      out += '\\n'
    elif c in string.printable:
      out += c
    else:
      out += '\\%d' % ord(c)

  return out

try:
  dialog = ['C0644 21 c\n',
            'file c file c file c\n\0',
            'C0644 21 d\n',
            'file d file d file d\n\0']

  proc = subprocess.Popen(['scp', '-v', '-d', '-t', '/tmp'],
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)

  r = [proc.stdout, proc.stderr]
  w = []
  e = [proc.stdout, proc.stderr]

  fl = fcntl.fcntl(proc.stdout, fcntl.F_GETFL)
  fcntl.fcntl(proc.stdout, fcntl.F_SETFL, fl | os.O_NONBLOCK)
  fl = fcntl.fcntl(proc.stderr, fcntl.F_GETFL)
  fcntl.fcntl(proc.stderr, fcntl.F_SETFL, fl | os.O_NONBLOCK)

  stdin_closed = False
  while proc.returncode is None:
    (readable, _, errorable) = select.select(r, w, e)

    for flo in readable:
      if flo == proc.stdout:
        d = os.read(proc.stdout.fileno(), 1024)
        if len(d) > 0:
          sys.stdout.write('O: %s\n' % printable(d))

          if len(dialog) > 0:
            sys.stdout.write('I: %s\n' % printable(dialog[0]))
            os.write(proc.stdin.fileno(), dialog[0])
            dialog = dialog[1:]

          if len(dialog) == 0 and not stdin_closed:
            sys.stdout.write('>>> stdin closed\n')
            proc.stdin.close()
            stdin_closed = True

        else:
          sys.stdout.write('>>> stdout closed\n')
          r.remove(proc.stdout)
          e.remove(proc.stdout)

      elif flo == proc.stderr:
        d = os.read(proc.stderr.fileno(), 1024)
        if len(d) > 0:
          sys.stdout.write('E: %s\n' % printable(d))
        else:
          sys.stdout.write('>>> stderr closed\n')
          r.remove(proc.stderr)
          e.remove(proc.stderr)

      else:
        sys.stdout.write('>>> Unknown readable: %s: %s\n'
                         %(repr(flo), flo.read()))

    for flo in errorable:
      sys.stdout.write('>>> Error on %s\n' % repr(flo))
      r.remove(flo)
      e.remove(flo)

    proc.poll()

  print '#: %s' % proc.returncode

except:
  exc = sys.exc_info()
  for tb in traceback.format_exception(exc[0], exc[1], exc[2]):
    print tb
    del tb