builder: fx-team-linux64
slave: bld-linux64-spot-094
starttime: 1469590471.31
results: exception (4)
buildid: 20160726203413
builduid: 61b716ac94bc4295b197f9b9c8593644
revision: 820cfffb1fe2b87a09d1c57d06d02107be8ac778

========= Started set props: master (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:31.309570) =========
master: http://buildbot-master94.bb.releng.use1.mozilla.com:8001/
========= Finished set props: master (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:31.310001) =========

========= Started downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:31.310279) =========
========= Finished downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:31.326420) =========

========= Started 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:31.326714) =========
rm -rf scripts properties
 in dir /builds/slave/fx-team-l64-000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-rf', 'scripts', 'properties']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-094.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003688
========= master_lag: 0.05 =========
========= Finished 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:31.378909) =========

========= Started 'bash -c ...' (results: 0, elapsed: 2 secs) (at 2016-07-26 20:34:31.379252) =========
bash -c 'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo integration/fx-team --rev 820cfffb1fe2b87a09d1c57d06d02107be8ac778 --destination scripts --debug'
 in dir /builds/slave/fx-team-l64-000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', u'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo integration/fx-team --rev 820cfffb1fe2b87a09d1c57d06d02107be8ac778 --destination scripts --debug']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-094.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
2016-07-26 20:34:31,419 truncating revision to first 12 chars
2016-07-26 20:34:31,420 Setting DEBUG logging.
2016-07-26 20:34:31,420 attempt 1/10
2016-07-26 20:34:31,420 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/integration/fx-team/820cfffb1fe2?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-07-26 20:34:33,358 unpacking tar archive at: fx-team-820cfffb1fe2/testing/mozharness/
program finished with exit code 0
elapsedTime=2.540718
========= master_lag: 0.01 =========
========= Finished 'bash -c ...' (results: 0, elapsed: 2 secs) (at 2016-07-26 20:34:33.931550) =========

========= Started set props: script_repo_revision (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:33.931865) =========
script_repo_revision: 820cfffb1fe2b87a09d1c57d06d02107be8ac778
========= Finished set props: script_repo_revision (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:33.932270) =========

========= Started downloading to oauth.txt (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:33.932532) =========
========= Finished downloading to oauth.txt (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:33.944733) =========

========= Started remove old nonce (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:33.945010) =========
rm -f nonce
 in dir /builds/slave/fx-team-l64-000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-f', 'nonce']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-094.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003469
========= master_lag: 0.05 =========
========= Finished remove old nonce (results: 0, elapsed: 0 secs) (at 2016-07-26 20:34:33.994771) =========

========= Started download_token exception (results: 4, elapsed: 45 mins, 20 secs) (at 2016-07-26 20:34:33.995060) =========
Slave: bld-linux64-spot-094
IP: 10.134.55.132
Duration: 25200
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing1.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing3.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing5.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://signing4.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing7.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing2.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required
URI: https://mac-v2-signing6.srv.releng.scl3.mozilla.com:9110/token
<buildbotcustom.steps.signing.SigningServerAuthenication instance at 0x9e1bb90>: token generation failed, error message: 401 Authorization Required

<style type="text/css">
    p.error {
      color: red;
      font-family: Verdana, Arial, helvetica, sans-serif;
      font-weight: bold;
    }

    div {
      font-family: Verdana, Arial, helvetica, sans-serif;
    }

    div.stackTrace {
    }

    div.frame {
      padding: 1em;
      background: white;
      border-bottom: thin black dashed;
    }

    div.firstFrame {
      padding: 1em;
      background: white;
      border-top: thin black dashed;
      border-bottom: thin black dashed;
    }

    div.location {
    }

    div.snippet {
      margin-bottom: 0.5em;
      margin-left: 1em;
      background: #FFFFDD;
    }

    div.snippetHighlightLine {
      color: red;
    }

    span.code {
      font-family: "Courier New", courier, monotype;
    }

    span.function {
      font-weight: bold;
      font-family: "Courier New", courier, monotype;
    }

    table.variables {
      border-collapse: collapse;
      margin-left: 1em;
    }

    td.varName {
      vertical-align: top;
      font-weight: bold;
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    td.varValue {
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    div.variables {
      margin-bottom: 0.5em;
    }

    span.heading {
      font-weight: bold;
    }

    div.dict {
      background: #cccc99;
      padding: 2px;
      float: left;
    }

    td.dictKey {
      background: #ffff99;
      font-weight: bold;
    }

    td.dictValue {
      background: #ffff99;
    }

    div.list {
      background: #7777cc;
      padding: 2px;
      float: left;
    }

    div.listItem {
      background: #9999ff;
    }

    div.instance {
      background: #cc7777;
      padding: 2px;
      float: left;
    }

    span.instanceName {
      font-weight: bold;
      display: block;
    }

    span.instanceRepr {
      background: #ff9999;
      font-family: "Courier New", courier, monotype;
    }

    div.function {
      background: orange;
      font-weight: bold;
      float: left;
    }
</style>
<a href="#tbend">
<p class="error">&lt;type 'exceptions.Exception'&gt;: No more signing servers to try.</p>
</a><div class="stackTrace"></div><a name="tbend"> </a>
<p class="error">&lt;type 'exceptions.Exception'&gt;: No more signing servers to try.</p>
Traceback (most recent call last):
Failure: exceptions.Exception: No more signing servers to try.
========= Finished download_token exception (results: 4, elapsed: 45 mins, 20 secs) (at 2016-07-26 21:19:54.888092) =========

========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Started no change (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:54.889244) =========
bash -c 'for file in `ls -1`; do cat $file; done'
 in dir /builds/slave/fx-team-l64-000000000000000000/properties (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', 'for file in `ls -1`; do cat $file; done']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-094.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-000000000000000000/properties
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.005482

========= master_lag: 0.05 =========
========= Finished no change (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:54.940197) =========

========= Started 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:54.940502) =========
rm -f oauth.txt
 in dir /builds/slave/fx-team-l64-000000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-f', 'oauth.txt']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-094.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-000000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003516
========= master_lag: 0.05 =========
========= Finished 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2016-07-26 21:19:54.990799) =========

========= Started reboot skipped (results: 3, elapsed: 0 secs) (at 2016-07-26 21:19:54.991129) =========
========= Finished reboot skipped (results: 3, elapsed: 0 secs) (at 2016-07-26 21:19:54.991381) =========

========= Total master_lag: 0.20 =========