builder: release-mozilla-release_firefox_macosx64_update_verify slave: bld-lion-r5-002 starttime: 1515091897.77 results: failure (2) revision: afa87f9be3a8852da3a30f286b15ae599c7874f6 ========= Started set props: master (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:37.765641) ========= master: http://buildbot-master85.bb.releng.scl3.mozilla.com:8001/ ========= Finished set props: master (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:37.766241) ========= ========= Started downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:37.766531) ========= ========= Finished downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:38.360509) ========= ========= Started 'rm -rf ...' (results: 0, elapsed: 1 secs) (at 2018-01-04 10:51:38.360891) ========= rm -rf scripts properties in dir /builds/slave/rel-m-rel_fx_m64_u_v-000000000/. (timeout 1200 secs) watching logfiles {} argv: ['rm', '-rf', 'scripts', 'properties'] environment: Apple_PubSub_Socket_Render=/tmp/launch-I5kKe0/Render CCACHE_COMPRESS=1 CCACHE_DIR=/builds/ccache CCACHE_UMASK=002 CHOWN_REVERT=~/bin/chown_revert CHOWN_ROOT=~/bin/chown_root DISPLAY=/tmp/launch-YU20d7/org.x:0 GIT_SHARE_BASE_DIR=/builds/git-shared HG_SHARE_BASE_DIR=/builds/hg-shared HOME=/Users/cltbld IDLEIZER_DISABLE_SHUTDOWN=true LC_ALL=C LOGNAME=cltbld MOZ_AUTOMATION=1 MOZ_CRASHREPORTER_NO_REPORT=1 MOZ_OBJDIR=obj-firefox PATH=/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin PWD=/builds/slave/rel-m-rel_fx_m64_u_v-000000000 RUNNER_CONFIG_CMD=/opt/runner/bin/python2.7 /opt/runner/bin/runner -c /opt/runner/runner.cfg SHELL=/bin/bash SSH_AUTH_SOCK=/tmp/launch-0qmWZe/Listeners TINDERBOX_OUTPUT=1 TMPDIR=/var/folders/kx/0sqd_gns68b5f5st5c6330lm00000w/T/ TOOLTOOL_CACHE=/builds/tooltool_cache TOOLTOOL_HOME=/builds TWISTD_LOG_PATH=/builds/slave/twistd.log USER=cltbld VERSIONER_PYTHON_PREFER_32_BIT=no VERSIONER_PYTHON_VERSION=2.7 __CF_USER_TEXT_ENCODING=0x1C:0:0 using PTY: False program finished with exit code 0 elapsedTime=1.962246 ========= master_lag: 0.04 ========= ========= Finished 'rm -rf ...' (results: 0, elapsed: 1 secs) (at 2018-01-04 10:51:40.360335) ========= ========= Started 'hg clone ...' (results: 0, elapsed: 15 secs) (at 2018-01-04 10:51:40.360662) ========= hg clone https://hg.mozilla.org/build/tools scripts in dir /builds/slave/rel-m-rel_fx_m64_u_v-000000000/. (timeout 1320 secs) watching logfiles {} argv: ['hg', 'clone', 'https://hg.mozilla.org/build/tools', 'scripts'] environment: Apple_PubSub_Socket_Render=/tmp/launch-I5kKe0/Render CCACHE_COMPRESS=1 CCACHE_DIR=/builds/ccache CCACHE_UMASK=002 CHOWN_REVERT=~/bin/chown_revert CHOWN_ROOT=~/bin/chown_root DISPLAY=/tmp/launch-YU20d7/org.x:0 GIT_SHARE_BASE_DIR=/builds/git-shared HG_SHARE_BASE_DIR=/builds/hg-shared HOME=/Users/cltbld IDLEIZER_DISABLE_SHUTDOWN=true LC_ALL=C LOGNAME=cltbld MOZ_AUTOMATION=1 MOZ_CRASHREPORTER_NO_REPORT=1 MOZ_OBJDIR=obj-firefox PATH=/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin PWD=/builds/slave/rel-m-rel_fx_m64_u_v-000000000 RUNNER_CONFIG_CMD=/opt/runner/bin/python2.7 /opt/runner/bin/runner -c /opt/runner/runner.cfg SHELL=/bin/bash SSH_AUTH_SOCK=/tmp/launch-0qmWZe/Listeners TINDERBOX_OUTPUT=1 TMPDIR=/var/folders/kx/0sqd_gns68b5f5st5c6330lm00000w/T/ TOOLTOOL_CACHE=/builds/tooltool_cache TOOLTOOL_HOME=/builds TWISTD_LOG_PATH=/builds/slave/twistd.log USER=cltbld VERSIONER_PYTHON_PREFER_32_BIT=no VERSIONER_PYTHON_VERSION=2.7 __CF_USER_TEXT_ENCODING=0x1C:0:0 using PTY: False warning: connecting to hg.mozilla.org using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info applying clone bundle from https://hg.cdn.mozilla.net/build/tools/801485fa899c236830bd6cfe52012163903ccdc7.zstd-max.hg warning: connecting to hg.cdn.mozilla.net using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info adding changesets adding manifests adding file changes added 8156 changesets with 18363 changes to 2169 files (+2 heads) finished applying clone bundle warning: connecting to hg.mozilla.org using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info searching for changes adding changesets adding manifests adding file changes added 17 changesets with 61 changes to 25 files new changesets 22f7fa347c61:021896b0f53d updating to branch default 1058 files updated, 0 files merged, 0 files removed, 0 files unresolved program finished with exit code 0 elapsedTime=15.368815 ========= master_lag: 0.01 ========= ========= Finished 'hg clone ...' (results: 0, elapsed: 15 secs) (at 2018-01-04 10:51:55.744107) ========= ========= Started 'hg update ...' (results: 0, elapsed: 1 secs) (at 2018-01-04 10:51:55.744530) ========= hg update -C -r FIREFOX_57_0_4_BUILD1_RUNTIME in dir /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts (timeout 1200 secs) watching logfiles {} argv: ['hg', 'update', '-C', '-r', u'FIREFOX_57_0_4_BUILD1_RUNTIME'] environment: Apple_PubSub_Socket_Render=/tmp/launch-I5kKe0/Render CCACHE_COMPRESS=1 CCACHE_DIR=/builds/ccache CCACHE_UMASK=002 CHOWN_REVERT=~/bin/chown_revert CHOWN_ROOT=~/bin/chown_root DISPLAY=/tmp/launch-YU20d7/org.x:0 GIT_SHARE_BASE_DIR=/builds/git-shared HG_SHARE_BASE_DIR=/builds/hg-shared HOME=/Users/cltbld IDLEIZER_DISABLE_SHUTDOWN=true LC_ALL=C LOGNAME=cltbld MOZ_AUTOMATION=1 MOZ_CRASHREPORTER_NO_REPORT=1 MOZ_OBJDIR=obj-firefox PATH=/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin PWD=/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts RUNNER_CONFIG_CMD=/opt/runner/bin/python2.7 /opt/runner/bin/runner -c /opt/runner/runner.cfg SHELL=/bin/bash SSH_AUTH_SOCK=/tmp/launch-0qmWZe/Listeners TINDERBOX_OUTPUT=1 TMPDIR=/var/folders/kx/0sqd_gns68b5f5st5c6330lm00000w/T/ TOOLTOOL_CACHE=/builds/tooltool_cache TOOLTOOL_HOME=/builds TWISTD_LOG_PATH=/builds/slave/twistd.log USER=cltbld VERSIONER_PYTHON_PREFER_32_BIT=no VERSIONER_PYTHON_VERSION=2.7 __CF_USER_TEXT_ENCODING=0x1C:0:0 using PTY: False 6 files updated, 0 files merged, 0 files removed, 0 files unresolved program finished with exit code 0 elapsedTime=1.237863 ========= master_lag: 0.02 ========= ========= Finished 'hg update ...' (results: 0, elapsed: 1 secs) (at 2018-01-04 10:51:56.998593) ========= ========= Started set props: script_repo_revision (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:56.998983) ========= hg id -i in dir /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts (timeout 1200 secs) watching logfiles {} argv: ['hg', 'id', '-i'] environment: Apple_PubSub_Socket_Render=/tmp/launch-I5kKe0/Render CCACHE_COMPRESS=1 CCACHE_DIR=/builds/ccache CCACHE_UMASK=002 CHOWN_REVERT=~/bin/chown_revert CHOWN_ROOT=~/bin/chown_root DISPLAY=/tmp/launch-YU20d7/org.x:0 GIT_SHARE_BASE_DIR=/builds/git-shared HG_SHARE_BASE_DIR=/builds/hg-shared HOME=/Users/cltbld IDLEIZER_DISABLE_SHUTDOWN=true LC_ALL=C LOGNAME=cltbld MOZ_AUTOMATION=1 MOZ_CRASHREPORTER_NO_REPORT=1 MOZ_OBJDIR=obj-firefox PATH=/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin PWD=/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts RUNNER_CONFIG_CMD=/opt/runner/bin/python2.7 /opt/runner/bin/runner -c /opt/runner/runner.cfg SHELL=/bin/bash SSH_AUTH_SOCK=/tmp/launch-0qmWZe/Listeners TINDERBOX_OUTPUT=1 TMPDIR=/var/folders/kx/0sqd_gns68b5f5st5c6330lm00000w/T/ TOOLTOOL_CACHE=/builds/tooltool_cache TOOLTOOL_HOME=/builds TWISTD_LOG_PATH=/builds/slave/twistd.log USER=cltbld VERSIONER_PYTHON_PREFER_32_BIT=no VERSIONER_PYTHON_VERSION=2.7 __CF_USER_TEXT_ENCODING=0x1C:0:0 using PTY: False 0445994a8de7 program finished with exit code 0 elapsedTime=0.291296 script_repo_revision: '0445994a8de7' ========= master_lag: 0.02 ========= ========= Finished set props: script_repo_revision (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:57.307061) ========= ========= Started 'bash scripts/scripts/release/updates/chunked-verify.sh' failed (results: 2, elapsed: 0 secs) (at 2018-01-04 10:51:57.307431) ========= bash scripts/scripts/release/updates/chunked-verify.sh in dir /builds/slave/rel-m-rel_fx_m64_u_v-000000000/. (timeout 1200 secs) watching logfiles {} argv: ['bash', 'scripts/scripts/release/updates/chunked-verify.sh'] environment: Apple_PubSub_Socket_Render=/tmp/launch-I5kKe0/Render CCACHE_COMPRESS=1 CCACHE_DIR=/builds/ccache CCACHE_UMASK=002 CHOWN_REVERT=~/bin/chown_revert CHOWN_ROOT=~/bin/chown_root DISPLAY=/tmp/launch-YU20d7/org.x:0 GIT_SHARE_BASE_DIR=/builds/git-shared HG_SHARE_BASE_DIR=/builds/hg-shared HOME=/Users/cltbld IDLEIZER_DISABLE_SHUTDOWN=true LC_ALL=C LOGNAME=cltbld MOZ_AUTOMATION=1 MOZ_CRASHREPORTER_NO_REPORT=1 MOZ_OBJDIR=obj-firefox PATH=/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin PROPERTIES_FILE=/builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json PWD=/builds/slave/rel-m-rel_fx_m64_u_v-000000000 RUNNER_CONFIG_CMD=/opt/runner/bin/python2.7 /opt/runner/bin/runner -c /opt/runner/runner.cfg SHELL=/bin/bash SSH_AUTH_SOCK=/tmp/launch-0qmWZe/Listeners TINDERBOX_OUTPUT=1 TMPDIR=/var/folders/kx/0sqd_gns68b5f5st5c6330lm00000w/T/ TOOLTOOL_CACHE=/builds/tooltool_cache TOOLTOOL_HOME=/builds TWISTD_LOG_PATH=/builds/slave/twistd.log USER=cltbld VERSIONER_PYTHON_PREFER_32_BIT=no VERSIONER_PYTHON_VERSION=2.7 __CF_USER_TEXT_ENCODING=0x1C:0:0 using PTY: False + set -o pipefail ++ dirname scripts/scripts/release/updates/chunked-verify.sh + pushd scripts/scripts/release/updates ++ pwd + MY_DIR=/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates + popd + SCRIPTS_DIR=/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../.. + PYTHON=/tools/python/bin/python + '[' '!' -x /tools/python/bin/python ']' + PYTHON=python + JSONTOOL='python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py' ++ pwd + workdir=/builds/slave/rel-m-rel_fx_m64_u_v-000000000 + platform= + configDict= + chunks= + thisChunk= + channel= + '[' -n /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json -a -f /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json ']' + python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.NO_BBCONFIG /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json 1 ++ python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.NO_BBCONFIG /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json + NO_BBCONFIG=1 + python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.CHANNEL /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json Traceback (most recent call last): File "/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py", line 25, in v = v[k] KeyError: 'CHANNEL' + python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.VERIFY_CONFIG /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json release-firefox-macosx64.cfg ++ python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.VERIFY_CONFIG /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json + VERIFY_CONFIG=release-firefox-macosx64.cfg + python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.TOTAL_CHUNKS /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json 12 ++ python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.TOTAL_CHUNKS /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json + chunks=12 + python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.THIS_CHUNK /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json 12 ++ python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/utils/jsontool.py -k properties.THIS_CHUNK /builds/slave/rel-m-rel_fx_m64_u_v-000000000/buildprops.json + thisChunk=12 + '[' -z 1 -a -z '' ']' +++ cd /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../.. +++ pwd ++ basename /builds/slave/rel-m-rel_fx_m64_u_v-000000000 + SLAVEBUILDDIR=rel-m-rel_fx_m64_u_v-000000000 + '[' -z 1 ']' + python -u /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../buildfarm/maintenance/purge_builds.py -s 16 -n info -n 'rel-*' -n 'tb-rel-*' -n rel-m-rel_fx_m64_u_v-000000000 859.11 GB of space available + '[' -z release-firefox-macosx64.cfg -a -n 1 ']' + '[' -z 1 ']' + '[' -n '' ']' + EXTRA_PARAMS= + python /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/chunked-verify.py --chunks 12 --this-chunk 12 --verify-config release-firefox-macosx64.cfg + tee /builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../../verify_log.txt Traceback (most recent call last): File "/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/chunked-verify.py", line 78, in verifyConfig.read(path.join(UPDATE_VERIFY_DIR, verifyConfigFile)) File "/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../lib/python/release/updates/verify.py", line 84, in read self._addEntry(self._parseLine(line), first) File "/builds/slave/rel-m-rel_fx_m64_u_v-000000000/scripts/scripts/release/updates/../../../lib/python/release/updates/verify.py", line 54, in _parseLine "Unknown key '%s' found on line:\n%s" % (m["key"], line)) release.updates.verify.UpdateVerifyError: Unknown key 'updater_package' found on line: release="57.0.3" product="Firefox" platform="Darwin_x86_64-gcc3-u-i386-x86_64" build_id="20171226083017" locales="ach af an ar as ast az be bg bn-BD bn-IN br bs ca cak cs cy da de dsb el en-GB en-US en-ZA eo es-AR es-CL es-ES es-MX et eu fa ff fi fr fy-NL ga-IE gd gl gn gu-IN he hi-IN hr hsb hu hy-AM id is it ja-JP-mac ka kab kk km kn ko lij lt lv mai mk ml mr ms my nb-NO nl nn-NO or pa-IN pl pt-BR pt-PT rm ro ru si sk sl son sq sr sv-SE ta te th tr uk ur uz vi xh zh-CN zh-TW" channel="release-localtest" patch_types="complete partial" from="/firefox/releases/57.0.3/mac/%locale%/Firefox 57.0.3.dmg" aus_server="https://aus5.mozilla.org" ftp_server_from="https://archive.mozilla.org/pub" ftp_server_to="https://archive.mozilla.org/pub" to="/firefox/candidates/57.0.4-candidates/build1/mac/%locale%/Firefox 57.0.4.dmg" to_build_id="20180103231032" to_display_version="57.0.4" to_app_version="57.0.4" updater_package="/firefox/releases/57.0.3/mac/%locale%/Firefox 57.0.3.dmg" program finished with exit code 1 elapsedTime=0.594454 ========= master_lag: 0.02 ========= ========= Finished 'bash scripts/scripts/release/updates/chunked-verify.sh' failed (results: 2, elapsed: 0 secs) (at 2018-01-04 10:51:57.924165) ========= ========= Started no change (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:57.925436) ========= bash -c 'for file in `ls -1`; do cat $file; done' in dir /builds/slave/rel-m-rel_fx_m64_u_v-000000000/properties (timeout 1200 secs) watching logfiles {} argv: ['bash', '-c', 'for file in `ls -1`; do cat $file; done'] environment: Apple_PubSub_Socket_Render=/tmp/launch-I5kKe0/Render CCACHE_COMPRESS=1 CCACHE_DIR=/builds/ccache CCACHE_UMASK=002 CHOWN_REVERT=~/bin/chown_revert CHOWN_ROOT=~/bin/chown_root DISPLAY=/tmp/launch-YU20d7/org.x:0 GIT_SHARE_BASE_DIR=/builds/git-shared HG_SHARE_BASE_DIR=/builds/hg-shared HOME=/Users/cltbld IDLEIZER_DISABLE_SHUTDOWN=true LC_ALL=C LOGNAME=cltbld MOZ_AUTOMATION=1 MOZ_CRASHREPORTER_NO_REPORT=1 MOZ_OBJDIR=obj-firefox PATH=/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin PWD=/builds/slave/rel-m-rel_fx_m64_u_v-000000000/properties RUNNER_CONFIG_CMD=/opt/runner/bin/python2.7 /opt/runner/bin/runner -c /opt/runner/runner.cfg SHELL=/bin/bash SSH_AUTH_SOCK=/tmp/launch-0qmWZe/Listeners TINDERBOX_OUTPUT=1 TMPDIR=/var/folders/kx/0sqd_gns68b5f5st5c6330lm00000w/T/ TOOLTOOL_CACHE=/builds/tooltool_cache TOOLTOOL_HOME=/builds TWISTD_LOG_PATH=/builds/slave/twistd.log USER=cltbld VERSIONER_PYTHON_PREFER_32_BIT=no VERSIONER_PYTHON_VERSION=2.7 __CF_USER_TEXT_ENCODING=0x1C:0:0 using PTY: False program finished with exit code 0 elapsedTime=0.008990 ========= master_lag: 0.02 ========= ========= Finished no change (results: 0, elapsed: 0 secs) (at 2018-01-04 10:51:57.950307) ========= ========= Total master_lag: 0.12 =========