int-test mqtt <-> prediction
This commit is contained in:
naumueller
@@ -0,0 +1,27 @@
|
||||
// Copyright (c) 2009 The Chromium Authors. All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@@ -0,0 +1,13 @@
|
||||
Name: depot_tools
|
||||
Short Name: depot_tools
|
||||
URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git
|
||||
Date: 21 Sep 2022
|
||||
Revision: 08bb5c487f
|
||||
License: BSD
|
||||
License File: LICENSE
|
||||
|
||||
Description:
|
||||
Select tools extracted from depot_tools.
|
||||
|
||||
Local Modifications:
|
||||
- Remove dependency on vpython.
|
||||
@@ -0,0 +1,645 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""Download files from Google Storage based on SHA1 sums."""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import hashlib
|
||||
import optparse
|
||||
import os
|
||||
|
||||
try:
|
||||
import Queue as queue
|
||||
except ImportError: # For Py3 compatibility
|
||||
import queue
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
import subprocess2
|
||||
|
||||
|
||||
# Env vars that tempdir can be gotten from; minimally, this
|
||||
# needs to match python's tempfile module and match normal
|
||||
# unix standards.
|
||||
_TEMPDIR_ENV_VARS = ('TMPDIR', 'TEMP', 'TMP')
|
||||
|
||||
GSUTIL_DEFAULT_PATH = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
|
||||
# Maps sys.platform to what we actually want to call them.
|
||||
PLATFORM_MAPPING = {
|
||||
'cygwin': 'win',
|
||||
'darwin': 'mac',
|
||||
'linux': 'linux', # Python 3.3+.
|
||||
'linux2': 'linux', # Python < 3.3 uses "linux2" / "linux3".
|
||||
'win32': 'win',
|
||||
'aix6': 'aix',
|
||||
'aix7': 'aix',
|
||||
}
|
||||
|
||||
if sys.version_info.major == 2:
|
||||
# pylint: disable=redefined-builtin
|
||||
class FileNotFoundError(IOError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidFileError(IOError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidPlatformError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def GetNormalizedPlatform():
|
||||
"""Returns the result of sys.platform accounting for cygwin.
|
||||
Under cygwin, this will always return "win32" like the native Python."""
|
||||
if sys.platform == 'cygwin':
|
||||
return 'win32'
|
||||
return sys.platform
|
||||
|
||||
# Common utilities
|
||||
class Gsutil(object):
|
||||
"""Call gsutil with some predefined settings. This is a convenience object,
|
||||
and is also immutable.
|
||||
|
||||
HACK: This object is used directly by the external script
|
||||
`<depot_tools>/win_toolchain/get_toolchain_if_necessary.py`
|
||||
"""
|
||||
|
||||
MAX_TRIES = 5
|
||||
RETRY_BASE_DELAY = 5.0
|
||||
RETRY_DELAY_MULTIPLE = 1.3
|
||||
|
||||
def __init__(self, path, boto_path=None):
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError('GSUtil not found in %s' % path)
|
||||
self.path = path
|
||||
self.boto_path = boto_path
|
||||
|
||||
def get_sub_env(self):
|
||||
env = os.environ.copy()
|
||||
if self.boto_path == os.devnull:
|
||||
env['AWS_CREDENTIAL_FILE'] = ''
|
||||
env['BOTO_CONFIG'] = ''
|
||||
elif self.boto_path:
|
||||
env['AWS_CREDENTIAL_FILE'] = self.boto_path
|
||||
env['BOTO_CONFIG'] = self.boto_path
|
||||
|
||||
if PLATFORM_MAPPING[sys.platform] != 'win':
|
||||
env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS)
|
||||
|
||||
return env
|
||||
|
||||
def call(self, *args):
|
||||
cmd = [sys.executable, self.path]
|
||||
cmd.extend(args)
|
||||
return subprocess2.call(cmd, env=self.get_sub_env())
|
||||
|
||||
def check_call(self, *args):
|
||||
cmd = [sys.executable, self.path]
|
||||
cmd.extend(args)
|
||||
((out, err), code) = subprocess2.communicate(
|
||||
cmd,
|
||||
stdout=subprocess2.PIPE,
|
||||
stderr=subprocess2.PIPE,
|
||||
env=self.get_sub_env())
|
||||
|
||||
out = out.decode('utf-8', 'replace')
|
||||
err = err.decode('utf-8', 'replace')
|
||||
|
||||
# Parse output.
|
||||
status_code_match = re.search('status=([0-9]+)', err)
|
||||
if status_code_match:
|
||||
return (int(status_code_match.group(1)), out, err)
|
||||
if ('You are attempting to access protected data with '
|
||||
'no configured credentials.' in err):
|
||||
return (403, out, err)
|
||||
if 'matched no objects' in err:
|
||||
return (404, out, err)
|
||||
return (code, out, err)
|
||||
|
||||
def check_call_with_retries(self, *args):
|
||||
delay = self.RETRY_BASE_DELAY
|
||||
for i in range(self.MAX_TRIES):
|
||||
code, out, err = self.check_call(*args)
|
||||
if not code or i == self.MAX_TRIES - 1:
|
||||
break
|
||||
|
||||
time.sleep(delay)
|
||||
delay *= self.RETRY_DELAY_MULTIPLE
|
||||
|
||||
return code, out, err
|
||||
|
||||
|
||||
def check_platform(target):
|
||||
"""Checks if any parent directory of target matches (win|mac|linux)."""
|
||||
assert os.path.isabs(target)
|
||||
root, target_name = os.path.split(target)
|
||||
if not target_name:
|
||||
return None
|
||||
if target_name in ('linux', 'mac', 'win'):
|
||||
return target_name
|
||||
return check_platform(root)
|
||||
|
||||
|
||||
def get_sha1(filename):
|
||||
sha1 = hashlib.sha1()
|
||||
with open(filename, 'rb') as f:
|
||||
while True:
|
||||
# Read in 1mb chunks, so it doesn't all have to be loaded into memory.
|
||||
chunk = f.read(1024*1024)
|
||||
if not chunk:
|
||||
break
|
||||
sha1.update(chunk)
|
||||
return sha1.hexdigest()
|
||||
|
||||
|
||||
# Download-specific code starts here
|
||||
|
||||
def enumerate_input(input_filename, directory, recursive, ignore_errors, output,
|
||||
sha1_file, auto_platform):
|
||||
if sha1_file:
|
||||
if not os.path.exists(input_filename):
|
||||
if not ignore_errors:
|
||||
raise FileNotFoundError(
|
||||
'{} not found when attempting enumerate files to download.'.format(
|
||||
input_filename))
|
||||
print('%s not found.' % input_filename, file=sys.stderr)
|
||||
with open(input_filename, 'rb') as f:
|
||||
sha1_match = re.match(b'^([A-Za-z0-9]{40})$', f.read(1024).rstrip())
|
||||
if sha1_match:
|
||||
yield (sha1_match.groups(1)[0].decode('utf-8'), output)
|
||||
return
|
||||
if not ignore_errors:
|
||||
raise InvalidFileError('No sha1 sum found in %s.' % input_filename)
|
||||
print('No sha1 sum found in %s.' % input_filename, file=sys.stderr)
|
||||
return
|
||||
|
||||
if not directory:
|
||||
yield (input_filename, output)
|
||||
return
|
||||
|
||||
for root, dirs, files in os.walk(input_filename):
|
||||
if not recursive:
|
||||
for item in dirs[:]:
|
||||
dirs.remove(item)
|
||||
else:
|
||||
for exclude in ['.svn', '.git']:
|
||||
if exclude in dirs:
|
||||
dirs.remove(exclude)
|
||||
for filename in files:
|
||||
full_path = os.path.join(root, filename)
|
||||
if full_path.endswith('.sha1'):
|
||||
if auto_platform:
|
||||
# Skip if the platform does not match.
|
||||
target_platform = check_platform(os.path.abspath(full_path))
|
||||
if not target_platform:
|
||||
err = ('--auto_platform passed in but no platform name found in '
|
||||
'the path of %s' % full_path)
|
||||
if not ignore_errors:
|
||||
raise InvalidFileError(err)
|
||||
print(err, file=sys.stderr)
|
||||
continue
|
||||
current_platform = PLATFORM_MAPPING[sys.platform]
|
||||
if current_platform != target_platform:
|
||||
continue
|
||||
|
||||
with open(full_path, 'rb') as f:
|
||||
sha1_match = re.match(b'^([A-Za-z0-9]{40})$', f.read(1024).rstrip())
|
||||
if sha1_match:
|
||||
yield (
|
||||
sha1_match.groups(1)[0].decode('utf-8'),
|
||||
full_path.replace('.sha1', '')
|
||||
)
|
||||
else:
|
||||
if not ignore_errors:
|
||||
raise InvalidFileError('No sha1 sum found in %s.' % filename)
|
||||
print('No sha1 sum found in %s.' % filename, file=sys.stderr)
|
||||
|
||||
|
||||
def _validate_tar_file(tar, prefix):
|
||||
def _validate(tarinfo):
|
||||
"""Returns false if the tarinfo is something we explicitly forbid."""
|
||||
if tarinfo.issym() or tarinfo.islnk():
|
||||
return False
|
||||
if ('../' in tarinfo.name or
|
||||
'..\\' in tarinfo.name or
|
||||
not tarinfo.name.startswith(prefix)):
|
||||
return False
|
||||
return True
|
||||
return all(map(_validate, tar.getmembers()))
|
||||
|
||||
def _downloader_worker_thread(thread_num, q, force, base_url,
|
||||
gsutil, out_q, ret_codes, verbose, extract,
|
||||
delete=True):
|
||||
while True:
|
||||
input_sha1_sum, output_filename = q.get()
|
||||
if input_sha1_sum is None:
|
||||
return
|
||||
extract_dir = None
|
||||
if extract:
|
||||
if not output_filename.endswith('.tar.gz'):
|
||||
out_q.put('%d> Error: %s is not a tar.gz archive.' % (
|
||||
thread_num, output_filename))
|
||||
ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
|
||||
continue
|
||||
extract_dir = output_filename[:-len('.tar.gz')]
|
||||
if os.path.exists(output_filename) and not force:
|
||||
skip = get_sha1(output_filename) == input_sha1_sum
|
||||
if extract:
|
||||
# Additional condition for extract:
|
||||
# 1) extract_dir must exist
|
||||
# 2) .tmp flag file mustn't exist
|
||||
if not os.path.exists(extract_dir):
|
||||
out_q.put('%d> Extract dir %s does not exist, re-downloading...' %
|
||||
(thread_num, extract_dir))
|
||||
skip = False
|
||||
# .tmp file is created just before extraction and removed just after
|
||||
# extraction. If such file exists, it means the process was terminated
|
||||
# mid-extraction and therefore needs to be extracted again.
|
||||
elif os.path.exists(extract_dir + '.tmp'):
|
||||
out_q.put('%d> Detected tmp flag file for %s, '
|
||||
're-downloading...' % (thread_num, output_filename))
|
||||
skip = False
|
||||
if skip:
|
||||
continue
|
||||
|
||||
file_url = '%s/%s' % (base_url, input_sha1_sum)
|
||||
|
||||
try:
|
||||
if delete:
|
||||
os.remove(output_filename) # Delete the file if it exists already.
|
||||
except OSError:
|
||||
if os.path.exists(output_filename):
|
||||
out_q.put('%d> Warning: deleting %s failed.' % (
|
||||
thread_num, output_filename))
|
||||
if verbose:
|
||||
out_q.put('%d> Downloading %s@%s...' % (
|
||||
thread_num, output_filename, input_sha1_sum))
|
||||
code, _, err = gsutil.check_call('cp', file_url, output_filename)
|
||||
if code != 0:
|
||||
if code == 404:
|
||||
out_q.put('%d> File %s for %s does not exist, skipping.' % (
|
||||
thread_num, file_url, output_filename))
|
||||
ret_codes.put((1, 'File %s for %s does not exist.' % (
|
||||
file_url, output_filename)))
|
||||
elif code == 401:
|
||||
out_q.put(
|
||||
"""%d> Failed to fetch file %s for %s due to unauthorized access,
|
||||
skipping. Try running `gsutil.py config` and pass 0 if you don't
|
||||
know your project id.""" % (thread_num, file_url, output_filename))
|
||||
ret_codes.put(
|
||||
(1, 'Failed to fetch file %s for %s due to unauthorized access.' %
|
||||
(file_url, output_filename)))
|
||||
else:
|
||||
# Other error, probably auth related (bad ~/.boto, etc).
|
||||
out_q.put('%d> Failed to fetch file %s for %s, skipping. [Err: %s]' %
|
||||
(thread_num, file_url, output_filename, err))
|
||||
ret_codes.put((code, 'Failed to fetch file %s for %s. [Err: %s]' %
|
||||
(file_url, output_filename, err)))
|
||||
continue
|
||||
|
||||
remote_sha1 = get_sha1(output_filename)
|
||||
if remote_sha1 != input_sha1_sum:
|
||||
msg = ('%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' %
|
||||
(thread_num, remote_sha1, input_sha1_sum))
|
||||
out_q.put(msg)
|
||||
ret_codes.put((20, msg))
|
||||
continue
|
||||
|
||||
if extract:
|
||||
if not tarfile.is_tarfile(output_filename):
|
||||
out_q.put('%d> Error: %s is not a tar.gz archive.' % (
|
||||
thread_num, output_filename))
|
||||
ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
|
||||
continue
|
||||
with tarfile.open(output_filename, 'r:gz') as tar:
|
||||
dirname = os.path.dirname(os.path.abspath(output_filename))
|
||||
# If there are long paths inside the tarball we can get extraction
|
||||
# errors on windows due to the 260 path length limit (this includes
|
||||
# pwd). Use the extended path syntax.
|
||||
if sys.platform == 'win32':
|
||||
dirname = '\\\\?\\%s' % dirname
|
||||
if not _validate_tar_file(tar, os.path.basename(extract_dir)):
|
||||
out_q.put('%d> Error: %s contains files outside %s.' % (
|
||||
thread_num, output_filename, extract_dir))
|
||||
ret_codes.put((1, '%s contains invalid entries.' % (output_filename)))
|
||||
continue
|
||||
if os.path.exists(extract_dir):
|
||||
try:
|
||||
shutil.rmtree(extract_dir)
|
||||
out_q.put('%d> Removed %s...' % (thread_num, extract_dir))
|
||||
except OSError:
|
||||
out_q.put('%d> Warning: Can\'t delete: %s' % (
|
||||
thread_num, extract_dir))
|
||||
ret_codes.put((1, 'Can\'t delete %s.' % (extract_dir)))
|
||||
continue
|
||||
out_q.put('%d> Extracting %d entries from %s to %s' %
|
||||
(thread_num, len(tar.getmembers()),output_filename,
|
||||
extract_dir))
|
||||
with open(extract_dir + '.tmp', 'a'):
|
||||
tar.extractall(path=dirname)
|
||||
os.remove(extract_dir + '.tmp')
|
||||
# Set executable bit.
|
||||
if sys.platform == 'cygwin':
|
||||
# Under cygwin, mark all files as executable. The executable flag in
|
||||
# Google Storage will not be set when uploading from Windows, so if
|
||||
# this script is running under cygwin and we're downloading an
|
||||
# executable, it will be unrunnable from inside cygwin without this.
|
||||
st = os.stat(output_filename)
|
||||
os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
|
||||
elif sys.platform != 'win32':
|
||||
# On non-Windows platforms, key off of the custom header
|
||||
# "x-goog-meta-executable".
|
||||
code, out, err = gsutil.check_call('stat', file_url)
|
||||
if code != 0:
|
||||
out_q.put('%d> %s' % (thread_num, err))
|
||||
ret_codes.put((code, err))
|
||||
elif re.search(r'executable:\s*1', out):
|
||||
st = os.stat(output_filename)
|
||||
os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
|
||||
|
||||
|
||||
class PrinterThread(threading.Thread):
|
||||
def __init__(self, output_queue):
|
||||
super(PrinterThread, self).__init__()
|
||||
self.output_queue = output_queue
|
||||
self.did_print_anything = False
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
line = self.output_queue.get()
|
||||
# It's plausible we want to print empty lines: Explicit `is None`.
|
||||
if line is None:
|
||||
break
|
||||
self.did_print_anything = True
|
||||
print(line)
|
||||
|
||||
|
||||
def _data_exists(input_sha1_sum, output_filename, extract):
|
||||
"""Returns True if the data exists locally and matches the sha1.
|
||||
|
||||
This conservatively returns False for error cases.
|
||||
|
||||
Args:
|
||||
input_sha1_sum: Expected sha1 stored on disk.
|
||||
output_filename: The file to potentially download later. Its sha1 will be
|
||||
compared to input_sha1_sum.
|
||||
extract: Whether or not a downloaded file should be extracted. If the file
|
||||
is not extracted, this just compares the sha1 of the file. If the file
|
||||
is to be extracted, this only compares the sha1 of the target archive if
|
||||
the target directory already exists. The content of the target directory
|
||||
is not checked.
|
||||
"""
|
||||
extract_dir = None
|
||||
if extract:
|
||||
if not output_filename.endswith('.tar.gz'):
|
||||
# This will cause an error later. Conservativly return False to not bail
|
||||
# out too early.
|
||||
return False
|
||||
extract_dir = output_filename[:-len('.tar.gz')]
|
||||
if os.path.exists(output_filename):
|
||||
if not extract or os.path.exists(extract_dir):
|
||||
if get_sha1(output_filename) == input_sha1_sum:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def download_from_google_storage(
|
||||
input_filename, base_url, gsutil, num_threads, directory, recursive,
|
||||
force, output, ignore_errors, sha1_file, verbose, auto_platform, extract):
|
||||
|
||||
# Tuples of sha1s and paths.
|
||||
input_data = list(enumerate_input(
|
||||
input_filename, directory, recursive, ignore_errors, output, sha1_file,
|
||||
auto_platform))
|
||||
|
||||
# Sequentially check for the most common case and see if we can bail out
|
||||
# early before making any slow calls to gsutil.
|
||||
if not force and all(
|
||||
_data_exists(sha1, path, extract) for sha1, path in input_data):
|
||||
return 0
|
||||
|
||||
# Call this once to ensure gsutil's update routine is called only once. Only
|
||||
# needs to be done if we'll process input data in parallel, which can lead to
|
||||
# a race in gsutil's self-update on the first call. Note, this causes a
|
||||
# network call, therefore any fast bailout should be done before this point.
|
||||
if len(input_data) > 1:
|
||||
gsutil.check_call('version')
|
||||
|
||||
# Start up all the worker threads.
|
||||
all_threads = []
|
||||
download_start = time.time()
|
||||
stdout_queue = queue.Queue()
|
||||
work_queue = queue.Queue()
|
||||
ret_codes = queue.Queue()
|
||||
ret_codes.put((0, None))
|
||||
for thread_num in range(num_threads):
|
||||
t = threading.Thread(
|
||||
target=_downloader_worker_thread,
|
||||
args=[thread_num, work_queue, force, base_url,
|
||||
gsutil, stdout_queue, ret_codes, verbose, extract])
|
||||
t.daemon = True
|
||||
t.start()
|
||||
all_threads.append(t)
|
||||
printer_thread = PrinterThread(stdout_queue)
|
||||
printer_thread.daemon = True
|
||||
printer_thread.start()
|
||||
|
||||
# Populate our work queue.
|
||||
for sha1, path in input_data:
|
||||
work_queue.put((sha1, path))
|
||||
for _ in all_threads:
|
||||
work_queue.put((None, None)) # Used to tell worker threads to stop.
|
||||
|
||||
# Wait for all downloads to finish.
|
||||
for t in all_threads:
|
||||
t.join()
|
||||
stdout_queue.put(None)
|
||||
printer_thread.join()
|
||||
|
||||
# See if we ran into any errors.
|
||||
max_ret_code = 0
|
||||
for ret_code, message in ret_codes.queue:
|
||||
max_ret_code = max(ret_code, max_ret_code)
|
||||
if message:
|
||||
print(message, file=sys.stderr)
|
||||
|
||||
# Only print summary if any work was done.
|
||||
if printer_thread.did_print_anything:
|
||||
print('Downloading %d files took %1f second(s)' %
|
||||
(len(input_data), time.time() - download_start))
|
||||
return max_ret_code
|
||||
|
||||
|
||||
def main(args):
|
||||
usage = ('usage: %prog [options] target\n'
|
||||
'Target must be:\n'
|
||||
' (default) a sha1 sum ([A-Za-z0-9]{40}).\n'
|
||||
' (-s or --sha1_file) a .sha1 file, containing a sha1 sum on '
|
||||
'the first line.\n'
|
||||
' (-d or --directory) A directory to scan for .sha1 files.')
|
||||
parser = optparse.OptionParser(usage)
|
||||
parser.add_option('-o', '--output',
|
||||
help='Specify the output file name. Defaults to: '
|
||||
'(a) Given a SHA1 hash, the name is the SHA1 hash. '
|
||||
'(b) Given a .sha1 file or directory, the name will '
|
||||
'match (.*).sha1.')
|
||||
parser.add_option('-b', '--bucket',
|
||||
help='Google Storage bucket to fetch from.')
|
||||
parser.add_option('-e', '--boto',
|
||||
help='Specify a custom boto file.')
|
||||
parser.add_option('-c', '--no_resume', action='store_true',
|
||||
help='DEPRECATED: Resume download if file is '
|
||||
'partially downloaded.')
|
||||
parser.add_option('-f', '--force', action='store_true',
|
||||
help='Force download even if local file exists.')
|
||||
parser.add_option('-i', '--ignore_errors', action='store_true',
|
||||
help='Don\'t throw error if we find an invalid .sha1 file.')
|
||||
parser.add_option('-r', '--recursive', action='store_true',
|
||||
help='Scan folders recursively for .sha1 files. '
|
||||
'Must be used with -d/--directory')
|
||||
parser.add_option('-t', '--num_threads', default=1, type='int',
|
||||
help='Number of downloader threads to run.')
|
||||
parser.add_option('-d', '--directory', action='store_true',
|
||||
help='The target is a directory. '
|
||||
'Cannot be used with -s/--sha1_file.')
|
||||
parser.add_option('-s', '--sha1_file', action='store_true',
|
||||
help='The target is a file containing a sha1 sum. '
|
||||
'Cannot be used with -d/--directory.')
|
||||
parser.add_option('-g', '--config', action='store_true',
|
||||
help='Alias for "gsutil config". Run this if you want '
|
||||
'to initialize your saved Google Storage '
|
||||
'credentials. This will create a read-only '
|
||||
'credentials file in ~/.boto.depot_tools.')
|
||||
parser.add_option('-n', '--no_auth', action='store_true',
|
||||
help='Skip auth checking. Use if it\'s known that the '
|
||||
'target bucket is a public bucket.')
|
||||
parser.add_option('-p', '--platform',
|
||||
help='A regular expression that is compared against '
|
||||
'Python\'s sys.platform. If this option is specified, '
|
||||
'the download will happen only if there is a match.')
|
||||
parser.add_option('-a', '--auto_platform',
|
||||
action='store_true',
|
||||
help='Detects if any parent folder of the target matches '
|
||||
'(linux|mac|win). If so, the script will only '
|
||||
'process files that are in the paths that '
|
||||
'that matches the current platform.')
|
||||
parser.add_option('-u', '--extract',
|
||||
action='store_true',
|
||||
help='Extract a downloaded tar.gz file. '
|
||||
'Leaves the tar.gz file around for sha1 verification'
|
||||
'If a directory with the same name as the tar.gz '
|
||||
'file already exists, is deleted (to get a '
|
||||
'clean state in case of update.)')
|
||||
parser.add_option('-v', '--verbose', action='store_true', default=True,
|
||||
help='DEPRECATED: Defaults to True. Use --no-verbose '
|
||||
'to suppress.')
|
||||
parser.add_option('-q', '--quiet', action='store_false', dest='verbose',
|
||||
help='Suppresses diagnostic and progress information.')
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
# Make sure we should run at all based on platform matching.
|
||||
if options.platform:
|
||||
if options.auto_platform:
|
||||
parser.error('--platform can not be specified with --auto_platform')
|
||||
if not re.match(options.platform, GetNormalizedPlatform()):
|
||||
if options.verbose:
|
||||
print('The current platform doesn\'t match "%s", skipping.' %
|
||||
options.platform)
|
||||
return 0
|
||||
|
||||
# Set the boto file to /dev/null if we don't need auth.
|
||||
if options.no_auth:
|
||||
if (set(('http_proxy', 'https_proxy')).intersection(
|
||||
env.lower() for env in os.environ) and
|
||||
'NO_AUTH_BOTO_CONFIG' not in os.environ):
|
||||
print('NOTICE: You have PROXY values set in your environment, but gsutil'
|
||||
'in depot_tools does not (yet) obey them.',
|
||||
file=sys.stderr)
|
||||
print('Also, --no_auth prevents the normal BOTO_CONFIG environment'
|
||||
'variable from being used.',
|
||||
file=sys.stderr)
|
||||
print('To use a proxy in this situation, please supply those settings'
|
||||
'in a .boto file pointed to by the NO_AUTH_BOTO_CONFIG environment'
|
||||
'variable.',
|
||||
file=sys.stderr)
|
||||
options.boto = os.environ.get('NO_AUTH_BOTO_CONFIG', os.devnull)
|
||||
|
||||
# Make sure gsutil exists where we expect it to.
|
||||
if os.path.exists(GSUTIL_DEFAULT_PATH):
|
||||
gsutil = Gsutil(GSUTIL_DEFAULT_PATH,
|
||||
boto_path=options.boto)
|
||||
else:
|
||||
parser.error('gsutil not found in %s, bad depot_tools checkout?' %
|
||||
GSUTIL_DEFAULT_PATH)
|
||||
|
||||
# Passing in -g/--config will run our copy of GSUtil, then quit.
|
||||
if options.config:
|
||||
print('===Note from depot_tools===')
|
||||
print('If you do not have a project ID, enter "0" when asked for one.')
|
||||
print('===End note from depot_tools===')
|
||||
print()
|
||||
gsutil.check_call('version')
|
||||
return gsutil.call('config')
|
||||
|
||||
if not args:
|
||||
parser.error('Missing target.')
|
||||
if len(args) > 1:
|
||||
parser.error('Too many targets.')
|
||||
if not options.bucket:
|
||||
parser.error('Missing bucket. Specify bucket with --bucket.')
|
||||
if options.sha1_file and options.directory:
|
||||
parser.error('Both --directory and --sha1_file are specified, '
|
||||
'can only specify one.')
|
||||
if options.recursive and not options.directory:
|
||||
parser.error('--recursive specified but --directory not specified.')
|
||||
if options.output and options.directory:
|
||||
parser.error('--directory is specified, so --output has no effect.')
|
||||
if (not (options.sha1_file or options.directory)
|
||||
and options.auto_platform):
|
||||
parser.error('--auto_platform must be specified with either '
|
||||
'--sha1_file or --directory')
|
||||
|
||||
input_filename = args[0]
|
||||
|
||||
# Set output filename if not specified.
|
||||
if not options.output and not options.directory:
|
||||
if not options.sha1_file:
|
||||
# Target is a sha1 sum, so output filename would also be the sha1 sum.
|
||||
options.output = input_filename
|
||||
elif options.sha1_file:
|
||||
# Target is a .sha1 file.
|
||||
if not input_filename.endswith('.sha1'):
|
||||
parser.error('--sha1_file is specified, but the input filename '
|
||||
'does not end with .sha1, and no --output is specified. '
|
||||
'Either make sure the input filename has a .sha1 '
|
||||
'extension, or specify --output.')
|
||||
options.output = input_filename[:-5]
|
||||
else:
|
||||
parser.error('Unreachable state.')
|
||||
|
||||
base_url = 'gs://%s' % options.bucket
|
||||
|
||||
try:
|
||||
return download_from_google_storage(
|
||||
input_filename, base_url, gsutil, options.num_threads, options.directory,
|
||||
options.recursive, options.force, options.output, options.ignore_errors,
|
||||
options.sha1_file, options.verbose, options.auto_platform,
|
||||
options.extract)
|
||||
except FileNotFoundError as e:
|
||||
print("Fatal error: {}".format(e))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2014 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""Run a pinned gsutil."""
|
||||
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
try:
|
||||
import urllib2 as urllib
|
||||
except ImportError: # For Py3 compatibility
|
||||
import urllib.request as urllib
|
||||
|
||||
import zipfile
|
||||
|
||||
|
||||
GSUTIL_URL = 'https://storage.googleapis.com/pub/'
|
||||
API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/'
|
||||
|
||||
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
|
||||
|
||||
IS_WINDOWS = os.name == 'nt'
|
||||
|
||||
VERSION = '4.68'
|
||||
|
||||
|
||||
class InvalidGsutilError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def download_gsutil(version, target_dir):
|
||||
"""Downloads gsutil into the target_dir."""
|
||||
filename = 'gsutil_%s.zip' % version
|
||||
target_filename = os.path.join(target_dir, filename)
|
||||
|
||||
# Check if the target exists already.
|
||||
if os.path.exists(target_filename):
|
||||
md5_calc = hashlib.md5()
|
||||
with open(target_filename, 'rb') as f:
|
||||
while True:
|
||||
buf = f.read(4096)
|
||||
if not buf:
|
||||
break
|
||||
md5_calc.update(buf)
|
||||
local_md5 = md5_calc.hexdigest()
|
||||
|
||||
metadata_url = '%s%s' % (API_URL, filename)
|
||||
metadata = json.load(urllib.urlopen(metadata_url))
|
||||
remote_md5 = base64.b64decode(metadata['md5Hash']).decode('utf-8')
|
||||
|
||||
if local_md5 == remote_md5:
|
||||
return target_filename
|
||||
os.remove(target_filename)
|
||||
|
||||
# Do the download.
|
||||
url = '%s%s' % (GSUTIL_URL, filename)
|
||||
u = urllib.urlopen(url)
|
||||
with open(target_filename, 'wb') as f:
|
||||
while True:
|
||||
buf = u.read(4096)
|
||||
if not buf:
|
||||
break
|
||||
f.write(buf)
|
||||
return target_filename
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def temporary_directory(base):
|
||||
tmpdir = tempfile.mkdtemp(prefix='t', dir=base)
|
||||
try:
|
||||
yield tmpdir
|
||||
finally:
|
||||
if os.path.isdir(tmpdir):
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
def ensure_gsutil(version, target, clean):
|
||||
bin_dir = os.path.join(target, 'gsutil_%s' % version)
|
||||
gsutil_bin = os.path.join(bin_dir, 'gsutil', 'gsutil')
|
||||
gsutil_flag = os.path.join(bin_dir, 'gsutil', 'install.flag')
|
||||
# We assume that if gsutil_flag exists, then we have a good version
|
||||
# of the gsutil package.
|
||||
if not clean and os.path.isfile(gsutil_flag):
|
||||
# Everything is awesome! we're all done here.
|
||||
return gsutil_bin
|
||||
|
||||
if not os.path.exists(target):
|
||||
try:
|
||||
os.makedirs(target)
|
||||
except FileExistsError:
|
||||
# Another process is prepping workspace, so let's check if gsutil_bin is
|
||||
# present. If after several checks it's still not, continue with
|
||||
# downloading gsutil.
|
||||
delay = 2 # base delay, in seconds
|
||||
for _ in range(3): # make N attempts
|
||||
# sleep first as it's not expected to have file ready just yet.
|
||||
time.sleep(delay)
|
||||
delay *= 1.5 # next delay increased by that factor
|
||||
if os.path.isfile(gsutil_bin):
|
||||
return gsutil_bin
|
||||
|
||||
with temporary_directory(target) as instance_dir:
|
||||
# Clean up if we're redownloading a corrupted gsutil.
|
||||
cleanup_path = os.path.join(instance_dir, 'clean')
|
||||
try:
|
||||
os.rename(bin_dir, cleanup_path)
|
||||
except (OSError, IOError):
|
||||
cleanup_path = None
|
||||
if cleanup_path:
|
||||
shutil.rmtree(cleanup_path)
|
||||
|
||||
download_dir = os.path.join(instance_dir, 'd')
|
||||
target_zip_filename = download_gsutil(version, instance_dir)
|
||||
with zipfile.ZipFile(target_zip_filename, 'r') as target_zip:
|
||||
target_zip.extractall(download_dir)
|
||||
|
||||
shutil.move(download_dir, bin_dir)
|
||||
# Final check that the gsutil bin exists. This should never fail.
|
||||
if not os.path.isfile(gsutil_bin):
|
||||
raise InvalidGsutilError()
|
||||
# Drop a flag file.
|
||||
with open(gsutil_flag, 'w') as f:
|
||||
f.write('This flag file is dropped by gsutil.py')
|
||||
|
||||
return gsutil_bin
|
||||
|
||||
|
||||
def run_gsutil(target, args, clean=False):
|
||||
gsutil_bin = ensure_gsutil(VERSION, target, clean)
|
||||
args_opt = ['-o', 'GSUtil:software_update_check_period=0']
|
||||
|
||||
if sys.platform == 'darwin':
|
||||
# We are experiencing problems with multiprocessing on MacOS where gsutil.py
|
||||
# may hang.
|
||||
# This behavior is documented in gsutil codebase, and recommendation is to
|
||||
# set GSUtil:parallel_process_count=1.
|
||||
# https://github.com/GoogleCloudPlatform/gsutil/blob/06efc9dc23719fab4fd5fadb506d252bbd3fe0dd/gslib/command.py#L1331
|
||||
# https://github.com/GoogleCloudPlatform/gsutil/issues/1100
|
||||
args_opt.extend(['-o', 'GSUtil:parallel_process_count=1'])
|
||||
assert sys.platform != 'cygwin'
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
gsutil_bin
|
||||
] + args_opt + args
|
||||
return subprocess.call(cmd, shell=IS_WINDOWS)
|
||||
|
||||
|
||||
def parse_args():
|
||||
bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR)
|
||||
|
||||
# Help is disabled as it conflicts with gsutil -h, which controls headers.
|
||||
parser = argparse.ArgumentParser(add_help=False)
|
||||
|
||||
parser.add_argument('--clean', action='store_true',
|
||||
help='Clear any existing gsutil package, forcing a new download.')
|
||||
parser.add_argument('--target', default=bin_dir,
|
||||
help='The target directory to download/store a gsutil version in. '
|
||||
'(default is %(default)s).')
|
||||
|
||||
# These two args exist for backwards-compatibility but are no-ops.
|
||||
parser.add_argument('--force-version', default=VERSION,
|
||||
help='(deprecated, this flag has no effect)')
|
||||
parser.add_argument('--fallback',
|
||||
help='(deprecated, this flag has no effect)')
|
||||
|
||||
parser.add_argument('args', nargs=argparse.REMAINDER)
|
||||
|
||||
args, extras = parser.parse_known_args()
|
||||
if args.args and args.args[0] == '--':
|
||||
args.args.pop(0)
|
||||
if extras:
|
||||
args.args = extras + args.args
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
return run_gsutil(args.target, args.args, clean=args.clean)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1 @@
|
||||
dd736afb28430c9782750fc0fd5f0ed497399263
|
||||
@@ -0,0 +1 @@
|
||||
f1424c44ee758922823d6b37de43705955c99d7e
|
||||
@@ -0,0 +1 @@
|
||||
a1b33be85faf2578f3101d7806e443e1c0949498
|
||||
@@ -0,0 +1,258 @@
|
||||
# coding=utf-8
|
||||
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""Collection of subprocess wrapper functions.
|
||||
|
||||
In theory you shouldn't need anything else in subprocess, or this module failed.
|
||||
"""
|
||||
|
||||
import codecs
|
||||
import errno
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
||||
# Cache the string-escape codec to ensure subprocess can find it later.
|
||||
# See crbug.com/912292#c2 for context.
|
||||
if sys.version_info.major == 2:
|
||||
import Queue
|
||||
codecs.lookup('string-escape')
|
||||
# Sends stdout or stderr to os.devnull.
|
||||
DEVNULL = open(os.devnull, 'r+')
|
||||
else:
|
||||
import queue as Queue
|
||||
# pylint: disable=redefined-builtin
|
||||
basestring = (str, bytes)
|
||||
DEVNULL = subprocess.DEVNULL
|
||||
|
||||
|
||||
# Constants forwarded from subprocess.
|
||||
PIPE = subprocess.PIPE
|
||||
STDOUT = subprocess.STDOUT
|
||||
|
||||
|
||||
class CalledProcessError(subprocess.CalledProcessError):
|
||||
"""Augment the standard exception with more data."""
|
||||
def __init__(self, returncode, cmd, cwd, stdout, stderr):
|
||||
super(CalledProcessError, self).__init__(returncode, cmd, output=stdout)
|
||||
self.stdout = self.output # for backward compatibility.
|
||||
self.stderr = stderr
|
||||
self.cwd = cwd
|
||||
|
||||
def __str__(self):
|
||||
out = 'Command %r returned non-zero exit status %s' % (
|
||||
' '.join(self.cmd), self.returncode)
|
||||
if self.cwd:
|
||||
out += ' in ' + self.cwd
|
||||
if self.stdout:
|
||||
out += '\n' + self.stdout.decode('utf-8', 'ignore')
|
||||
if self.stderr:
|
||||
out += '\n' + self.stderr.decode('utf-8', 'ignore')
|
||||
return out
|
||||
|
||||
|
||||
class CygwinRebaseError(CalledProcessError):
|
||||
"""Occurs when cygwin's fork() emulation fails due to rebased dll."""
|
||||
|
||||
|
||||
## Utility functions
|
||||
|
||||
|
||||
def kill_pid(pid):
|
||||
"""Kills a process by its process id."""
|
||||
try:
|
||||
# Unable to import 'module'
|
||||
# pylint: disable=no-member,F0401
|
||||
import signal
|
||||
return os.kill(pid, signal.SIGTERM)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
def get_english_env(env):
|
||||
"""Forces LANG and/or LANGUAGE to be English.
|
||||
|
||||
Forces encoding to utf-8 for subprocesses.
|
||||
|
||||
Returns None if it is unnecessary.
|
||||
"""
|
||||
if sys.platform == 'win32':
|
||||
return None
|
||||
env = env or os.environ
|
||||
|
||||
# Test if it is necessary at all.
|
||||
is_english = lambda name: env.get(name, 'en').startswith('en')
|
||||
|
||||
if is_english('LANG') and is_english('LANGUAGE'):
|
||||
return None
|
||||
|
||||
# Requires modifications.
|
||||
env = env.copy()
|
||||
def fix_lang(name):
|
||||
if not is_english(name):
|
||||
env[name] = 'en_US.UTF-8'
|
||||
fix_lang('LANG')
|
||||
fix_lang('LANGUAGE')
|
||||
return env
|
||||
|
||||
|
||||
class Popen(subprocess.Popen):
|
||||
"""Wraps subprocess.Popen() with various workarounds.
|
||||
|
||||
- Forces English output since it's easier to parse the stdout if it is always
|
||||
in English.
|
||||
- Sets shell=True on windows by default. You can override this by forcing
|
||||
shell parameter to a value.
|
||||
- Adds support for DEVNULL to not buffer when not needed.
|
||||
- Adds self.start property.
|
||||
|
||||
Note: Popen() can throw OSError when cwd or args[0] doesn't exist. Translate
|
||||
exceptions generated by cygwin when it fails trying to emulate fork().
|
||||
"""
|
||||
# subprocess.Popen.__init__() is not threadsafe; there is a race between
|
||||
# creating the exec-error pipe for the child and setting it to CLOEXEC during
|
||||
# which another thread can fork and cause the pipe to be inherited by its
|
||||
# descendents, which will cause the current Popen to hang until all those
|
||||
# descendents exit. Protect this with a lock so that only one fork/exec can
|
||||
# happen at a time.
|
||||
popen_lock = threading.Lock()
|
||||
|
||||
def __init__(self, args, **kwargs):
|
||||
env = get_english_env(kwargs.get('env'))
|
||||
if env:
|
||||
kwargs['env'] = env
|
||||
if kwargs.get('env') is not None and sys.version_info.major != 2:
|
||||
# Subprocess expects environment variables to be strings in Python 3.
|
||||
def ensure_str(value):
|
||||
if isinstance(value, bytes):
|
||||
return value.decode()
|
||||
return value
|
||||
|
||||
kwargs['env'] = {
|
||||
ensure_str(k): ensure_str(v)
|
||||
for k, v in kwargs['env'].items()
|
||||
}
|
||||
if kwargs.get('shell') is None:
|
||||
# *Sigh*: Windows needs shell=True, or else it won't search %PATH% for
|
||||
# the executable, but shell=True makes subprocess on Linux fail when it's
|
||||
# called with a list because it only tries to execute the first item in
|
||||
# the list.
|
||||
kwargs['shell'] = bool(sys.platform=='win32')
|
||||
|
||||
if isinstance(args, basestring):
|
||||
tmp_str = args
|
||||
elif isinstance(args, (list, tuple)):
|
||||
tmp_str = ' '.join(args)
|
||||
else:
|
||||
raise CalledProcessError(None, args, kwargs.get('cwd'), None, None)
|
||||
if kwargs.get('cwd', None):
|
||||
tmp_str += '; cwd=%s' % kwargs['cwd']
|
||||
logging.debug(tmp_str)
|
||||
|
||||
try:
|
||||
with self.popen_lock:
|
||||
super(Popen, self).__init__(args, **kwargs)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EAGAIN and sys.platform == 'cygwin':
|
||||
# Convert fork() emulation failure into a CygwinRebaseError().
|
||||
raise CygwinRebaseError(
|
||||
e.errno,
|
||||
args,
|
||||
kwargs.get('cwd'),
|
||||
None,
|
||||
'Visit '
|
||||
'http://code.google.com/p/chromium/wiki/CygwinDllRemappingFailure '
|
||||
'to learn how to fix this error; you need to rebase your cygwin '
|
||||
'dlls')
|
||||
# Popen() can throw OSError when cwd or args[0] doesn't exist.
|
||||
raise OSError('Execution failed with error: %s.\n'
|
||||
'Check that %s or %s exist and have execution permission.'
|
||||
% (str(e), kwargs.get('cwd'), args[0]))
|
||||
|
||||
|
||||
def communicate(args, **kwargs):
|
||||
"""Wraps subprocess.Popen().communicate().
|
||||
|
||||
Returns ((stdout, stderr), returncode).
|
||||
|
||||
- If the subprocess runs for |nag_timer| seconds without producing terminal
|
||||
output, print a warning to stderr.
|
||||
- Automatically passes stdin content as input so do not specify stdin=PIPE.
|
||||
"""
|
||||
stdin = None
|
||||
# When stdin is passed as an argument, use it as the actual input data and
|
||||
# set the Popen() parameter accordingly.
|
||||
if 'stdin' in kwargs and isinstance(kwargs['stdin'], basestring):
|
||||
stdin = kwargs['stdin']
|
||||
kwargs['stdin'] = PIPE
|
||||
|
||||
proc = Popen(args, **kwargs)
|
||||
return proc.communicate(stdin), proc.returncode
|
||||
|
||||
|
||||
def call(args, **kwargs):
|
||||
"""Emulates subprocess.call().
|
||||
|
||||
Automatically convert stdout=PIPE or stderr=PIPE to DEVNULL.
|
||||
In no case they can be returned since no code path raises
|
||||
subprocess2.CalledProcessError.
|
||||
|
||||
Returns exit code.
|
||||
"""
|
||||
if kwargs.get('stdout') == PIPE:
|
||||
kwargs['stdout'] = DEVNULL
|
||||
if kwargs.get('stderr') == PIPE:
|
||||
kwargs['stderr'] = DEVNULL
|
||||
return communicate(args, **kwargs)[1]
|
||||
|
||||
|
||||
def check_call_out(args, **kwargs):
|
||||
"""Improved version of subprocess.check_call().
|
||||
|
||||
Returns (stdout, stderr), unlike subprocess.check_call().
|
||||
"""
|
||||
out, returncode = communicate(args, **kwargs)
|
||||
if returncode:
|
||||
raise CalledProcessError(
|
||||
returncode, args, kwargs.get('cwd'), out[0], out[1])
|
||||
return out
|
||||
|
||||
|
||||
def check_call(args, **kwargs):
|
||||
"""Emulate subprocess.check_call()."""
|
||||
check_call_out(args, **kwargs)
|
||||
return 0
|
||||
|
||||
|
||||
def capture(args, **kwargs):
|
||||
"""Captures stdout of a process call and returns it.
|
||||
|
||||
Returns stdout.
|
||||
|
||||
- Discards returncode.
|
||||
- Blocks stdin by default if not specified since no output will be visible.
|
||||
"""
|
||||
kwargs.setdefault('stdin', DEVNULL)
|
||||
|
||||
# Like check_output, deny the caller from using stdout arg.
|
||||
return communicate(args, stdout=PIPE, **kwargs)[0][0]
|
||||
|
||||
|
||||
def check_output(args, **kwargs):
|
||||
"""Emulates subprocess.check_output().
|
||||
|
||||
Captures stdout of a process call and returns stdout only.
|
||||
|
||||
- Throws if return code is not 0.
|
||||
- Works even prior to python 2.7.
|
||||
- Blocks stdin by default if not specified since no output will be visible.
|
||||
- As per doc, "The stdout argument is not allowed as it is used internally."
|
||||
"""
|
||||
kwargs.setdefault('stdin', DEVNULL)
|
||||
if 'stdout' in kwargs:
|
||||
raise ValueError('stdout argument not allowed, it would be overridden.')
|
||||
return check_call_out(args, stdout=PIPE, **kwargs)[0]
|
||||
@@ -0,0 +1 @@
|
||||
66882fadbf9e99cc00b8677d8c1e7e8b3cfdf4fe
|
||||
Reference in New Issue
Block a user