I think the idea here is to allow something like

pip install scipy

but for non-packages - so you can find where dials_regression etc. are when you do a fresh build, without having to remember where to git clone everything from if I pip install something I do not need to know where the bytes come from - same idea - and real world here is defined as “the usual way things work in Python” Warehouse is effectively a directory… For us in dials land this is very helpful as things come from all over the place Obviously the use of this is *optional* and you can just remember the URL’s if you want… Cheers Graeme On 13 Mar 2018, at 11:04, Pavel Afonine mailto:[email protected]> wrote: Just out of curiosity: what's your definition of "real world"? Pavel On 3/13/18 03:49, CCTBX commit wrote: Repository : ssh://g18-sc-serv-04.diamond.ac.uk/cctbx On branch : master ________________________________ commit 914c74bf195b5e5d4fa2c4145da3e5f89c7e5872 Author: Markus Gerstel mailto:[email protected] Date: Tue Mar 13 10:43:22 2018 +0000 New command: libtbx.install This downloads and configures from a select list of cctbx/libtbxish modules in a similar manner to pip in the real world. This will not update or install any libraries in the base build, but assuming those libraries are present libtbx.install should work. ________________________________ 914c74bf195b5e5d4fa2c4145da3e5f89c7e5872 libtbx/command_line/install.py | 118 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/libtbx/command_line/install.py b/libtbx/command_line/install.py new file mode 100644 index 000000000..47ad94eb5 --- /dev/null +++ b/libtbx/command_line/install.py @@ -0,0 +1,118 @@ +from __future__ import absolute_import, division, print_function + +import collections +import os +import sys +from optparse import SUPPRESS_HELP, OptionParser + +import procrunner + +import libtbx.load_env +from libtbx.auto_build.bootstrap import Toolbox + +# Basically 'pip' for selected libtbx/cctbx modules. + + +def is_source_repository(path): + return (path / '.git').isdir() or (path / '.svn').isdir() + +def run(args): + parser = OptionParser(usage="libtbx.install [package]", + description="Installs an additional cctbx package") + parser.add_option("-?", action="help", help=SUPPRESS_HELP) + options, args = parser.parse_args(args) + + modules_directory = list(filter(lambda dir: not is_source_repository(dir), libtbx.env.repository_paths)) + if not modules_directory: + sys.exit("No repository path candidate found. Can't install modules without an installation root.") + if len(modules_directory) > 1: + print("More than one repository path candidate found.") + installation_root = modules_directory[0] + print("Using %s as installation root" % abs(installation_root)) + + packages_to_configure = set() + + errors = False + for package in args: + if (installation_root / package).isdir(): + print("Skipping package %s: Directory already exists in installation root" % package) + errors = True + continue + if package not in warehouse: + print("Skipping package %s: Never heard of this before" % package) + errors = True + continue + + downloaded = False + for mech, call in mechanisms.items(): + if mech in warehouse[package]: + print("Attempting to obtain %s using %s..." % (package, mech), end='') + sys.stdout.flush() + if call(abs(installation_root / package), warehouse[package][mech]): + assert (installation_root / package).isdir(), "Installation failed" + downloaded = True + print("success") + break + else: + assert not (installation_root / package).isdir(), "Install mechanism %s did not fail cleanly" % mech + print("failed") + if not downloaded: + print("Skipping package %s: Could not install" % package) + errors = True + continue + + packages_to_configure.add(package) + + if packages_to_configure: + packages_to_configure = sorted(packages_to_configure) + print("Configuring packages %s" % ", ".join(packages_to_configure)) + os.chdir(abs(libtbx.env.build_path)) + result = procrunner.run_process(['libtbx.configure'] + packages_to_configure) + if result['exitcode']: errors = True + + if errors: + sys.exit(1) + +def install_git(location, source): + try: + result = procrunner.run_process(['git', 'clone', '--recursive', source, location], print_stdout=False, print_stderr=False) + if result['exitcode']: + return False + Toolbox.set_git_repository_config_to_rebase(os.path.join(location, '.git', 'config')) + return True + except OSError: + return False # git may not be installed + +def install_zip(location, source): + os.mkdir(location) + tempfile = os.path.join(location, '.tmp.zip') + etagfile = os.path.join(location, '..tmp.zip.etag') + def cleanup(): + try: os.remove(tempfile) + except OSError: pass + try: os.remove(etagfile) + except OSError: pass + if Toolbox.download_to_file(source['url'], tempfile) <= 0: + cleanup() + os.rmdir(location) + return False + Toolbox.unzip(tempfile, location, trim_directory=source.get('trim', 0)) + cleanup() + return True + +mechanisms = collections.OrderedDict(( + ('git-auth', install_git), + ('git-anon', install_git), + ('http-zip', install_zip), +)) + +warehouse = { + 'i19': { + 'git-auth': '[email protected]:/xia2/i19mailto:[email protected]:/xia2/i19', + 'git-anon': 'https://github.com/xia2/i19.git', + 'http-zip': { 'url': 'https://github.com/xia2/i19/archive/master.zip', 'trim': 1 }, + }, +} + +if __name__ == '__main__': + run(sys.argv[1:]) _______________________________________________ cctbxbb mailing list [email protected]mailto:[email protected] http://phenix-online.org/mailman/listinfo/cctbxbb -- This e-mail and any attachments may contain confidential, copyright and or privileged material, and are for the use of the intended addressee only. If you are not the intended addressee or an authorised recipient of the addressee please notify us of receipt by returning the e-mail and do not use, copy, retain, distribute or disclose the information in or attached to the e-mail. Any opinions expressed within this e-mail are those of the individual and not necessarily of Diamond Light Source Ltd. Diamond Light Source Ltd. cannot guarantee that this e-mail or any attachments are free from viruses and we cannot accept liability for any damage which you may sustain as a result of software viruses which may be transmitted in or with the message. Diamond Light Source Limited (company no. 4375679). Registered in England and Wales with its registered office at Diamond House, Harwell Science and Innovation Campus, Didcot, Oxfordshire, OX11 0DE, United Kingdom