Source code for anybox.recipe.openerp.base

# coding: utf-8
from os.path import join, basename
import os
import sys
import re
import urllib
import tarfile
import setuptools
import logging
import stat
import imp
import shutil
import ConfigParser
import distutils.core
import pkg_resources
try:
    from collections import OrderedDict
except ImportError:  # Python < 2.7
    from ordereddict import OrderedDict  # noqa
from zc.buildout.easy_install import MissingDistribution
from zc.buildout import UserError
from zc.buildout.easy_install import VersionConflict

try:  # need to distinguish, because of differing semantics
    from zc.buildout.easy_install import IncompatibleConstraintError
except ImportError:  # zc.buildout < 1.7
    IncompatibleConstraintError = None  # 'except None' is very fine
    from zc.buildout.easy_install import IncompatibleVersionError
else:
    IncompatibleVersionError = None

import zc.recipe.egg

import httplib
import rfc822
from urlparse import urlparse
from . import vcs
from . import utils
from .utils import option_splitlines, option_strip

logger = logging.getLogger(__name__)


[docs]def rfc822_time(h): """Parse RFC 2822-formatted http header and return a time int.""" rfc822.mktime_tz(rfc822.parsedate_tz(h))
[docs]class MainSoftware(object): """Placeholder to represent the main software instead of an addon location. Should just have a singleton instance: :data:`main_software`, whose meaning depends on the concrete recipe class using it. For example, in :class:`anybox.recipe.openerp.server.ServerRecipe`, :data:`main_software` represents the OpenObject server or the OpenERP standard distribution. """ def __str__(self): return 'Main Software'
main_software = MainSoftware() GP_VCS_EXTEND_DEVELOP = 'vcs-extend-develop'
[docs]class BaseRecipe(object): """Base class for other recipes. It implements notably fetching of the main software part plus addons. The :attr:`sources` attribute is a ``dict`` storing how to fetch the main software part and the specified addons, with the following structure: ``local path -> (type, location_spec, options)``, in which: :local path: is either the :data:`main_software` singleton (see :class:`MainSoftware`) or a local path to an addons directory. :type: can be either * ``'local'`` * ``'downloadable'`` * one of the supported vcs :location_spec: is, depending on the type, a tuple specifying how fetch is to be done: ``url``, or ``(vcs_url, vcs_revision)`` or ``None`` :addons options: are typically used to specify that the addons directory is actually a subdir of the specified one. VCS support classes (see :mod:`anybox.recipe.openerp.vcs`) can implemented their dedicated options The :attr:`merges` attribute is a ``dict`` storing how to fetch additional changes to merge into VCS type sources: ``local path -> [(type, location_spec, options), ... ]`` See :attr:`sources` for the meaning of the various components. Note that in :attr:`merges`, values are a list of triples instead of only a single triple as values in :attr:`sources` because there can be multiple merges on the same local path. """ release_dl_url = {'6.0': 'http://nightly.odoo.com/old/openerp-6/', '6.1': 'http://nightly.odoo.com/old/openerp-6.1/', '5.0': 'http://nightly.odoo.com/old/openerp-5/', } nightly_dl_url = {'6.0': 'http://nightly.odoo.com/6.0/6.0/', '6.1': 'http://nightly.odoo.com/6.1/nightly/src/', '7.0': 'http://nightly.odoo.com/7.0/nightly/src/', '8.0': 'http://nightly.odoo.com/8.0/nightly/src/', 'trunk': 'http://nightly.odoo.com/trunk/nightly/src/', } recipe_requirements = () # distribution required for the recipe itself recipe_requirements_paths = () # a default value is useful in unit tests requirements = () # requirements for what the recipe installs to run soft_requirements = () # subset of requirements that's not necessary addons_paths = () # Caching logic for the main OpenERP part (e.g, without addons) # Can be 'filename' or 'http-head' main_http_caching = 'filename' is_git_layout = False """True if this is the git layout, as seen from the move to GitHub. In this layout, the standard addons other than ``base`` are in a ``addons`` directory right next to the ``openerp`` package. """ def __init__(self, buildout, name, options): self.requirements = list(self.requirements) self.recipe_requirements_path = [] self.buildout, self.name, self.options = buildout, name, options self.b_options = self.buildout['buildout'] self.buildout_dir = self.b_options['directory'] # GR: would prefer lower() but doing as in 'zc.recipe.egg' # (later) the standard way for all booleans is to use # options.query_bool() or get_bool(), but it doesn't lower() at all self.offline = self.b_options['offline'] == 'true' self.clean = options.get('clean') == 'true' clear_locks = options.get('vcs-clear-locks', '').lower() self.vcs_clear_locks = clear_locks == 'true' clear_retry = options.get('vcs-clear-retry', '').lower() self.clear_retry = clear_retry == 'true' # same as in zc.recipe.eggs self.extra_paths = [ join(self.buildout_dir, p.strip()) for p in option_splitlines(self.options.get('extra-paths')) ] self.options['extra-paths'] = os.linesep.join(self.extra_paths) self.downloads_dir = self.make_absolute( self.b_options.get('openerp-downloads-directory', 'downloads')) self.version_wanted = None # from the buildout self.version_detected = None # string from the openerp setup.py self.parts = self.buildout['buildout']['parts-directory'] self.openerp_dir = None self.archive_filename = None self.archive_path = None # downloaded tar.gz if options.get('scripts') is None: options['scripts'] = '' # a dictionnary of messages to display in case a distribution is # not installable. self.missing_deps_instructions = { 'PIL': ("You don't need to require it for OpenERP any more, since " "the recipe automatically adds a dependency to Pillow. " "If you really need it for other reasons, installing it " "system-wide is a good option. "), } self.openerp_installed = [] self.etc = self.make_absolute(options.get('etc-directory', 'etc')) self.bin_dir = self.buildout['buildout']['bin-directory'] self.config_path = join(self.etc, self.name + '.cfg') for d in self.downloads_dir, self.etc: if not os.path.exists(d): logger.info('Created %s/ directory' % basename(d)) os.mkdir(d) self.sources = OrderedDict() self.merges = OrderedDict() self.parse_addons(options) self.parse_version() self.parse_revisions(options) self.parse_merges(options)
[docs] def parse_version(self): """Set the main software in :attr:`sources` and related attributes. """ self.version_wanted = option_strip(self.options.get('version')) if self.version_wanted is None: raise UserError('You must specify the version') self.preinstall_version_check() version_split = self.version_wanted.split() if len(version_split) == 1: # version can be a simple version name, such as 6.1-1 major_wanted = self.version_wanted[:3] pattern = self.release_filenames[major_wanted] if pattern is None: raise UserError('OpenERP version %r' 'is not supported' % self.version_wanted) self.archive_filename = pattern % self.version_wanted self.archive_path = join(self.downloads_dir, self.archive_filename) base_url = self.options.get( 'base_url', self.release_dl_url[major_wanted]) self.sources[main_software] = ( 'downloadable', '/'.join((base_url.strip('/'), self.archive_filename)), None) return # in all other cases, the first token is the type of version type_spec = version_split[0] if type_spec in ('local', 'path'): self.openerp_dir = join(self.buildout_dir, version_split[1]) self.sources[main_software] = ('local', None) elif type_spec == 'url': url = version_split[1] self.archive_filename = urlparse(url).path.split('/')[-1] self.archive_path = join(self.downloads_dir, self.archive_filename) self.sources[main_software] = ('downloadable', url, None) elif type_spec == 'nightly': if len(version_split) != 3: raise UserError( "Unrecognized nightly version specification: " "%r (expecting series, number) % version_split[1:]") self.nightly_series, self.version_wanted = version_split[1:] type_spec = 'downloadable' if self.version_wanted == 'latest': self.main_http_caching = 'http-head' series = self.nightly_series self.archive_filename = ( self.nightly_filenames[series] % self.version_wanted) self.archive_path = join(self.downloads_dir, self.archive_filename) base_url = self.options.get('base_url', self.nightly_dl_url[series]) self.sources[main_software] = ( 'downloadable', '/'.join((base_url.strip('/'), self.archive_filename)), None) else: # VCS types type_spec, url, repo_dir, self.version_wanted = version_split[0:4] options = dict(opt.split('=') for opt in version_split[4:]) self.openerp_dir = join(self.parts, repo_dir) self.sources[main_software] = (type_spec, (url, self.version_wanted), options)
[docs] def preinstall_version_check(self): """Perform version checks before any attempt to install. To be subclassed. """
[docs] def install_recipe_requirements(self): """Install requirements for the recipe to run.""" to_install = self.recipe_requirements eggs_option = os.linesep.join(to_install) eggs = zc.recipe.egg.Eggs(self.buildout, '', dict(eggs=eggs_option)) ws = eggs.install() _, ws = eggs.working_set() self.recipe_requirements_paths = [ws.by_key[dist].location for dist in to_install] sys.path.extend(self.recipe_requirements_paths)
[docs] def merge_requirements(self): """Merge eggs option with self.requirements.""" if 'eggs' not in self.options: self.options['eggs'] = '\n'.join(self.requirements) else: self.options['eggs'] += '\n' + '\n'.join(self.requirements)
[docs] def install_requirements(self): """Install egg requirements and scripts. If some distributions are known as soft requirements, will retry without them """ while True: missing = None eggs_recipe = zc.recipe.egg.Scripts(self.buildout, '', self.options) try: eggs_recipe.install() except MissingDistribution as exc: missing = exc.data[0].project_name except VersionConflict as exc: # GR not 100% sure, but this should mean a conflict with an # already loaded version (don't know what can lead to this # 'already', have seen it with zc.buildout itself only so far) # In any case, removing the requirement can't make for a sane # recovery raise except IncompatibleVersionError as exc: # zc.buildout < 1.7 # In exc's attrs, we have the version but not the project name raise except IncompatibleConstraintError as exc: missing = exc.args[2].project_name except UserError, exc: # happens only for zc.buildout >= 2.0 missing = exc.message.split(os.linesep)[0].split()[-1] missing = re.split(r'[=<>]', missing)[0] else: break logger.error("Could not find or install %r. " + self.missing_deps_instructions.get(missing, '') + " Original exception %s.%s says: %s", missing, exc.__class__.__module__, exc.__class__.__name__, exc) if missing not in self.soft_requirements: raise exc eggs = set(self.options['eggs'].split(os.linesep)) if missing not in eggs: logger.error("Soft requirement %r is also an indirect " "dependency (either of OpenERP/Odoo or of " "one listed in config file). Can't retry.", missing) raise exc logger.warn("%r is a direct soft requirement, " "retrying without it", missing) eggs.discard(missing) self.options['eggs'] = os.linesep.join(eggs) self.eggs_reqs, self.eggs_ws = eggs_recipe.working_set() self.ws = self.eggs_ws
[docs] def apply_version_dependent_decisions(self): """Store some booleans depending on detected version. To be refined by subclasses. """ pass
@property def major_version(self): detected = self.version_detected if detected is None: return None return utils.major_version(detected)
[docs] def read_release(self): """Try and read the release.py file directly. Should be used only if reading setup.py failed, which happens with OpenERP 5.0 """ with open(join(self.openerp_dir, 'bin', 'release.py'), 'rb') as f: mod = imp.load_module('release', f, 'release.py', ('.py', 'r', imp.PY_SOURCE)) self.version_detected = mod.version
[docs] def read_openerp_setup(self): """Ugly method to extract requirements & version from ugly setup.py. Primarily designed for 6.0, but works with 6.1 as well. """ old_setup = setuptools.setup old_distutils_setup = distutils.core.setup # 5.0 directly imports this def new_setup(*args, **kw): self.requirements.extend(kw.get('install_requires', ())) self.version_detected = kw['version'] setuptools.setup = new_setup distutils.core.setup = new_setup sys.path.insert(0, '.') with open(join(self.openerp_dir, 'setup.py'), 'rb') as f: saved_argv = sys.argv sys.argv = ['setup.py', 'develop'] try: imp.load_module('setup', f, 'setup.py', ('.py', 'r', imp.PY_SOURCE)) except SystemExit as exception: msg = exception.message if not isinstance(msg, int) and 'dsextras' in msg: raise EnvironmentError( 'Please first install PyGObject and PyGTK !') else: try: self.read_release() except Exception as exc: raise EnvironmentError( 'Problem while reading OpenERP release.py: ' + exc.message) except ImportError, exception: if 'babel' in exception.message: raise EnvironmentError( 'OpenERP setup.py has an unwanted import Babel.\n' '=> First install Babel on your system or ' 'virtualenv :(\n' '(sudo aptitude install python-babel, ' 'or pip install babel)') else: raise exception except Exception, exception: raise EnvironmentError('Problem while reading OpenERP ' 'setup.py: ' + exception.message) finally: sys.argv = saved_argv sys.path.pop(0) setuptools.setup = old_setup distutils.core.setup = old_distutils_setup self.apply_version_dependent_decisions()
[docs] def make_absolute(self, path): """Make a path absolute if needed. If not already absolute, it is interpreted as relative to the buildout directory.""" if os.path.isabs(path): return path return join(self.buildout_dir, path)
[docs] def sandboxed_tar_extract(self, sandbox, tarfile, first=None): """Extract those members that are below the tarfile path 'sandbox'. The tarfile module official doc warns against attacks with .. in tar. The option to start with a first member is useful for this case, since the recipe consumes a first member in the tar file to get the openerp main directory in parts. It is taken for granted that this first member has already been checked. """ if first is not None: tarfile.extract(first) for tinfo in tarfile: if tinfo.name.startswith(sandbox + '/'): tarfile.extract(tinfo) else: logger.warn('Tarball member %r is outside of %r. Ignored.', tinfo, sandbox)
def _produce_setup_without_pil(self, src_directory): """Create a copy of setup.py without PIL and return a path to it.""" new_setup_path = join(src_directory, 'setup.nopil.py') with open(join(src_directory, 'setup.py')) as inp: setup_str = inp.read() with open(new_setup_path, 'w') as out: out.write(setup_str.replace("'PIL',", '')) return new_setup_path
[docs] def develop(self, src_directory, setup_has_pil=False): """Develop the specified source distribution. Any call to ``zc.recipe.eggs`` will use that developped version. :meth:`develop` launches a subprocess, to which we need to forward the paths to requirements via PYTHONPATH. :param setup_has_pil: if ``True``, an altered version of setup that does not require PIL is produced to perform the develop, so that installation can be done with ``Pillow`` instead. Recent enough versions of OpenERP/Odoo are directly based on Pillow. :returns: project name of the distribution that's been "developed" This is useful for OpenERP/Odoo itself, whose project name changed within the 8.0 stable branch. """ logger.debug("Developing %r", src_directory) develop_dir = self.b_options['develop-eggs-directory'] pythonpath_bak = os.getenv('PYTHONPATH') os.putenv('PYTHONPATH', ':'.join(self.recipe_requirements_paths)) if setup_has_pil: setup = self._produce_setup_without_pil(src_directory) else: setup = src_directory try: egg_link = zc.buildout.easy_install.develop(setup, develop_dir) finally: if setup_has_pil: os.unlink(setup) suffix = '.egg-link' if pythonpath_bak is None: os.unsetenv('PYTHONPATH') else: os.putenv('PYTHONPATH', pythonpath_bak) if not egg_link.endswith(suffix): raise RuntimeError( "Development of OpenERP/Odoo distribution " "produced an unexpected egg link: %r" % egg_link) return os.path.basename(egg_link)[:-len(suffix)]
[docs] def parse_addons(self, options): """Parse the addons options into :attr:`sources`. See :class:`BaseRecipe` for the structure of :attr:`sources`. """ for line in option_splitlines(options.get('addons')): split = line.split() if not split: return try: loc_type = split[0] spec_len = 2 if loc_type == 'local' else 4 options = dict(opt.split('=') for opt in split[spec_len:]) if loc_type == 'local': addons_dir = split[1] location_spec = None else: # vcs repo_url, addons_dir, repo_rev = split[1:4] location_spec = (repo_url, repo_rev) except: raise UserError("Could not parse addons line: %r. " "Please check format " % line) addons_dir = addons_dir.rstrip('/') # trailing / can be harmful group = options.get('group') if group: split = os.path.split(addons_dir) addons_dir = os.path.join(split[0], group, split[1]) self.sources[addons_dir] = (loc_type, location_spec, options)
[docs] def parse_merges(self, options): """Parse the merge options into :attr:`merges`. See :class:`BaseRecipe` for the structure of :attr:`merges`. """ for line in option_splitlines(options.get('merges')): split = line.split() if not split: return loc_type = split[0] if loc_type not in ('bzr', 'git'): raise UserError("Only merges of type 'bzr' and 'git' are " "currently supported.") options = dict(opt.split('=') for opt in split[4:]) if loc_type == 'bzr': options['bzr-init'] = 'merge' else: options['merge'] = True repo_url, local_dir, repo_rev = split[1:4] location_spec = (repo_url, repo_rev) local_dir = local_dir.rstrip('/') # trailing / can be harmful self.merges.setdefault(local_dir, []).append( (loc_type, location_spec, options))
[docs] def parse_revisions(self, options): """Parse revisions options and update :attr:`sources`. It is assumed that :attr:`sources` has already been populated, and notably has a :data:`main_software` entry. This allows for easy fixing of revisions in an extension buildout See :class:`BaseRecipe` for the structure of :attr:`sources`. """ for line in option_splitlines(options.get('revisions')): split = line.split() if len(split) > 2: raise UserError("Invalid revisions line: %r" % line) # addon or main software if len(split) == 2: local_path = split[0] else: local_path = main_software revision = split[-1] source = self.sources.get(local_path) if source is None: # considered harmless for now logger.warn("Ignoring attempt to fix revision on unknown " "source %r. You may have a leftover to clean", local_path) continue if source[0] in ('downloadable', 'local'): raise UserError("In revision line %r : can't fix a revision " "for non-vcs source" % line) logger.info("%s will be on revision %r", local_path, revision) self.sources[local_path] = ((source[0], (source[1][0], revision)) + source[2:])
[docs] def retrieve_addons(self): """Peform all lookup and downloads specified in :attr:`sources`. See :class:`BaseRecipe` for the structure of :attr:`sources`. """ self.addons_paths = [] for local_dir, source_spec in self.sources.items(): if local_dir is main_software: continue loc_type, loc_spec, addons_options = source_spec local_dir = self.make_absolute(local_dir) options = dict(offline=self.offline, clear_locks=self.vcs_clear_locks, clean=self.clean) if loc_type == 'git': options['depth'] = self.options.get('git-depth') options.update(addons_options) group = addons_options.get('group') group_dir = None if group: if loc_type == 'local': raise UserError( "Automatic grouping of addons is not supported for " "local addons such as %r, because the recipe " "considers that write operations in a local " "directory is " "outside of its reponsibilities (in other words, " "it's better if " "you create yourself the intermediate directory." % ( local_dir, )) group_dir = os.path.dirname(local_dir) if not os.path.exists(group_dir): os.makedirs(group_dir) if loc_type != 'local': for k, v in self.options.items(): if k.startswith(loc_type + '-'): options[k] = v repo_url, repo_rev = loc_spec vcs.get_update(loc_type, local_dir, repo_url, repo_rev, clear_retry=self.clear_retry, **options) elif self.clean: utils.clean_object_files(local_dir) subdir = addons_options.get('subdir') if group_dir: addons_dir = group_dir else: addons_dir = local_dir if subdir: addons_dir = join(addons_dir, subdir) manifest = os.path.join(addons_dir, '__openerp__.py') manifest_pre_v6 = os.path.join(addons_dir, '__terp__.py') if os.path.isfile(manifest) or os.path.isfile(manifest_pre_v6): raise UserError("Standalone addons such as %r " "are now supported by means " "of the explicit 'group' option. Please " "update your buildout configuration. " % ( addons_dir)) if addons_dir not in self.addons_paths: self.addons_paths.append(addons_dir)
[docs] def revert_sources(self): """Revert all sources to the revisions specified in :attr:`sources`. """ for target, desc in self.sources.iteritems(): if desc[0] in ('local', 'downloadable'): continue vcs_type, vcs_spec, options = desc local_dir = self.openerp_dir if target is main_software else target local_dir = self.make_absolute(local_dir) repo = vcs.repo(vcs_type, local_dir, vcs_spec[0], **options) try: repo.revert(vcs_spec[1]) except NotImplementedError: logger.warn("vcs-revert: not implemented for %s " "repository at %s", vcs_type, local_dir) else: logger.info("Reverted %s repository at %s", vcs_type, local_dir)
[docs] def retrieve_merges(self): """Peform all VCS merges specified in :attr:`merges`. """ if self.options.get('vcs-revert', '').strip().lower() == 'on-merge': logger.info("Reverting all sources before merge") self.revert_sources() for local_dir, source_specs in self.merges.items(): for source_spec in source_specs: loc_type, loc_spec, merge_options = source_spec local_dir = self.make_absolute(local_dir) options = dict(offline=self.offline, clear_locks=self.vcs_clear_locks) options.update(merge_options) for k, v in self.options.items(): if k.startswith(loc_type + '-'): options[k] = v repo_url, repo_rev = loc_spec vcs.get_update(loc_type, local_dir, repo_url, repo_rev, clear_retry=self.clear_retry, **options)
[docs] def main_download(self): """HTTP download for main part of the software to self.archive_path. """ if self.offline: raise IOError("%s not found, and offline " "mode requested" % self.archive_path) url = self.sources[main_software][1] logger.info("Downloading %s ..." % url) try: msg = urllib.urlretrieve(url, self.archive_path) if msg[1].type == 'text/html': os.unlink(self.archive_path) raise LookupError( 'Wanted version %r not found on server (tried %s)' % ( self.version_wanted, url)) except (tarfile.TarError, IOError): # GR: ContentTooShortError subclasses IOError os.unlink(self.archive_path) raise IOError('The archive does not seem valid: ' + repr(self.archive_path))
[docs] def is_stale_http_head(self): """Tell if the download is stale by doing a HEAD request. Assumes the correct date had been written upon download. This is the same system as in GNU Wget 1.12. It works even if the server does not implement conditional responses such as 304 """ archivestat = os.stat(self.archive_path) length, modified = archivestat.st_size, archivestat.st_mtime url = self.sources[main_software][1] logger.info("Checking if %s if fresh wrt %s", self.archive_path, url) parsed = urlparse(url) if parsed.scheme == 'https': cnx_cls = httplib.HTTPSConnection else: cnx_cls = httplib.HTTPConnection try: cnx = cnx_cls(parsed.netloc) cnx.request('HEAD', parsed.path) # TODO query ? fragment ? res = cnx.getresponse() except IOError: return True if res.status != 200: return True if int(res.getheader('Content-Length')) != length: return True head_modified = res.getheader('Last-Modified') logger.debug("Last-modified from HEAD request: %s", head_modified) if rfc822_time(head_modified) > modified: return True logger.info("No need to re-download %s", self.archive_path)
[docs] def retrieve_main_software(self): """Lookup or fetch the main software. See :class:`MainSoftware` and :class:`BaseRecipe` for explanations. """ source = self.sources[main_software] type_spec = source[0] logger.info('Selected install type: %s', type_spec) if type_spec == 'local': logger.info('Local directory chosen, nothing to do') if self.clean: utils.clean_object_files(self.openerp_dir) elif type_spec == 'downloadable': # download if needed if ((self.archive_path and not os.path.exists(self.archive_path)) or (self.main_http_caching == 'http-head' and self.is_stale_http_head())): self.main_download() logger.info(u'Inspecting %s ...' % self.archive_path) tar = tarfile.open(self.archive_path) first = tar.next() # Everything that follows assumes all tarball members # are inside a directory with an expected name such # as openerp-6.1-1 assert(first.isdir()) extracted_name = first.name.split('/')[0] self.openerp_dir = join(self.parts, extracted_name) # protection against malicious tarballs assert(not os.path.isabs(extracted_name)) assert(self.openerp_dir.startswith(self.parts)) logger.info("Cleaning existing %s", self.openerp_dir) if os.path.exists(self.openerp_dir): shutil.rmtree(self.openerp_dir) logger.info(u'Extracting %s ...' % self.archive_path) self.sandboxed_tar_extract(extracted_name, tar, first=first) tar.close() else: url, rev = source[1] options = dict((k, v) for k, v in self.options.iteritems() if k.startswith(type_spec + '-')) if type_spec == 'git': options['depth'] = options.pop('git-depth', None) options.update(source[2]) if self.clean: options['clean'] = True vcs.get_update(type_spec, self.openerp_dir, url, rev, offline=self.offline, clear_retry=self.clear_retry, **options)
def _register_extra_paths(self): """Add openerp paths into the extra-paths (used in scripts' sys.path). This is useful up to the 6.0 series only, because in later version, the 'openerp' directory is a proper distribution that we develop, with the effect of putting it on the path automatically. """ extra = self.extra_paths if self.major_version < (6, 1): extra.extend((join(self.openerp_dir, 'bin'), join(self.openerp_dir, 'bin', 'addons'))) self.options['extra-paths'] = os.linesep.join(extra)
[docs] def install(self): os.chdir(self.parts) freeze_to = self.options.get('freeze-to') extract_downloads_to = self.options.get('extract-downloads-to') if ((freeze_to is not None or extract_downloads_to is not None) and not self.offline): raise UserError("To freeze a part, you must run offline " "so that there's no modification from what " "you just tested. Please rerun with -o.") if extract_downloads_to is not None and freeze_to is None: freeze_to = os.path.join(extract_downloads_to, 'extracted_from.cfg') self.retrieve_main_software() self.retrieve_addons() self.retrieve_merges() self.install_recipe_requirements() os.chdir(self.openerp_dir) # GR probably not needed any more self.read_openerp_setup() if (self.sources[main_software][0] == 'downloadable' and self.version_wanted == 'latest'): self.nightly_version = self.version_detected.split('-', 1)[1] logger.warn("Detected 'nightly latest version', you may want to " "fix it in your config file for replayability: \n " "version = " + self.dump_nightly_latest_version()) self.finalize_addons_paths() self._register_extra_paths() if self.version_detected is None: raise EnvironmentError('Version of OpenERP could not be detected') self.merge_requirements() self.install_requirements() self._install_startup_scripts() # create the config file if os.path.exists(self.config_path): os.remove(self.config_path) logger.info('Creating config file: %s', os.path.relpath(self.config_path, self.buildout_dir)) self._create_default_config() # modify the config file according to recipe options config = ConfigParser.RawConfigParser() config.read(self.config_path) for recipe_option in self.options: if '.' not in recipe_option: continue section, option = recipe_option.split('.', 1) if not config.has_section(section): config.add_section(section) config.set(section, option, self.options[recipe_option]) with open(self.config_path, 'wb') as configfile: config.write(configfile) if extract_downloads_to: self.extract_downloads_to(extract_downloads_to) if freeze_to: self.freeze_to(freeze_to) return self.openerp_installed
[docs] def dump_nightly_latest_version(self): """After download/analysis of 'nightly latest', give equivalent spec. """ return ' '.join(('nightly', self.nightly_series, self.nightly_version))
[docs] def freeze_to(self, out_config_path): """Create an extension buildout freezing current revisions & versions. """ logger.info("Freezing part %r to config file %r", self.name, out_config_path) out_conf = ConfigParser.ConfigParser() frozen = getattr(self.buildout, '_openerp_recipe_frozen', None) if frozen is None: frozen = self.buildout._openerp_recipe_frozen = set() if out_config_path in frozen: # read configuration started by other recipe out_conf.read(self.make_absolute(out_config_path)) else: self._prepare_frozen_buildout(out_conf) self._freeze_egg_versions(out_conf, 'versions') out_conf.add_section(self.name) addons_option = [] self.local_modifications = [] for local_path, source in self.sources.items(): source_type = source[0] if source_type == 'local': continue if local_path is main_software: if source_type == 'downloadable': self._freeze_downloadable_main_software(out_conf) else: # vcs abspath = self.openerp_dir self.cleanup_openerp_dir() else: abspath = self.make_absolute(local_path) if source_type == 'downloadable': continue required_rev = source[1][1] revision = self._freeze_vcs_source( source_type, abspath, required_rev) # here it would be tempting not to repeat the freeze if # the resulting revision is equal to revision_rev, BUT # if revision_rev itself comes from use of the 'revisons' option # then we could have masking of that in the extended buildout # this could be taken care of for aesthetics by careful use of # += and actually specification of what it should do for # the 'revisions' option. In the meanwhile, let's just play safe if local_path is main_software: addons_option.insert(0, '%s ; main software part' % revision) # actually, that comment will be lost if this is not the # last part (dropped upon reread) else: addons_option.append(' '.join((local_path, revision))) if addons_option: out_conf.set(self.name, 'revisions', os.linesep.join(addons_option)) if self.local_modifications: logger.error( "Uncommitted changes and/or untracked files in: %s" "Unsafe to freeze. Please commit or revert and test again !", os.linesep.join( ['', ''] + [' - ' + p for p in self.local_modifications] + ['', ''])) sys.exit(17) # GR I like that number with open(self.make_absolute(out_config_path), 'w') as out: out_conf.write(out) frozen.add(out_config_path)
def _get_gp_vcs_develops(self): """Return a tuple of (raw, parsed) vcs-extends-develop specifications. """ lines = self.b_options.get( GP_VCS_EXTEND_DEVELOP) if not lines: return () try: import pip.req except ImportError: logger.error("You have vcs-extends-develop distributions " "but pip is not available. That means that " "gp.vcsdevelop is not properly installed. Did " "you ever run that buildout ?") raise return tuple((line, pip.req.parse_editable(line)) for line in option_splitlines(lines)) def _prepare_frozen_buildout(self, conf): """Create the 'buildout' section in conf.""" conf.add_section('buildout') conf.set('buildout', 'extends', self.buildout_cfg_name()) conf.add_section('versions') conf.set('buildout', 'versions', 'versions') # freezing for gp.vcsdevelop extends = [] for raw, parsed in self._get_gp_vcs_develops(): local_path = parsed[0] hash_split = raw.rsplit('#') url = hash_split[0] rev_split = url.rsplit('@', 1) url = rev_split[0] revspec = rev_split[1] if len(rev_split) == 2 else None vcs_type = url.split('+', 1)[0] # vcs-develop process adds .egg-info file (often forgotten in VCS # ignore files) and changes setup.cfg. # For now we'll have to allow local modifications. revision = self._freeze_vcs_source(vcs_type, self.make_absolute(local_path), revspec, pip_compatible=True, allow_local_modification=True) extends.append('%s@%s#%s' % (url, revision, hash_split[1])) conf.set('buildout', GP_VCS_EXTEND_DEVELOP, os.linesep.join(extends)) def _freeze_downloadable_main_software(self, conf): """If needed, sets the main version option in ConfigParser. Currently does not dump the fully resolved URL, since future reproduction may be better done with another URL base holding archived old versions : it's better to let tomorrow logic handle that from higher level information. """ if self.version_wanted == 'latest': conf.set(self.name, 'version', self.dump_nightly_latest_version()) def _freeze_egg_versions(self, conf, section, exclude=()): """Update a ConfigParser section with current working set egg versions. """ versions = dict((name, conf.get(section, name)) for name in conf.options(section)) versions.update((name, egg.version) for name, egg in self.ws.by_key.items() if name not in exclude and egg.precedence != pkg_resources.DEVELOP_DIST ) for name, version in versions.items(): conf.set(section, name, version) # forbidding picked versions if this zc.buildout supports it right away # i.e. we are on zc.buildout >= 2.0 allow_picked = self.options.get('freeze-allow-picked-versions', '') if allow_picked.strip() == 'false': pick_opt = 'allow-picked-versions' if pick_opt in self.b_options: conf.set('buildout', pick_opt, 'false') def _freeze_vcs_source(self, vcs_type, abspath, revspec, pip_compatible=False, allow_local_modification=False): """Return the frozen revision for the state of that VCS source. :param vcs_type: self-explanatory :param abspath: absolute path to local repository :param revspec: the revision specification as required in the buildout configuration (can be suitable in itself and better than what we can produce) :param pip_compatible: if ``True``, a pip compatible revision number is issued. This depends on the precise vcs. :returns: ``None`` if ``revspec`` is already a frozen and reproducible specification. """ repo = vcs.repo(vcs_type, abspath, '') # no need of remote URL if not allow_local_modification and repo.uncommitted_changes(): self.local_modifications.append(abspath) if revspec is not None and repo.is_local_fixed_revision(revspec): return revspec parents = repo.parents(pip_compatible=pip_compatible) if len(parents) > 1: self.local_modifications.append(abspath) return parents[0]
[docs] def extract_downloads_to(self, target_dir, outconf_name='release.cfg'): """Extract anything that has been downloaded to target_dir. This doesn't copy intermediary buildout configurations nor local parts. In the purpose of making a self-contained and offline playable archive, these are assumed to be already taken care of. """ logger.info("Extracting part %r to directory %r and config file %r " "therein.", self.name, target_dir, outconf_name) target_dir = self.make_absolute(target_dir) out_conf = ConfigParser.ConfigParser() all_extracted = getattr(self.buildout, '_openerp_recipe_extracted', None) if all_extracted is None: all_extracted = self.buildout._openerp_recipe_extracted = {} out_config_path = join(target_dir, outconf_name) # GR TODO this will fail if same target dir has been used with # a different outconf_name if target_dir in all_extracted: # read configuration started by other recipe out_conf.read(out_config_path) extracted = all_extracted[target_dir] else: self._prepare_extracted_buildout(out_conf, target_dir) extracted = all_extracted[target_dir] = set() self._freeze_egg_versions(out_conf, 'versions') self._extract_sources(out_conf, target_dir, extracted) with open(out_config_path, 'w') as out: out_conf.write(out)
def _extract_sources(self, out_conf, target_dir, extracted): """Core extraction method. out_conf is a ConfigParser instance to write to extracted is a technical set used to know what targets have already been written by previous parts and store for subsequent ones. """ if not os.path.exists(target_dir): os.mkdir(target_dir) out_conf.add_section(self.name) # remove bzr extra if needed pkg_extras, recipe_cls = self.options['recipe'].split(':') extra_match = re.match(r'(.*?)\[(.*?)\]', pkg_extras) if extra_match is not None: recipe_pkg = extra_match.group(1) extras = set(e.strip() for e in extra_match.group(2).split(',')) extras.discard('bzr') extracted_recipe = recipe_pkg if extras: extracted_recipe += '[%s]' % ','.join(extras) extracted_recipe += ':' + recipe_cls out_conf.set(self.name, 'recipe', extracted_recipe) addons_option = [] for local_path, source in self.sources.items(): source_type = source[0] if local_path is main_software: rel_path = self._extract_main_software(source_type, target_dir, extracted) out_conf.set(self.name, 'version', 'local ' + rel_path) continue # stripping the group option that won't be usefult # and actually harming for extracted buildout conf options = source[2] group = options.pop('group', None) if group: target_local_path = os.path.dirname(local_path) if group != os.path.basename(target_local_path): raise RuntimeError( "Inconsistent configuration that " "should not happen: group=%r, but resulting path %r " "does not have it as its parent" % (group, local_path)) else: target_local_path = local_path addons_line = ['local', target_local_path] addons_line.extend('%s=%s' % (opt, val) for opt, val in options.items()) addons_option.append(' '.join(addons_line)) abspath = self.make_absolute(local_path) if source_type == 'downloadable': shutil.copytree(abspath, os.path.join(target_dir, local_path)) elif source_type != 'local': # vcs self._extract_vcs_source(source_type, abspath, target_dir, local_path, extracted) out_conf.set(self.name, 'addons', os.linesep.join(addons_option)) if self.options.get('revisions'): out_conf.set(self.name, 'revisions', '') # GR hacky way to make a comment for a void value. Indeed, # "revisions = ; comment" is not recognized as an inline comment # because of overall stripping and a need for whitespace before # the semicolon (sigh) out_conf.set(self.name, '; about revisions', "the extended buildout '%s' uses the 'revisions' " "option. The present override disables it " "because it makes no sense after extraction and " "replacement by the " "'local' scheme" % self.buildout_cfg_name()) def _extract_vcs_source(self, vcs_type, repo_path, target_dir, local_path, extracted): """Extract a VCS source. The extracted argument is a set of previously extracted targets. This is because some VCS will refuse an empty directory (bzr does) """ repo_path = self.make_absolute(repo_path) target_path = os.path.join(target_dir, local_path) if not os.path.exists(target_path): os.makedirs(target_path) if target_path in extracted: return repo = vcs.repo(vcs_type, repo_path, '') # no need of remote URL repo.archive(target_path) extracted.add(target_path) def _extract_main_software(self, source_type, target_dir, extracted): """Extract the main software to target_dir and return relative path. As this is for extract_downloads_to, local main software is not extracted (supposed to be taken care of by the tool that does the archival of buildout dir itself). The extracted set avoids extracting twice to same target (refused by some VCSes anyway) """ if not self.openerp_dir.startswith(self.buildout_dir): raise RuntimeError( "Main openerp directory %r outside of buildout " "directory, don't know how to handle that" % self.openerp_dir) local_path = self.openerp_dir[len(self.buildout_dir + os.sep):] target_path = join(target_dir, local_path) if target_path in extracted: return local_path if source_type == 'downloadable': shutil.copytree(self.openerp_dir, target_path) elif source_type != 'local': # see docstring for 'local' self._extract_vcs_source(source_type, self.openerp_dir, target_dir, local_path, extracted) return local_path def _prepare_extracted_buildout(self, conf, target_dir): """Create the 'buildout' section in ``conf``. Also takes care of gp.vcsdevelop driven distributions. In most cases, at this stage, gp.vcsdevelop and regular develop distributions are expressed with absolute paths. This method will make them local in the destination ``conf`` Regular develop distributions pointing outside of buildout directory are kept as is, assuming this has been specified in absolute form in the config file, hence to some resources that are outside of the recipe control, that are therefore expected to be deployed before hand on target systems. """ conf.add_section('buildout') conf.set('buildout', 'extends', self.buildout_cfg_name()) conf.add_section('versions') conf.set('buildout', 'versions', 'versions') develops = set(option_splitlines(self.b_options.get('develop'))) extracted = set() for raw, parsed in self._get_gp_vcs_develops(): local_path = parsed[0] abs_path = self.make_absolute(local_path) vcs_type = raw.split('+', 1)[0] self._extract_vcs_source(vcs_type, abs_path, target_dir, local_path, extracted) develops.add(abs_path) # looks silly, but better for uniformity bdir = os.path.join(self.buildout_dir, '') conf.set('buildout', 'develop', os.linesep.join(d[len(bdir):] if d.startswith(bdir) else d for d in develops)) # remove gp.vcsdevelop from extensions exts = self.buildout['buildout'].get('extensions', '').split() if 'gp.vcsdevelop' in exts: exts.remove('gp.vcsdevelop') conf.set('buildout', 'extensions', '\n'.join(exts)) def _install_script(self, name, content): """Install and register a scripbont with prescribed name and content. Return the script path """ path = join(self.bin_dir, name) f = open(path, 'w') f.write(content) f.close() os.chmod(path, stat.S_IRWXU) self.openerp_installed.append(path) return path def _install_startup_scripts(self): raise NotImplementedError def _create_default_config(self): raise NotImplementedError update = install def _60_fix_root_path(self): """Correction of root path for OpenERP 6.0 pure python install Actual implementation is up to subclasses """ def _60_default_addons_path(self): """Set the default addons path for OpenERP 6.0 pure python install Actual implementation is up to subclasses """ def _default_addons_path(self): """Set the default addons path for OpenERP > 6.0 pure python install Actual implementation is up to subclasses """
[docs] def finalize_addons_paths(self, check_existence=True): """Add implicit paths and serialize in the addons_path option. if check_existence is True, all the paths will be checked for existence. """ opt_key = 'options.addons_path' if opt_key in self.options: raise UserError("In part %r, direct use of %s is prohibited. " "please use addons lines with type 'local' " "instead." % (self.name, opt_key)) if self.major_version <= (6, 0): base_addons = join(self.openerp_dir, 'bin', 'addons') else: base_addons = join(self.openerp_dir, 'openerp', 'addons') if os.path.exists(base_addons): self.addons_paths.insert(0, base_addons) self.insert_odoo_git_addons(base_addons) if check_existence: for path in self.addons_paths: assert os.path.isdir(path), ( "Not a directory: %r (aborting)" % path) self.options['options.addons_path'] = ','.join(self.addons_paths) if self.major_version <= (6, 0): self._60_fix_root_path()
[docs] def insert_odoo_git_addons(self, base_addons): """Insert the standard, non-base addons bundled within Odoo git repo. See `lp:1327756 <https://bugs.launchpad.net/anybox.recipe.openerp/+bug/1327756>`_ These addons are also part of the Github branch for prior versions, therefore we cannot rely on version knowledge; we check for existence instead. If not found (e.g, we are on a nightly for OpenERP <= 7), this method does nothing. The ordering of the different paths of addons is important. When several addons at different paths have the same name, the first of them being found is used. This can be used, for instance, to replace an official addon by another one by placing a different addons' path before the official one. If the official addons' path is already set in the config file (e.g. at the end), it will leave it at the end of the paths list, if it is not set, it will be placed at the beginning just after ``base`` addons' path. Care is taken not to break configurations that corrected this manually with a ``local`` source in the ``addons`` option. :param base_addons: the path to previously detected ``base`` addons, to properly insert right after them """ odoo_git_addons = join(self.openerp_dir, 'addons') if not os.path.isdir(odoo_git_addons): return self.is_git_layout = True addons_paths = self.addons_paths try: insert_at = addons_paths.index(base_addons) + 1 except ValueError: insert_at = 0 try: addons_paths.index(odoo_git_addons) except ValueError: addons_paths.insert(insert_at, odoo_git_addons)
[docs] def cleanup_openerp_dir(self): """Revert local modifications that have been made during installation. These can be, e.g., forbidden by the freeze process.""" # from here we can't guess whether it's 'openerp' or 'odoo'. # Nothing guarantees that this method is called after develop(). # It is in practice now, but one day, the extraction as a separate # script of freeze/extract will become a reality. for proj_name in ('openerp', 'odoo'): egg_info_dir = join(self.openerp_dir, proj_name + '.egg-info') if os.path.exists(egg_info_dir): shutil.rmtree(egg_info_dir) # if setup.py has been rewritten without PIL, it is cleaned # during the process itself
[docs] def buildout_cfg_name(self, argv=None): """Return the name of the config file that's been called. """ # not using optparse because it's not obvious how to tell it to # consider just one option and ignore the others. if argv is None: argv = sys.argv[1:] # -c FILE or --config FILE syntax for opt in ('-c', '--config'): try: i = argv.index(opt) except ValueError: continue else: return argv[i+1] # --config=FILE syntax prefix = "--config=" for a in argv: if a.startswith(prefix): return a[len(prefix):] return 'buildout.cfg'